001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     * 
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     * 
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import org.apache.hadoop.conf.Configuration;
021    import org.apache.oozie.util.Instrumentable;
022    import org.apache.oozie.util.Instrumentation;
023    import org.apache.oozie.util.XLog;
024    import org.apache.oozie.util.XConfiguration;
025    import org.apache.oozie.ErrorCode;
026    
027    import java.io.File;
028    import java.io.FileInputStream;
029    import java.io.IOException;
030    import java.io.InputStream;
031    import java.io.StringWriter;
032    import java.util.HashSet;
033    import java.util.Map;
034    import java.util.Set;
035    import java.util.Arrays;
036    import org.apache.oozie.util.ZKUtils;
037    
038    /**
039     * Built in service that initializes the services configuration.
040     * <p/>
041     * The configuration loading sequence is identical to Hadoop configuration loading sequence.
042     * <p/>
043     * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values
044     * are loaded from a site configuration file from the Oozie configuration directory.
045     * <p/>
046     * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as
047     * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the
048     * initialization of the <code>ConfigurationService</code> fails.
049     * <p/>
050     * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory.
051     * <p/>
052     * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment
053     * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory.
054     * <p/>
055     * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values.
056     * <p/>
057     * The configuration service logs details on how the configuration was loaded as well as what properties were overrode
058     * via system properties settings.
059     */
060    public class ConfigurationService implements Service, Instrumentable {
061        private static final String INSTRUMENTATION_GROUP = "configuration";
062    
063        public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService.";
064    
065        public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties";
066    
067        /**
068         * System property that indicates the configuration directory.
069         */
070        public static final String OOZIE_CONFIG_DIR = "oozie.config.dir";
071    
072    
073        /**
074         * System property that indicates the data directory.
075         */
076        public static final String OOZIE_DATA_DIR = "oozie.data.dir";
077    
078        /**
079         * System property that indicates the name of the site configuration file to load.
080         */
081        public static final String OOZIE_CONFIG_FILE = "oozie.config.file";
082    
083        private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
084        private static final Set<String> CONF_SYS_PROPS = new HashSet<String>();
085    
086        private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
087        private static final Set<String> MASK_PROPS = new HashSet<String>();
088    
089        static {
090    
091            //all this properties are seeded as system properties, no need to log changes
092            IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
093            IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR);
094            IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR);
095            IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE);
096            IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR);
097            IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR);
098            IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE);
099            IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD);
100    
101            CONF_SYS_PROPS.add("oozie.http.hostname");
102            CONF_SYS_PROPS.add("oozie.http.port");
103            CONF_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID);
104    
105            // These properties should be masked when displayed because they contain sensitive info (e.g. password)
106            MASK_PROPS.add(JPAService.CONF_PASSWORD);
107            MASK_PROPS.add("oozie.authentication.signature.secret");
108        }
109    
110        public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml";
111        public static final String SITE_CONFIG_FILE = "oozie-site.xml";
112    
113        private static XLog log = XLog.getLog(ConfigurationService.class);
114    
115        private String configDir;
116        private String configFile;
117    
118        private LogChangesConfiguration configuration;
119    
120        public ConfigurationService() {
121            log = XLog.getLog(ConfigurationService.class);
122        }
123    
124        /**
125         * Initialize the log service.
126         *
127         * @param services services instance.
128         * @throws ServiceException thrown if the log service could not be initialized.
129         */
130        public void init(Services services) throws ServiceException {
131            configDir = getConfigurationDirectory();
132            configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE);
133            if (configFile.contains("/")) {
134                throw new ServiceException(ErrorCode.E0022, configFile);
135            }
136            log.info("Oozie home dir  [{0}]", Services.getOozieHome());
137            log.info("Oozie conf dir  [{0}]", configDir);
138            log.info("Oozie conf file [{0}]", configFile);
139            configFile = new File(configDir, configFile).toString();
140            configuration = loadConf();
141        }
142    
143        public static String getConfigurationDirectory() throws ServiceException {
144            String oozieHome = Services.getOozieHome();
145            String configDir = System.getProperty(OOZIE_CONFIG_DIR, null);
146            File file = configDir == null
147                    ? new File(oozieHome, "conf")
148                    : new File(configDir);
149            if (!file.exists()) {
150                throw new ServiceException(ErrorCode.E0024, configDir);
151            }
152            return file.getPath();
153        }
154    
155        /**
156         * Destroy the configuration service.
157         */
158        public void destroy() {
159            configuration = null;
160        }
161    
162        /**
163         * Return the public interface for configuration service.
164         *
165         * @return {@link ConfigurationService}.
166         */
167        public Class<? extends Service> getInterface() {
168            return ConfigurationService.class;
169        }
170    
171        /**
172         * Return the services configuration.
173         *
174         * @return the services configuration.
175         */
176        public Configuration getConf() {
177            if (configuration == null) {
178                throw new IllegalStateException("Not initialized");
179            }
180            return configuration;
181        }
182    
183        /**
184         * Return Oozie configuration directory.
185         *
186         * @return Oozie configuration directory.
187         */
188        public String getConfigDir() {
189            return configDir;
190        }
191    
192        private InputStream getDefaultConfiguration() throws ServiceException, IOException {
193            ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
194            InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE);
195            if (inputStream == null) {
196                throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE);
197            }
198            return inputStream;
199        }
200    
201        private LogChangesConfiguration loadConf() throws ServiceException {
202            XConfiguration configuration;
203            try {
204                InputStream inputStream = getDefaultConfiguration();
205                configuration = new XConfiguration(inputStream);
206                File file = new File(configFile);
207                if (!file.exists()) {
208                    log.info("Missing site configuration file [{0}]", configFile);
209                }
210                else {
211                    inputStream = new FileInputStream(configFile);
212                    XConfiguration siteConfiguration = new XConfiguration(inputStream);
213                    XConfiguration.injectDefaults(configuration, siteConfiguration);
214                    configuration = siteConfiguration;
215                }
216            }
217            catch (IOException ex) {
218                throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
219            }
220    
221            if (log.isTraceEnabled()) {
222                try {
223                    StringWriter writer = new StringWriter();
224                    for (Map.Entry<String, String> entry : configuration) {
225                        String value = getValue(configuration, entry.getKey());
226                        writer.write(" " + entry.getKey() + " = " + value + "\n");
227                    }
228                    writer.close();
229                    log.trace("Configuration:\n{0}---", writer.toString());
230                }
231                catch (IOException ex) {
232                    throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
233                }
234            }
235    
236            String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
237            if (ignoreSysProps != null) {
238                IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
239            }
240    
241            for (Map.Entry<String, String> entry : configuration) {
242                String sysValue = System.getProperty(entry.getKey());
243                if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
244                    log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
245                    configuration.set(entry.getKey(), sysValue);
246                }
247            }
248            for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
249                String name = (String) entry.getKey();
250                if (!IGNORE_SYS_PROPS.contains(name)) {
251                    if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
252                        if (configuration.get(name) == null) {
253                            log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
254                        }
255                    }
256                }
257            }
258    
259            //Backward compatible, we should still support -Dparam.
260            for (String key : CONF_SYS_PROPS) {
261                String sysValue = System.getProperty(key);
262                if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) {
263                    log.info("Overriding configuration with system property. Key [{0}], Value [{1}] ", key, sysValue);
264                    configuration.set(key, sysValue);
265                }
266            }
267    
268            return new LogChangesConfiguration(configuration);
269        }
270    
271        private class LogChangesConfiguration extends XConfiguration {
272    
273            public LogChangesConfiguration(Configuration conf) {
274                for (Map.Entry<String, String> entry : conf) {
275                    if (get(entry.getKey()) == null) {
276                        setValue(entry.getKey(), entry.getValue());
277                    }
278                }
279            }
280    
281            public String[] getStrings(String name) {
282                String s = get(name);
283                return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0];
284            }
285    
286            public String get(String name, String defaultValue) {
287                String value = get(name);
288                if (value == null) {
289                    boolean maskValue = MASK_PROPS.contains(name);
290                    value = defaultValue;
291                    String logValue = (maskValue) ? "**MASKED**" : defaultValue;
292                    log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, logValue);
293                }
294                return value;
295            }
296    
297            public void set(String name, String value) {
298                setValue(name, value);
299                boolean maskValue = MASK_PROPS.contains(name);
300                value = (maskValue) ? "**MASKED**" : value;
301                log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value);
302            }
303    
304            private void setValue(String name, String value) {
305                super.set(name, value);
306            }
307    
308        }
309    
310        /**
311         * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and
312         * config file used.
313         *
314         * @param instr instrumentation to use.
315         */
316        public void instrument(Instrumentation instr) {
317            instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() {
318                public String getValue() {
319                    return configDir;
320                }
321            });
322            instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
323                public String getValue() {
324                    return configFile;
325                }
326            });
327        }
328    
329        /**
330         * Return a configuration with all sensitive values masked.
331         *
332         * @return masked configuration.
333         */
334        public Configuration getMaskedConfiguration() {
335            XConfiguration maskedConf = new XConfiguration();
336            Configuration conf = getConf();
337            for (Map.Entry<String, String> entry : conf) {
338                String name = entry.getKey();
339                String value = getValue(conf, name);
340                maskedConf.set(name, value);
341            }
342            return maskedConf;
343        }
344    
345        private String getValue(Configuration config, String key) {
346            String value;
347            if (MASK_PROPS.contains(key)) {
348                value = "**MASKED**";
349            }
350            else {
351                value = config.get(key);
352            }
353            return value;
354        }
355    }