001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.service;
019
020 import org.apache.hadoop.conf.Configuration;
021 import org.apache.oozie.util.Instrumentable;
022 import org.apache.oozie.util.Instrumentation;
023 import org.apache.oozie.util.XLog;
024 import org.apache.oozie.util.XConfiguration;
025 import org.apache.oozie.ErrorCode;
026
027 import java.io.File;
028 import java.io.FileInputStream;
029 import java.io.IOException;
030 import java.io.InputStream;
031 import java.io.StringWriter;
032 import java.util.HashSet;
033 import java.util.Map;
034 import java.util.Set;
035 import java.util.Arrays;
036 import org.apache.oozie.util.ZKUtils;
037
038 /**
039 * Built in service that initializes the services configuration.
040 * <p/>
041 * The configuration loading sequence is identical to Hadoop configuration loading sequence.
042 * <p/>
043 * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values
044 * are loaded from a site configuration file from the Oozie configuration directory.
045 * <p/>
046 * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as
047 * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the
048 * initialization of the <code>ConfigurationService</code> fails.
049 * <p/>
050 * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory.
051 * <p/>
052 * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment
053 * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory.
054 * <p/>
055 * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values.
056 * <p/>
057 * The configuration service logs details on how the configuration was loaded as well as what properties were overrode
058 * via system properties settings.
059 */
060 public class ConfigurationService implements Service, Instrumentable {
061 private static final String INSTRUMENTATION_GROUP = "configuration";
062
063 public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService.";
064
065 public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties";
066
067 /**
068 * System property that indicates the configuration directory.
069 */
070 public static final String OOZIE_CONFIG_DIR = "oozie.config.dir";
071
072
073 /**
074 * System property that indicates the data directory.
075 */
076 public static final String OOZIE_DATA_DIR = "oozie.data.dir";
077
078 /**
079 * System property that indicates the name of the site configuration file to load.
080 */
081 public static final String OOZIE_CONFIG_FILE = "oozie.config.file";
082
083 private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
084 private static final Set<String> CONF_SYS_PROPS = new HashSet<String>();
085
086 private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
087 private static final Set<String> MASK_PROPS = new HashSet<String>();
088
089 static {
090
091 //all this properties are seeded as system properties, no need to log changes
092 IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
093 IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR);
094 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR);
095 IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE);
096 IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR);
097 IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR);
098 IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE);
099 IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD);
100
101 CONF_SYS_PROPS.add("oozie.http.hostname");
102 CONF_SYS_PROPS.add("oozie.http.port");
103 CONF_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID);
104
105 // These properties should be masked when displayed because they contain sensitive info (e.g. password)
106 MASK_PROPS.add(JPAService.CONF_PASSWORD);
107 MASK_PROPS.add("oozie.authentication.signature.secret");
108 }
109
110 public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml";
111 public static final String SITE_CONFIG_FILE = "oozie-site.xml";
112
113 private static XLog log = XLog.getLog(ConfigurationService.class);
114
115 private String configDir;
116 private String configFile;
117
118 private LogChangesConfiguration configuration;
119
120 public ConfigurationService() {
121 log = XLog.getLog(ConfigurationService.class);
122 }
123
124 /**
125 * Initialize the log service.
126 *
127 * @param services services instance.
128 * @throws ServiceException thrown if the log service could not be initialized.
129 */
130 public void init(Services services) throws ServiceException {
131 configDir = getConfigurationDirectory();
132 configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE);
133 if (configFile.contains("/")) {
134 throw new ServiceException(ErrorCode.E0022, configFile);
135 }
136 log.info("Oozie home dir [{0}]", Services.getOozieHome());
137 log.info("Oozie conf dir [{0}]", configDir);
138 log.info("Oozie conf file [{0}]", configFile);
139 configFile = new File(configDir, configFile).toString();
140 configuration = loadConf();
141 }
142
143 public static String getConfigurationDirectory() throws ServiceException {
144 String oozieHome = Services.getOozieHome();
145 String configDir = System.getProperty(OOZIE_CONFIG_DIR, null);
146 File file = configDir == null
147 ? new File(oozieHome, "conf")
148 : new File(configDir);
149 if (!file.exists()) {
150 throw new ServiceException(ErrorCode.E0024, configDir);
151 }
152 return file.getPath();
153 }
154
155 /**
156 * Destroy the configuration service.
157 */
158 public void destroy() {
159 configuration = null;
160 }
161
162 /**
163 * Return the public interface for configuration service.
164 *
165 * @return {@link ConfigurationService}.
166 */
167 public Class<? extends Service> getInterface() {
168 return ConfigurationService.class;
169 }
170
171 /**
172 * Return the services configuration.
173 *
174 * @return the services configuration.
175 */
176 public Configuration getConf() {
177 if (configuration == null) {
178 throw new IllegalStateException("Not initialized");
179 }
180 return configuration;
181 }
182
183 /**
184 * Return Oozie configuration directory.
185 *
186 * @return Oozie configuration directory.
187 */
188 public String getConfigDir() {
189 return configDir;
190 }
191
192 private InputStream getDefaultConfiguration() throws ServiceException, IOException {
193 ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
194 InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE);
195 if (inputStream == null) {
196 throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE);
197 }
198 return inputStream;
199 }
200
201 private LogChangesConfiguration loadConf() throws ServiceException {
202 XConfiguration configuration;
203 try {
204 InputStream inputStream = getDefaultConfiguration();
205 configuration = new XConfiguration(inputStream);
206 File file = new File(configFile);
207 if (!file.exists()) {
208 log.info("Missing site configuration file [{0}]", configFile);
209 }
210 else {
211 inputStream = new FileInputStream(configFile);
212 XConfiguration siteConfiguration = new XConfiguration(inputStream);
213 XConfiguration.injectDefaults(configuration, siteConfiguration);
214 configuration = siteConfiguration;
215 }
216 }
217 catch (IOException ex) {
218 throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
219 }
220
221 if (log.isTraceEnabled()) {
222 try {
223 StringWriter writer = new StringWriter();
224 for (Map.Entry<String, String> entry : configuration) {
225 String value = getValue(configuration, entry.getKey());
226 writer.write(" " + entry.getKey() + " = " + value + "\n");
227 }
228 writer.close();
229 log.trace("Configuration:\n{0}---", writer.toString());
230 }
231 catch (IOException ex) {
232 throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
233 }
234 }
235
236 String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
237 if (ignoreSysProps != null) {
238 IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
239 }
240
241 for (Map.Entry<String, String> entry : configuration) {
242 String sysValue = System.getProperty(entry.getKey());
243 if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
244 log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
245 configuration.set(entry.getKey(), sysValue);
246 }
247 }
248 for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
249 String name = (String) entry.getKey();
250 if (!IGNORE_SYS_PROPS.contains(name)) {
251 if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
252 if (configuration.get(name) == null) {
253 log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
254 }
255 }
256 }
257 }
258
259 //Backward compatible, we should still support -Dparam.
260 for (String key : CONF_SYS_PROPS) {
261 String sysValue = System.getProperty(key);
262 if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) {
263 log.info("Overriding configuration with system property. Key [{0}], Value [{1}] ", key, sysValue);
264 configuration.set(key, sysValue);
265 }
266 }
267
268 return new LogChangesConfiguration(configuration);
269 }
270
271 private class LogChangesConfiguration extends XConfiguration {
272
273 public LogChangesConfiguration(Configuration conf) {
274 for (Map.Entry<String, String> entry : conf) {
275 if (get(entry.getKey()) == null) {
276 setValue(entry.getKey(), entry.getValue());
277 }
278 }
279 }
280
281 public String[] getStrings(String name) {
282 String s = get(name);
283 return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0];
284 }
285
286 public String get(String name, String defaultValue) {
287 String value = get(name);
288 if (value == null) {
289 boolean maskValue = MASK_PROPS.contains(name);
290 value = defaultValue;
291 String logValue = (maskValue) ? "**MASKED**" : defaultValue;
292 log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, logValue);
293 }
294 return value;
295 }
296
297 public void set(String name, String value) {
298 setValue(name, value);
299 boolean maskValue = MASK_PROPS.contains(name);
300 value = (maskValue) ? "**MASKED**" : value;
301 log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value);
302 }
303
304 private void setValue(String name, String value) {
305 super.set(name, value);
306 }
307
308 }
309
310 /**
311 * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and
312 * config file used.
313 *
314 * @param instr instrumentation to use.
315 */
316 public void instrument(Instrumentation instr) {
317 instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() {
318 public String getValue() {
319 return configDir;
320 }
321 });
322 instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
323 public String getValue() {
324 return configFile;
325 }
326 });
327 }
328
329 /**
330 * Return a configuration with all sensitive values masked.
331 *
332 * @return masked configuration.
333 */
334 public Configuration getMaskedConfiguration() {
335 XConfiguration maskedConf = new XConfiguration();
336 Configuration conf = getConf();
337 for (Map.Entry<String, String> entry : conf) {
338 String name = entry.getKey();
339 String value = getValue(conf, name);
340 maskedConf.set(name, value);
341 }
342 return maskedConf;
343 }
344
345 private String getValue(Configuration config, String key) {
346 String value;
347 if (MASK_PROPS.contains(key)) {
348 value = "**MASKED**";
349 }
350 else {
351 value = config.get(key);
352 }
353 return value;
354 }
355 }