001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     * 
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     * 
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import org.apache.commons.logging.LogFactory;
021    import org.apache.log4j.LogManager;
022    import org.apache.log4j.PropertyConfigurator;
023    import org.apache.oozie.util.XLogFilter;
024    import org.apache.oozie.util.Instrumentable;
025    import org.apache.oozie.util.Instrumentation;
026    import org.apache.oozie.util.XLog;
027    import org.apache.oozie.util.XConfiguration;
028    import org.apache.oozie.BuildInfo;
029    import org.apache.oozie.ErrorCode;
030    import org.apache.hadoop.conf.Configuration;
031    
032    import java.io.File;
033    import java.io.FileInputStream;
034    import java.io.IOException;
035    import java.io.InputStream;
036    import java.net.URL;
037    import java.util.Properties;
038    import java.util.Map;
039    import java.util.regex.Pattern;
040    
041    /**
042     * Built-in service that initializes and manages Logging via Log4j.
043     * <p/>
044     * Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>.
045     * <p/>
046     * The file name can be changed by setting the Java System property <code>oozie.log4j.file</code>.
047     * <p/>
048     * The Log4j configuration files must be a properties file.
049     * <p/>
050     * The Log4j configuration file is first looked in the Oozie configuration directory see {@link ConfigurationService}.
051     * If the file is not found there, it is looked in the classpath.
052     * <p/>
053     * If the Log4j configuration file is loaded from Oozie configuration directory, automatic reloading is enabled.
054     * <p/>
055     * If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled.
056     * <p/>
057     * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>. The default
058     * value is 10 seconds.
059     * <p>
060     * <p>
061     * Unlike most of the other Services, XLogService isn't easily overridable because Services depends on XLogService being available
062     */
063    public class XLogService implements Service, Instrumentable {
064        private static final String INSTRUMENTATION_GROUP = "logging";
065    
066        /**
067         * System property that indicates the logs directory.
068         */
069        public static final String OOZIE_LOG_DIR = "oozie.log.dir";
070    
071        /**
072         * System property that indicates the log4j configuration file to load.
073         */
074        public static final String LOG4J_FILE = "oozie.log4j.file";
075    
076        /**
077         * System property that indicates the reload interval of the configuration file.
078         */
079        public static final String LOG4J_RELOAD = "oozie.log4j.reload";
080    
081        /**
082         * Default value for the log4j configuration file if {@link #LOG4J_FILE} is not set.
083         */
084        public static final String DEFAULT_LOG4J_PROPERTIES = "oozie-log4j.properties";
085    
086        /**
087         * Default value for the reload interval if {@link #LOG4J_RELOAD} is not set.
088         */
089        public static final String DEFAULT_RELOAD_INTERVAL = "10";
090    
091        private XLog log;
092        private long interval;
093        private boolean fromClasspath;
094        private String log4jFileName;
095        private boolean logOverWS = true;
096    
097        private static final String STARTUP_MESSAGE = "{E}"
098                + " ******************************************************************************* {E}"
099                + "  STARTUP MSG: Oozie BUILD_VERSION [{0}] compiled by [{1}] on [{2}]{E}"
100                + "  STARTUP MSG:       revision [{3}]@[{4}]{E}"
101                + "*******************************************************************************";
102    
103        private String oozieLogPath;
104        private String oozieLogName;
105        private int oozieLogRotation = -1;
106    
107        public XLogService() {
108        }
109        
110        public String getOozieLogPath() {
111            return oozieLogPath;
112        }
113        
114        public String getOozieLogName() {
115            return oozieLogName;
116        }
117    
118        /**
119         * Initialize the log service.
120         *
121         * @param services services instance.
122         * @throws ServiceException thrown if the log service could not be initialized.
123         */
124        public void init(Services services) throws ServiceException {
125            String oozieHome = Services.getOozieHome();
126            String oozieLogs = System.getProperty(OOZIE_LOG_DIR, oozieHome + "/logs");
127            System.setProperty(OOZIE_LOG_DIR, oozieLogs);
128            try {
129                LogManager.resetConfiguration();
130                log4jFileName = System.getProperty(LOG4J_FILE, DEFAULT_LOG4J_PROPERTIES);
131                if (log4jFileName.contains("/")) {
132                    throw new ServiceException(ErrorCode.E0011, log4jFileName);
133                }
134                if (!log4jFileName.endsWith(".properties")) {
135                    throw new ServiceException(ErrorCode.E0012, log4jFileName);
136                }
137                String configPath = ConfigurationService.getConfigurationDirectory();
138                File log4jFile = new File(configPath, log4jFileName);
139                if (log4jFile.exists()) {
140                    fromClasspath = false;
141                }
142                else {
143                    ClassLoader cl = Thread.currentThread().getContextClassLoader();
144                    URL log4jUrl = cl.getResource(log4jFileName);
145                    if (log4jUrl == null) {
146                        throw new ServiceException(ErrorCode.E0013, log4jFileName, configPath);
147                    }
148                    fromClasspath = true;
149                }
150    
151                if (fromClasspath) {
152                    ClassLoader cl = Thread.currentThread().getContextClassLoader();
153                    URL log4jUrl = cl.getResource(log4jFileName);
154                    PropertyConfigurator.configure(log4jUrl);
155                }
156                else {
157                    interval = Long.parseLong(System.getProperty(LOG4J_RELOAD, DEFAULT_RELOAD_INTERVAL));
158                    PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000);
159                }
160    
161                log = new XLog(LogFactory.getLog(getClass()));
162    
163                log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION),
164                        BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo()
165                                .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty(
166                                BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VC_URL));
167    
168                String from = (fromClasspath) ? "CLASSPATH" : configPath;
169                String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec";
170                log.info("Log4j configuration file [{0}]", log4jFileName);
171                log.info("Log4j configuration file loaded from [{0}]", from);
172                log.info("Log4j reload interval [{0}]", reload);
173    
174                XLog.Info.reset();
175                XLog.Info.defineParameter(USER);
176                XLog.Info.defineParameter(GROUP);
177                XLogFilter.reset();
178                XLogFilter.defineParameter(USER);
179                XLogFilter.defineParameter(GROUP);
180    
181                // Getting configuration for oozie log via WS
182                ClassLoader cl = Thread.currentThread().getContextClassLoader();
183                InputStream is = (fromClasspath) ? cl.getResourceAsStream(log4jFileName) : new FileInputStream(log4jFile);
184                extractInfoForLogWebService(is);
185            }
186            catch (IOException ex) {
187                throw new ServiceException(ErrorCode.E0010, ex.getMessage(), ex);
188            }
189        }
190    
191        private void extractInfoForLogWebService(InputStream is) throws IOException {
192            logOverWS = true;
193            Properties props = new Properties();
194            props.load(is);
195    
196            Configuration conf = new XConfiguration();
197            for (Map.Entry entry : props.entrySet()) {
198                conf.set((String) entry.getKey(), (String) entry.getValue());
199            }
200            String logFile = conf.get("log4j.appender.oozie.File");
201            if (logFile == null) {
202                log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' "
203                        + "appender");
204                logOverWS = false;
205            }
206            else {
207                logFile = logFile.trim();
208                int i = logFile.lastIndexOf("/");
209                if (i == -1) {
210                    log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender",
211                            logFile);
212                    logOverWS = false;
213                }
214                else {
215                    String appenderClass = conf.get("log4j.appender.oozie");
216                    if (appenderClass == null) {
217                        log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie]");
218                        logOverWS = false;
219                    }
220                    else if (appenderClass.equals("org.apache.log4j.DailyRollingFileAppender")) {
221                        String pattern = conf.get("log4j.appender.oozie.DatePattern");
222                        if (pattern == null) {
223                            log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie.DatePattern]");
224                            logOverWS = false;
225                        }
226                        else {
227                            pattern = pattern.trim();
228                            if (pattern.endsWith("HH")) {
229                                oozieLogRotation = 60 * 60;
230                            }
231                            else if (pattern.endsWith("dd")) {
232                                    oozieLogRotation = 60 * 60 * 24;
233                            }
234                            else {
235                                log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'",
236                                        pattern);
237                                logOverWS = false;
238                            }
239                            if (oozieLogRotation > 0) {
240                                oozieLogPath = logFile.substring(0, i);
241                                oozieLogName = logFile.substring(i + 1);
242                            }
243                        }
244                    }
245                    else if (appenderClass.equals("org.apache.log4j.rolling.RollingFileAppender")) {
246                        String pattern = conf.get("log4j.appender.oozie.RollingPolicy.FileNamePattern");
247                        if (pattern == null) {
248                            log.warn("Oozie WS log will be disabled, missing property "
249                                    + "[log4j.appender.oozie.RollingPolicy.FileNamePattern]");
250                            logOverWS = false;
251                        }
252                        else {
253                            pattern = pattern.trim();
254                            if (pattern.matches(Pattern.quote(logFile) + ".*-%d\\{yyyy-MM-dd-HH\\}(\\.gz)?")) {
255                                oozieLogRotation = 60 * 60;
256                            }
257                            else {
258                                log.warn("Oozie WS log will be disabled, RollingPolicy.FileNamePattern [{0}] should end with " 
259                                        + "'-%d{yyyy-MM-dd-HH}' or '-%d{yyyy-MM-dd-HH}.gz' and also start with the value of "
260                                        + "log4j.appender.oozie.File [{1}]", pattern, logFile);
261                                logOverWS = false;
262                            }
263                            if (oozieLogRotation > 0) {
264                                oozieLogPath = logFile.substring(0, i);
265                                oozieLogName = logFile.substring(i + 1);
266                            }
267                        }
268                    }
269                    else {
270                        log.warn("Oozie WS log will be disabled, log4j.appender.oozie [" + appenderClass + "] should be "
271                                + "either org.apache.log4j.DailyRollingFileAppender or org.apache.log4j.rolling.RollingFileAppender "
272                                + "to enable it");
273                        logOverWS = false;
274                    }
275                }
276            }
277        }
278    
279        /**
280         * Destroy the log service.
281         */
282        public void destroy() {
283            LogManager.shutdown();
284            XLog.Info.reset();
285            XLogFilter.reset();
286        }
287    
288        /**
289         * Server log info constant.
290         */
291        public static final String SERVER = "SERVER";
292    
293        /**
294         * Group log info constant.
295         */
296        public static final String USER = "USER";
297    
298        /**
299         * Group log info constant.
300         */
301        public static final String GROUP = "GROUP";
302    
303        /**
304         * Return the public interface for log service.
305         *
306         * @return {@link XLogService}.
307         */
308        public Class<? extends Service> getInterface() {
309            return XLogService.class;
310        }
311    
312        /**
313         * Instruments the log service.
314         * <p/>
315         * It sets instrumentation variables indicating the config file, reload interval and if loaded from the classpath.
316         *
317         * @param instr instrumentation to use.
318         */
319        public void instrument(Instrumentation instr) {
320            instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
321                public String getValue() {
322                    return log4jFileName;
323                }
324            });
325            instr.addVariable(INSTRUMENTATION_GROUP, "reload.interval", new Instrumentation.Variable<Long>() {
326                public Long getValue() {
327                    return interval;
328                }
329            });
330            instr.addVariable(INSTRUMENTATION_GROUP, "from.classpath", new Instrumentation.Variable<Boolean>() {
331                public Boolean getValue() {
332                    return fromClasspath;
333                }
334            });
335            instr.addVariable(INSTRUMENTATION_GROUP, "log.over.web-service", new Instrumentation.Variable<Boolean>() {
336                public Boolean getValue() {
337                    return logOverWS;
338                }
339            });
340        }
341    
342        boolean getLogOverWS() {
343            return logOverWS;
344        }
345    
346        int getOozieLogRotation() {
347            return oozieLogRotation;
348        }
349    
350        String getLog4jProperties() {
351            return log4jFileName;
352        }
353    
354        boolean getFromClasspath() {
355            return fromClasspath;
356        }
357    
358    }