001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.util;
019    
020    import java.io.File;
021    import java.io.IOException;
022    import java.io.Writer;
023    import java.util.ArrayList;
024    import java.util.Calendar;
025    import java.util.Collections;
026    import java.util.Date;
027    import java.util.regex.Matcher;
028    import java.util.regex.Pattern;
029    import java.io.BufferedReader;
030    
031    /**
032     * XLogStreamer streams the given log file to writer after applying the given filter.
033     */
034    public class XLogStreamer {
035        private static XLog LOG = XLog.getLog(XLogStreamer.class);
036        private String logFile;
037        private String logPath;
038        private XLogFilter logFilter;
039        private long logRotation;
040    
041        public XLogStreamer(XLogFilter logFilter, String logPath, String logFile, long logRotationSecs) {
042            this.logFilter = logFilter;
043            if (logFile == null) {
044                logFile = "oozie-app.log";
045            }
046            this.logFile = logFile;
047            this.logPath = logPath;
048            this.logRotation = logRotationSecs * 1000l;
049        }
050    
051        /**
052         * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after
053         * applying the filters.
054         *
055         * @param writer
056         * @param startTime
057         * @param endTime
058         * @throws IOException
059         */
060        public void streamLog(Writer writer, Date startTime, Date endTime, int bufferLen) throws IOException {
061            // Get a Reader for the log file(s)
062            BufferedReader reader = new BufferedReader(getReader(startTime, endTime));
063            try {
064                if(logFilter.isDebugMode()){
065                    writer.write(logFilter.getDebugMessage());
066                }
067                // Process the entire logs from the reader using the logFilter
068                new TimestampedMessageParser(reader, logFilter).processRemaining(writer, bufferLen);
069            }
070            finally {
071                reader.close();
072            }
073        }
074    
075        /**
076         * Returns a BufferedReader configured to read the log files based on the given startTime and endTime.
077         *
078         * @param startTime
079         * @param endTime
080         * @return A BufferedReader for the log files
081         * @throws IOException
082         */
083    
084        private MultiFileReader getReader(Date startTime, Date endTime) throws IOException {
085            logFilter.calculateAndValidateDateRange(startTime, endTime);
086            return new MultiFileReader(getFileList(logFilter.getStartDate(), logFilter.getEndDate()));
087        }
088    
089        public BufferedReader makeReader(Date startTime, Date endTime) throws IOException {
090            return new BufferedReader(getReader(startTime,endTime));
091        }
092    
093        /**
094         * Gets the log file list for specific date range.
095         *
096         * @param startTime the start time
097         * @param endTime the end time
098         * @return log file list
099         * @throws IOException Signals that an I/O exception has occurred.
100         */
101        private ArrayList<File> getFileList(Date startTime, Date endTime) throws IOException {
102            long startTimeMillis = 0;
103            long endTimeMillis;
104            if (startTime != null) {
105                startTimeMillis = startTime.getTime();
106            }
107            if (endTime == null) {
108                endTimeMillis = System.currentTimeMillis();
109            }
110            else {
111                endTimeMillis = endTime.getTime();
112            }
113            File dir = new File(logPath);
114            return getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile);
115        }
116    
117        /**
118         * File along with the modified time which will be used to sort later.
119         */
120        public class FileInfo implements Comparable<FileInfo> {
121            File file;
122            long modTime;
123    
124            public FileInfo(File file, long modTime) {
125                this.file = file;
126                this.modTime = modTime;
127            }
128    
129            public File getFile() {
130                return file;
131            }
132    
133            public long getModTime() {
134                return modTime;
135            }
136    
137            public int compareTo(FileInfo fileInfo) {
138                long diff = this.modTime - fileInfo.modTime;
139                if (diff > 0) {
140                    return 1;
141                }
142                else if (diff < 0) {
143                    return -1;
144                }
145                else {
146                    return 0;
147                }
148            }
149        }
150    
151        /**
152         * Gets the file list that will have the logs between startTime and endTime.
153         *
154         * @param dir
155         * @param startTime
156         * @param endTime
157         * @param logRotationTime
158         * @param logFile
159         * @return List of files to be streamed
160         */
161        private ArrayList<File> getFileList(File dir, long startTime, long endTime, long logRotationTime, String logFile) {
162            String[] children = dir.list();
163            ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
164            if (children == null) {
165                return new ArrayList<File>();
166            }
167            else {
168                for (int i = 0; i < children.length; i++) {
169                    String fileName = children[i];
170                    if (!fileName.startsWith(logFile) && !fileName.equals(logFile)) {
171                        continue;
172                    }
173                    File file = new File(dir.getAbsolutePath(), fileName);
174                    if (fileName.endsWith(".gz")) {
175                        long gzFileCreationTime = getGZFileCreationTime(fileName, startTime, endTime);
176                        if (gzFileCreationTime != -1) {
177                            fileList.add(new FileInfo(file, gzFileCreationTime));
178                        }
179                        continue;
180                    }
181                    long modTime = file.lastModified();
182                    if (modTime < startTime) {
183                        continue;
184                    }
185                    if (modTime / logRotationTime > (endTime / logRotationTime + 1)) {
186                        continue;
187                    }
188                    fileList.add(new FileInfo(file, modTime));
189                }
190            }
191            Collections.sort(fileList);
192            ArrayList<File> files = new ArrayList<File>(fileList.size());
193            for (FileInfo info : fileList) {
194                files.add(info.getFile());
195            }
196            return files;
197        }
198    
199        /**
200         * This pattern matches the end of a gzip filename to have a format like "-YYYY-MM-dd-HH.gz" with capturing groups for each part
201         * of the date
202         */
203        public static final Pattern gzTimePattern = Pattern.compile(".*-(\\d\\d\\d\\d)-(\\d\\d)-(\\d\\d)-(\\d\\d)\\.gz");
204    
205        /**
206         * Returns the creation time of the .gz archive if it is relevant to the job
207         *
208         * @param fileName
209         * @param startTime
210         * @param endTime
211         * @return Modification time of .gz file after checking if it is relevant to the job
212         */
213        private long getGZFileCreationTime(String fileName, long startTime, long endTime) {
214            // Default return value of -1 to exclude the file
215            long returnVal = -1;
216    
217            // Include oozie.log as oozie.log.gz if it is accidentally GZipped
218            if (fileName.equals("oozie.log.gz")) {
219                LOG.warn("oozie.log has been GZipped, which is unexpected");
220                // Return a value other than -1 to include the file in list
221                returnVal = 0;
222            } else {
223                Matcher m = gzTimePattern.matcher(fileName);
224                if (m.matches() && m.groupCount() == 4) {
225                    int year = Integer.parseInt(m.group(1));
226                    int month = Integer.parseInt(m.group(2));
227                    int day = Integer.parseInt(m.group(3));
228                    int hour = Integer.parseInt(m.group(4));
229                    int minute = 0;
230                    Calendar calendarEntry = Calendar.getInstance();
231                    calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August)
232                    long logFileStartTime = calendarEntry.getTimeInMillis();
233                    long milliSecondsPerHour = 3600000;
234                    long logFileEndTime = logFileStartTime + milliSecondsPerHour;
235                    /*  To check whether the log content is there in the initial or later part of the log file or
236                        the log content is contained entirely within this log file or
237                        the entire log file contains the event log where the event spans across hours
238                    */
239                    if ((startTime >= logFileStartTime && startTime <= logFileEndTime)
240                            || (endTime >= logFileStartTime && endTime <= logFileEndTime)
241                            || (startTime <= logFileStartTime && endTime >= logFileEndTime)) {
242                        returnVal = logFileStartTime;
243                    }
244                } else {
245                    LOG.debug("Filename " + fileName + " does not match the expected format");
246                    returnVal = -1;
247                }
248            }
249            return returnVal;
250        }
251    }