001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.util;
019
020 import java.io.File;
021 import java.io.IOException;
022 import java.io.Writer;
023 import java.util.ArrayList;
024 import java.util.Calendar;
025 import java.util.Collections;
026 import java.util.Date;
027 import java.util.regex.Matcher;
028 import java.util.regex.Pattern;
029 import java.io.BufferedReader;
030
031 /**
032 * XLogStreamer streams the given log file to writer after applying the given filter.
033 */
034 public class XLogStreamer {
035 private static XLog LOG = XLog.getLog(XLogStreamer.class);
036 private String logFile;
037 private String logPath;
038 private XLogFilter logFilter;
039 private long logRotation;
040
041 public XLogStreamer(XLogFilter logFilter, String logPath, String logFile, long logRotationSecs) {
042 this.logFilter = logFilter;
043 if (logFile == null) {
044 logFile = "oozie-app.log";
045 }
046 this.logFile = logFile;
047 this.logPath = logPath;
048 this.logRotation = logRotationSecs * 1000l;
049 }
050
051 /**
052 * Gets the files that are modified between startTime and endTime in the given logPath and streams the log after
053 * applying the filters.
054 *
055 * @param writer
056 * @param startTime
057 * @param endTime
058 * @throws IOException
059 */
060 public void streamLog(Writer writer, Date startTime, Date endTime, int bufferLen) throws IOException {
061 // Get a Reader for the log file(s)
062 BufferedReader reader = new BufferedReader(getReader(startTime, endTime));
063 try {
064 if(logFilter.isDebugMode()){
065 writer.write(logFilter.getDebugMessage());
066 }
067 // Process the entire logs from the reader using the logFilter
068 new TimestampedMessageParser(reader, logFilter).processRemaining(writer, bufferLen);
069 }
070 finally {
071 reader.close();
072 }
073 }
074
075 /**
076 * Returns a BufferedReader configured to read the log files based on the given startTime and endTime.
077 *
078 * @param startTime
079 * @param endTime
080 * @return A BufferedReader for the log files
081 * @throws IOException
082 */
083
084 private MultiFileReader getReader(Date startTime, Date endTime) throws IOException {
085 logFilter.calculateAndValidateDateRange(startTime, endTime);
086 return new MultiFileReader(getFileList(logFilter.getStartDate(), logFilter.getEndDate()));
087 }
088
089 public BufferedReader makeReader(Date startTime, Date endTime) throws IOException {
090 return new BufferedReader(getReader(startTime,endTime));
091 }
092
093 /**
094 * Gets the log file list for specific date range.
095 *
096 * @param startTime the start time
097 * @param endTime the end time
098 * @return log file list
099 * @throws IOException Signals that an I/O exception has occurred.
100 */
101 private ArrayList<File> getFileList(Date startTime, Date endTime) throws IOException {
102 long startTimeMillis = 0;
103 long endTimeMillis;
104 if (startTime != null) {
105 startTimeMillis = startTime.getTime();
106 }
107 if (endTime == null) {
108 endTimeMillis = System.currentTimeMillis();
109 }
110 else {
111 endTimeMillis = endTime.getTime();
112 }
113 File dir = new File(logPath);
114 return getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile);
115 }
116
117 /**
118 * File along with the modified time which will be used to sort later.
119 */
120 public class FileInfo implements Comparable<FileInfo> {
121 File file;
122 long modTime;
123
124 public FileInfo(File file, long modTime) {
125 this.file = file;
126 this.modTime = modTime;
127 }
128
129 public File getFile() {
130 return file;
131 }
132
133 public long getModTime() {
134 return modTime;
135 }
136
137 public int compareTo(FileInfo fileInfo) {
138 long diff = this.modTime - fileInfo.modTime;
139 if (diff > 0) {
140 return 1;
141 }
142 else if (diff < 0) {
143 return -1;
144 }
145 else {
146 return 0;
147 }
148 }
149 }
150
151 /**
152 * Gets the file list that will have the logs between startTime and endTime.
153 *
154 * @param dir
155 * @param startTime
156 * @param endTime
157 * @param logRotationTime
158 * @param logFile
159 * @return List of files to be streamed
160 */
161 private ArrayList<File> getFileList(File dir, long startTime, long endTime, long logRotationTime, String logFile) {
162 String[] children = dir.list();
163 ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
164 if (children == null) {
165 return new ArrayList<File>();
166 }
167 else {
168 for (int i = 0; i < children.length; i++) {
169 String fileName = children[i];
170 if (!fileName.startsWith(logFile) && !fileName.equals(logFile)) {
171 continue;
172 }
173 File file = new File(dir.getAbsolutePath(), fileName);
174 if (fileName.endsWith(".gz")) {
175 long gzFileCreationTime = getGZFileCreationTime(fileName, startTime, endTime);
176 if (gzFileCreationTime != -1) {
177 fileList.add(new FileInfo(file, gzFileCreationTime));
178 }
179 continue;
180 }
181 long modTime = file.lastModified();
182 if (modTime < startTime) {
183 continue;
184 }
185 if (modTime / logRotationTime > (endTime / logRotationTime + 1)) {
186 continue;
187 }
188 fileList.add(new FileInfo(file, modTime));
189 }
190 }
191 Collections.sort(fileList);
192 ArrayList<File> files = new ArrayList<File>(fileList.size());
193 for (FileInfo info : fileList) {
194 files.add(info.getFile());
195 }
196 return files;
197 }
198
199 /**
200 * This pattern matches the end of a gzip filename to have a format like "-YYYY-MM-dd-HH.gz" with capturing groups for each part
201 * of the date
202 */
203 public static final Pattern gzTimePattern = Pattern.compile(".*-(\\d\\d\\d\\d)-(\\d\\d)-(\\d\\d)-(\\d\\d)\\.gz");
204
205 /**
206 * Returns the creation time of the .gz archive if it is relevant to the job
207 *
208 * @param fileName
209 * @param startTime
210 * @param endTime
211 * @return Modification time of .gz file after checking if it is relevant to the job
212 */
213 private long getGZFileCreationTime(String fileName, long startTime, long endTime) {
214 // Default return value of -1 to exclude the file
215 long returnVal = -1;
216
217 // Include oozie.log as oozie.log.gz if it is accidentally GZipped
218 if (fileName.equals("oozie.log.gz")) {
219 LOG.warn("oozie.log has been GZipped, which is unexpected");
220 // Return a value other than -1 to include the file in list
221 returnVal = 0;
222 } else {
223 Matcher m = gzTimePattern.matcher(fileName);
224 if (m.matches() && m.groupCount() == 4) {
225 int year = Integer.parseInt(m.group(1));
226 int month = Integer.parseInt(m.group(2));
227 int day = Integer.parseInt(m.group(3));
228 int hour = Integer.parseInt(m.group(4));
229 int minute = 0;
230 Calendar calendarEntry = Calendar.getInstance();
231 calendarEntry.set(year, month - 1, day, hour, minute); // give month-1(Say, 7 for August)
232 long logFileStartTime = calendarEntry.getTimeInMillis();
233 long milliSecondsPerHour = 3600000;
234 long logFileEndTime = logFileStartTime + milliSecondsPerHour;
235 /* To check whether the log content is there in the initial or later part of the log file or
236 the log content is contained entirely within this log file or
237 the entire log file contains the event log where the event spans across hours
238 */
239 if ((startTime >= logFileStartTime && startTime <= logFileEndTime)
240 || (endTime >= logFileStartTime && endTime <= logFileEndTime)
241 || (startTime <= logFileStartTime && endTime >= logFileEndTime)) {
242 returnVal = logFileStartTime;
243 }
244 } else {
245 LOG.debug("Filename " + fileName + " does not match the expected format");
246 returnVal = -1;
247 }
248 }
249 return returnVal;
250 }
251 }