001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.oozie.util;
020
021 import java.io.IOException;
022 import java.util.ArrayList;
023 import java.util.Date;
024 import java.util.HashMap;
025 import java.util.List;
026 import java.util.Map;
027 import java.util.regex.Matcher;
028 import java.util.regex.Pattern;
029
030 import org.apache.commons.lang.StringUtils;
031 import org.apache.oozie.service.Services;
032
033 import com.google.common.annotations.VisibleForTesting;
034
035 /**
036 * Filter that will construct the regular expression that will be used to filter the log statement. And also checks if
037 * the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by "|")
038 * jobId appName actionId token
039 */
040 public class XLogFilter {
041
042 private static final int LOG_TIME_BUFFER = 2; // in min
043 public static String MAX_ACTIONLIST_SCAN_DURATION = "oozie.service.XLogStreamingService.actionlist.max.log.scan.duration";
044 public static String MAX_SCAN_DURATION = "oozie.service.XLogStreamingService.max.log.scan.duration";
045 private Map<String, Integer> logLevels;
046 private final Map<String, String> filterParams;
047 private static List<String> parameters = new ArrayList<String>();
048 private boolean noFilter;
049 private Pattern filterPattern;
050 private XLogUserFilterParam userLogFilter;
051 private Date endDate;
052 private Date startDate;
053 private boolean isActionList = false;
054 private String formattedEndDate;
055 private String formattedStartDate;
056
057 // TODO Patterns to be read from config file
058 private static final String DEFAULT_REGEX = "[^\\]]*";
059
060 public static final String ALLOW_ALL_REGEX = "(.*)";
061 private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)";
062 private static final String WHITE_SPACE_REGEX = "\\s+";
063 private static final String LOG_LEVEL_REGEX = "(\\w+)";
064 private static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX
065 + WHITE_SPACE_REGEX;
066 private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX);
067
068 public XLogFilter() {
069 this(new XLogUserFilterParam());
070 }
071
072 public XLogFilter(XLogUserFilterParam userLogFilter) {
073 filterParams = new HashMap<String, String>();
074 for (int i = 0; i < parameters.size(); i++) {
075 filterParams.put(parameters.get(i), DEFAULT_REGEX);
076 }
077 logLevels = null;
078 noFilter = true;
079 filterPattern = null;
080 setUserLogFilter(userLogFilter);
081 }
082
083 public void setLogLevel(String logLevel) {
084 if (logLevel != null && logLevel.trim().length() > 0) {
085 this.logLevels = new HashMap<String, Integer>();
086 String[] levels = logLevel.split("\\|");
087 for (int i = 0; i < levels.length; i++) {
088 String s = levels[i].trim().toUpperCase();
089 try {
090 XLog.Level.valueOf(s);
091 }
092 catch (Exception ex) {
093 continue;
094 }
095 this.logLevels.put(levels[i].toUpperCase(), 1);
096 }
097 }
098 }
099
100 public void setParameter(String filterParam, String value) {
101 if (filterParams.containsKey(filterParam)) {
102 noFilter = false;
103 filterParams.put(filterParam, value);
104 }
105 }
106
107 public static void defineParameter(String filterParam) {
108 parameters.add(filterParam);
109 }
110
111 public boolean isFilterPresent() {
112 if (noFilter && logLevels == null) {
113 return false;
114 }
115 return true;
116 }
117
118 /**
119 * Checks if the logLevel and logMessage goes through the logFilter.
120 *
121 * @param logParts
122 * @return
123 */
124 public boolean matches(ArrayList<String> logParts) {
125 if (getStartDate() != null) {
126 if (logParts.get(0).substring(0, 19).compareTo(getFormattedStartDate()) < 0) {
127 return false;
128 }
129 }
130 String logLevel = logParts.get(1);
131 String logMessage = logParts.get(2);
132 if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) {
133 Matcher logMatcher = filterPattern.matcher(logMessage);
134 return logMatcher.matches();
135 }
136 else {
137 return false;
138 }
139 }
140
141 /**
142 * Splits the log line into timestamp, logLevel and remaining log message. Returns array containing timestamp,
143 * logLevel, and logMessage if the pattern matches i.e A new log statement, else returns null.
144 *
145 * @param logLine
146 * @return Array containing log level and log message
147 */
148 public ArrayList<String> splitLogMessage(String logLine) {
149 Matcher splitter = SPLITTER_PATTERN.matcher(logLine);
150 if (splitter.matches()) {
151 ArrayList<String> logParts = new ArrayList<String>();
152 logParts.add(splitter.group(1));// timestamp
153 logParts.add(splitter.group(2));// log level
154 logParts.add(splitter.group(3));// Log Message
155 return logParts;
156 }
157 else {
158 return null;
159 }
160 }
161
162 /**
163 * Constructs the regular expression according to the filter and assigns it to fileterPattarn. ".*" will be assigned
164 * if no filters are set.
165 */
166 public void constructPattern() {
167 if (noFilter && logLevels == null) {
168 filterPattern = Pattern.compile(ALLOW_ALL_REGEX);
169 return;
170 }
171 StringBuilder sb = new StringBuilder();
172 if (noFilter) {
173 sb.append("(.*)");
174 }
175 else {
176 sb.append("(.* ");
177 for (int i = 0; i < parameters.size(); i++) {
178 sb.append(parameters.get(i) + "\\[");
179 sb.append(filterParams.get(parameters.get(i)) + "\\] ");
180 }
181 sb.append(".*)");
182 }
183 if (!StringUtils.isEmpty(userLogFilter.getSearchText())) {
184 sb.append(userLogFilter.getSearchText() + ".*");
185 }
186 filterPattern = Pattern.compile(sb.toString());
187 }
188
189 public static void reset() {
190 parameters.clear();
191 }
192
193 @VisibleForTesting
194 public final Map<String, String> getFilterParams() {
195 return filterParams;
196 }
197
198 public XLogUserFilterParam getUserLogFilter() {
199 return userLogFilter;
200 }
201
202 public void setUserLogFilter(XLogUserFilterParam userLogFilter) {
203 this.userLogFilter = userLogFilter;
204 setLogLevel(userLogFilter.getLogLevel());
205 }
206
207 public Date getEndDate() {
208 return endDate;
209 }
210
211 public String getFormattedEndDate() {
212 return formattedEndDate;
213 }
214
215 public String getFormattedStartDate() {
216 return formattedStartDate;
217 }
218
219 public Date getStartDate() {
220 return startDate;
221 }
222
223 public boolean isDebugMode() {
224 return userLogFilter.isDebug();
225 }
226
227 public int getLogLimit() {
228 return userLogFilter.getLimit();
229 }
230
231 public String getDebugMessage() {
232 return "Log start time = " + getStartDate() + ". Log end time = " + getEndDate() + ". User Log Filter = "
233 + getUserLogFilter() + System.getProperty("line.separator");
234 }
235
236 public boolean isActionList() {
237 return isActionList;
238 }
239
240 public void setActionList(boolean isActionList) {
241 this.isActionList = isActionList;
242 }
243
244 public static Pattern getSplitterPattern() {
245 return SPLITTER_PATTERN;
246 }
247 private void calculateScanDate(Date jobStartTime, Date jobEndTime) throws IOException {
248
249 if (userLogFilter.getStartDate() != null) {
250 startDate = userLogFilter.getStartDate();
251 }
252 else if (userLogFilter.getStartOffset() != -1) {
253 startDate = adjustOffset(jobStartTime, userLogFilter.getStartOffset());
254 }
255 else {
256 startDate = jobStartTime;
257 }
258
259 if (userLogFilter.getEndDate() != null) {
260 endDate = userLogFilter.getEndDate();
261 }
262 else if (userLogFilter.getEndOffset() != -1) {
263 // If user has specified startdate as absolute then end offset will be on user start date,
264 // else end offset will be calculated on job startdate.
265 if (userLogFilter.getStartDate() != null) {
266 endDate = adjustOffset(startDate, userLogFilter.getEndOffset());
267 }
268 else {
269 endDate = adjustOffset(jobStartTime, userLogFilter.getEndOffset());
270 }
271 }
272 else {
273 endDate = jobEndTime;
274 }
275 // if recent offset is specified then start time = endtime - offset
276 if (getUserLogFilter().getRecent() != -1) {
277 startDate = adjustOffset(endDate, userLogFilter.getRecent() * -1);
278 }
279
280 //add buffer iff dates are not asbsolute
281 if (userLogFilter.getStartDate() == null) {
282 startDate = adjustOffset(startDate, -LOG_TIME_BUFFER);
283 }
284 if (userLogFilter.getEndDate() == null) {
285 endDate = adjustOffset(endDate, LOG_TIME_BUFFER);
286 }
287
288 formattedEndDate = XLogUserFilterParam.dt.get().format(getEndDate());
289 formattedStartDate = XLogUserFilterParam.dt.get().format(getStartDate());
290 }
291
292 /**
293 * Calculate and validate date range.
294 *
295 * @param jobStartTime the job start time
296 * @param jobEndTime the job end time
297 * @throws IOException Signals that an I/O exception has occurred.
298 */
299 public void calculateAndValidateDateRange(Date jobStartTime, Date jobEndTime) throws IOException {
300 // for testcase, otherwise jobStartTime and jobEndTime will be always set
301 if (jobStartTime == null || jobEndTime == null) {
302 return;
303 }
304 calculateScanDate(jobStartTime, jobEndTime);
305
306 if (startDate.after(endDate)) {
307 throw new IOException("Start time should be less than end time. startTime = " + startDate + " endtime = "
308 + endDate);
309 }
310 long diffHours = (endDate.getTime() - startDate.getTime()) / (60 * 60 * 1000);
311 if (isActionList) {
312 int actionLogDuration = Services.get().getConf().getInt(MAX_ACTIONLIST_SCAN_DURATION, -1);
313 if (actionLogDuration == -1) {
314 return;
315 }
316 if (diffHours > actionLogDuration) {
317 throw new IOException(
318 "Request log streaming time range with action list is higher than configured. Please reduce the scan "
319 + "time range. Input range (hours) = " + diffHours
320 + " system allowed (hours) with action list = " + actionLogDuration);
321 }
322 }
323 else {
324 int logDuration = Services.get().getConf().getInt(MAX_SCAN_DURATION, -1);
325 if (logDuration == -1) {
326 return;
327 }
328 if (diffHours > logDuration) {
329 throw new IOException(
330 "Request log streaming time range is higher than configured. Please reduce the scan time range. For coord"
331 + " jobs you can provide action list to reduce log scan time range. Input range (hours) = "
332 + diffHours + " system allowed (hours) = " + logDuration);
333 }
334 }
335 }
336
337 /**
338 * Adjust offset, offset will always be in min.
339 *
340 * @param date the date
341 * @param offset the offset
342 * @return the date
343 * @throws IOException Signals that an I/O exception has occurred.
344 */
345 public Date adjustOffset(Date date, int offset) throws IOException {
346 return org.apache.commons.lang.time.DateUtils.addMinutes(date, offset);
347 }
348
349 }