001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    
019    package org.apache.oozie.util;
020    
021    import java.io.IOException;
022    import java.util.ArrayList;
023    import java.util.Date;
024    import java.util.HashMap;
025    import java.util.List;
026    import java.util.Map;
027    import java.util.regex.Matcher;
028    import java.util.regex.Pattern;
029    
030    import org.apache.commons.lang.StringUtils;
031    import org.apache.oozie.service.Services;
032    
033    import com.google.common.annotations.VisibleForTesting;
034    
035    /**
036     * Filter that will construct the regular expression that will be used to filter the log statement. And also checks if
037     * the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by "|")
038     * jobId appName actionId token
039     */
040    public class XLogFilter {
041    
042        private static final int LOG_TIME_BUFFER = 2; // in min
043        public static String MAX_ACTIONLIST_SCAN_DURATION = "oozie.service.XLogStreamingService.actionlist.max.log.scan.duration";
044        public static String MAX_SCAN_DURATION = "oozie.service.XLogStreamingService.max.log.scan.duration";
045        private Map<String, Integer> logLevels;
046        private final Map<String, String> filterParams;
047        private static List<String> parameters = new ArrayList<String>();
048        private boolean noFilter;
049        private Pattern filterPattern;
050        private XLogUserFilterParam userLogFilter;
051        private Date endDate;
052        private Date startDate;
053        private boolean isActionList = false;
054        private String formattedEndDate;
055        private String formattedStartDate;
056    
057        // TODO Patterns to be read from config file
058        private static final String DEFAULT_REGEX = "[^\\]]*";
059    
060        public static final String ALLOW_ALL_REGEX = "(.*)";
061        private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)";
062        private static final String WHITE_SPACE_REGEX = "\\s+";
063        private static final String LOG_LEVEL_REGEX = "(\\w+)";
064        private static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX
065                + WHITE_SPACE_REGEX;
066        private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX);
067    
068        public XLogFilter() {
069            this(new XLogUserFilterParam());
070        }
071    
072        public XLogFilter(XLogUserFilterParam userLogFilter) {
073            filterParams = new HashMap<String, String>();
074            for (int i = 0; i < parameters.size(); i++) {
075                filterParams.put(parameters.get(i), DEFAULT_REGEX);
076            }
077            logLevels = null;
078            noFilter = true;
079            filterPattern = null;
080            setUserLogFilter(userLogFilter);
081        }
082    
083        public void setLogLevel(String logLevel) {
084            if (logLevel != null && logLevel.trim().length() > 0) {
085                this.logLevels = new HashMap<String, Integer>();
086                String[] levels = logLevel.split("\\|");
087                for (int i = 0; i < levels.length; i++) {
088                    String s = levels[i].trim().toUpperCase();
089                    try {
090                        XLog.Level.valueOf(s);
091                    }
092                    catch (Exception ex) {
093                        continue;
094                    }
095                    this.logLevels.put(levels[i].toUpperCase(), 1);
096                }
097            }
098        }
099    
100        public void setParameter(String filterParam, String value) {
101            if (filterParams.containsKey(filterParam)) {
102                noFilter = false;
103                filterParams.put(filterParam, value);
104            }
105        }
106    
107        public static void defineParameter(String filterParam) {
108            parameters.add(filterParam);
109        }
110    
111        public boolean isFilterPresent() {
112            if (noFilter && logLevels == null) {
113                return false;
114            }
115            return true;
116        }
117    
118        /**
119         * Checks if the logLevel and logMessage goes through the logFilter.
120         *
121         * @param logParts
122         * @return
123         */
124        public boolean matches(ArrayList<String> logParts) {
125            if (getStartDate() != null) {
126                if (logParts.get(0).substring(0, 19).compareTo(getFormattedStartDate()) < 0) {
127                    return false;
128                }
129            }
130            String logLevel = logParts.get(1);
131            String logMessage = logParts.get(2);
132            if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) {
133                Matcher logMatcher = filterPattern.matcher(logMessage);
134                return logMatcher.matches();
135            }
136            else {
137                return false;
138            }
139        }
140    
141        /**
142         * Splits the log line into timestamp, logLevel and remaining log message. Returns array containing timestamp,
143         * logLevel, and logMessage if the pattern matches i.e A new log statement, else returns null.
144         *
145         * @param logLine
146         * @return Array containing log level and log message
147         */
148        public ArrayList<String> splitLogMessage(String logLine) {
149            Matcher splitter = SPLITTER_PATTERN.matcher(logLine);
150            if (splitter.matches()) {
151                ArrayList<String> logParts = new ArrayList<String>();
152                logParts.add(splitter.group(1));// timestamp
153                logParts.add(splitter.group(2));// log level
154                logParts.add(splitter.group(3));// Log Message
155                return logParts;
156            }
157            else {
158                return null;
159            }
160        }
161    
162        /**
163         * Constructs the regular expression according to the filter and assigns it to fileterPattarn. ".*" will be assigned
164         * if no filters are set.
165         */
166        public void constructPattern() {
167            if (noFilter && logLevels == null) {
168                filterPattern = Pattern.compile(ALLOW_ALL_REGEX);
169                return;
170            }
171            StringBuilder sb = new StringBuilder();
172            if (noFilter) {
173                sb.append("(.*)");
174            }
175            else {
176                sb.append("(.* ");
177                for (int i = 0; i < parameters.size(); i++) {
178                    sb.append(parameters.get(i) + "\\[");
179                    sb.append(filterParams.get(parameters.get(i)) + "\\] ");
180                }
181                sb.append(".*)");
182            }
183            if (!StringUtils.isEmpty(userLogFilter.getSearchText())) {
184                sb.append(userLogFilter.getSearchText() + ".*");
185            }
186            filterPattern = Pattern.compile(sb.toString());
187        }
188    
189        public static void reset() {
190            parameters.clear();
191        }
192    
193        @VisibleForTesting
194        public final Map<String, String> getFilterParams() {
195            return filterParams;
196        }
197    
198        public XLogUserFilterParam getUserLogFilter() {
199            return userLogFilter;
200        }
201    
202        public void setUserLogFilter(XLogUserFilterParam userLogFilter) {
203            this.userLogFilter = userLogFilter;
204            setLogLevel(userLogFilter.getLogLevel());
205        }
206    
207        public Date getEndDate() {
208            return endDate;
209        }
210    
211        public String getFormattedEndDate() {
212            return formattedEndDate;
213        }
214    
215        public String getFormattedStartDate() {
216            return formattedStartDate;
217        }
218    
219        public Date getStartDate() {
220            return startDate;
221        }
222    
223        public boolean isDebugMode() {
224            return userLogFilter.isDebug();
225        }
226    
227        public int getLogLimit() {
228            return userLogFilter.getLimit();
229        }
230    
231        public String getDebugMessage() {
232            return "Log start time = " + getStartDate() + ". Log end time = " + getEndDate() + ". User Log Filter = "
233                    + getUserLogFilter() + System.getProperty("line.separator");
234        }
235    
236        public boolean isActionList() {
237            return isActionList;
238        }
239    
240        public void setActionList(boolean isActionList) {
241            this.isActionList = isActionList;
242        }
243    
244        public static Pattern getSplitterPattern() {
245            return SPLITTER_PATTERN;
246        }
247        private void calculateScanDate(Date jobStartTime, Date jobEndTime) throws IOException {
248    
249            if (userLogFilter.getStartDate() != null) {
250                startDate = userLogFilter.getStartDate();
251            }
252            else if (userLogFilter.getStartOffset() != -1) {
253                startDate = adjustOffset(jobStartTime, userLogFilter.getStartOffset());
254            }
255            else {
256                startDate = jobStartTime;
257            }
258    
259            if (userLogFilter.getEndDate() != null) {
260                endDate = userLogFilter.getEndDate();
261            }
262            else if (userLogFilter.getEndOffset() != -1) {
263                // If user has specified startdate as absolute then end offset will be on user start date,
264                // else end offset will be calculated on job startdate.
265                if (userLogFilter.getStartDate() != null) {
266                    endDate = adjustOffset(startDate, userLogFilter.getEndOffset());
267                }
268                else {
269                    endDate = adjustOffset(jobStartTime, userLogFilter.getEndOffset());
270                }
271            }
272            else {
273                endDate = jobEndTime;
274            }
275            // if recent offset is specified then start time = endtime - offset
276            if (getUserLogFilter().getRecent() != -1) {
277                startDate = adjustOffset(endDate, userLogFilter.getRecent() * -1);
278            }
279    
280            //add buffer iff dates are not asbsolute
281            if (userLogFilter.getStartDate() == null) {
282                startDate = adjustOffset(startDate, -LOG_TIME_BUFFER);
283            }
284            if (userLogFilter.getEndDate() == null) {
285                endDate = adjustOffset(endDate, LOG_TIME_BUFFER);
286            }
287    
288            formattedEndDate = XLogUserFilterParam.dt.get().format(getEndDate());
289            formattedStartDate = XLogUserFilterParam.dt.get().format(getStartDate());
290        }
291    
292        /**
293         * Calculate and validate date range.
294         *
295         * @param jobStartTime the job start time
296         * @param jobEndTime the job end time
297         * @throws IOException Signals that an I/O exception has occurred.
298         */
299        public void calculateAndValidateDateRange(Date jobStartTime, Date jobEndTime) throws IOException {
300            // for testcase, otherwise jobStartTime and jobEndTime will be always set
301            if (jobStartTime == null || jobEndTime == null) {
302                return;
303            }
304            calculateScanDate(jobStartTime, jobEndTime);
305    
306            if (startDate.after(endDate)) {
307                throw new IOException("Start time should be less than end time. startTime = " + startDate + " endtime = "
308                        + endDate);
309            }
310            long diffHours = (endDate.getTime() - startDate.getTime()) / (60 * 60 * 1000);
311            if (isActionList) {
312                int actionLogDuration = Services.get().getConf().getInt(MAX_ACTIONLIST_SCAN_DURATION, -1);
313                if (actionLogDuration == -1) {
314                    return;
315                }
316                if (diffHours > actionLogDuration) {
317                    throw new IOException(
318                            "Request log streaming time range with action list is higher than configured. Please reduce the scan "
319                                    + "time range. Input range (hours) = " + diffHours
320                                    + " system allowed (hours) with action list = " + actionLogDuration);
321                }
322            }
323            else {
324                int logDuration = Services.get().getConf().getInt(MAX_SCAN_DURATION, -1);
325                if (logDuration == -1) {
326                    return;
327                }
328                if (diffHours > logDuration) {
329                    throw new IOException(
330                            "Request log streaming time range is higher than configured. Please reduce the scan time range. For coord"
331                                    + " jobs you can provide action list to reduce log scan time range. Input range (hours) = "
332                                    + diffHours + " system allowed (hours) = " + logDuration);
333                }
334            }
335        }
336    
337        /**
338         * Adjust offset, offset will always be in min.
339         *
340         * @param date the date
341         * @param offset the offset
342         * @return the date
343         * @throws IOException Signals that an I/O exception has occurred.
344         */
345        public Date adjustOffset(Date date, int offset) throws IOException {
346            return org.apache.commons.lang.time.DateUtils.addMinutes(date, offset);
347        }
348    
349    }