001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import org.apache.hadoop.fs.FSDataOutputStream;
021    import org.apache.hadoop.fs.FileSystem;
022    import org.apache.hadoop.fs.Path;
023    import org.apache.hadoop.conf.Configuration;
024    import org.apache.oozie.ErrorCode;
025    import org.apache.oozie.util.XLog;
026    import org.apache.oozie.util.XLogFilter;
027    
028    import java.io.BufferedReader;
029    import java.io.BufferedWriter;
030    import java.io.File;
031    import java.io.FileNotFoundException;
032    import java.io.FileReader;
033    import java.io.IOException;
034    import java.io.OutputStreamWriter;
035    import java.net.URI;
036    import java.sql.Connection;
037    import java.sql.DriverManager;
038    import java.sql.ResultSet;
039    import java.sql.Statement;
040    import java.sql.SQLException;
041    import java.util.regex.Matcher;
042    import java.util.regex.Pattern;
043    import java.util.ArrayList;
044    import java.util.HashMap;
045    import java.util.List;
046    
047    /**
048     * The XLogCopyService copies oozie.log contents onto HDFS. By default,
049     * it runs every 5 minutes.
050     */
051    public class XLogCopyService implements Service {
052    
053        public static final String CONF_PREFIX = Service.CONF_PREFIX + "XLogCopyService.";
054    
055        /**
056         * Time interval, in minutes, at which the service will be scheduled to run.
057         */
058        public static final String CONF_SERVICE_INTERVAL = CONF_PREFIX + "interval";
059    
060        /**
061         * HDFS directory where oozie log will be forward to.
062         */
063        public static final String CONF_HDFS_LOG_DIR = CONF_PREFIX + "hdfs.log.dir";
064    
065        public static final String CONF_LOG_PURGE = CONF_PREFIX + "purge.enable";
066    
067        public static String HDFS_LOG_DIR;
068    
069        public static int CURRENT_LINE_NUMBER = 0;
070    
071        public static String LAST_COMPLETE_LOG_FILE_NAME = "oozie.log";
072    
073        public static String OOZIE_INSTANCE_ID = "";
074    
075        public static HashMap<String, String> jdbcConf = new HashMap<String, String>();
076    
077        public static String LOGPROGRESS_GET_QUERY = "select data from OOZIE_SYS where name = '";
078    
079        public static String LOGPROGRESS_UPDATE_QUERY = "update OOZIE_SYS set data = '";
080    
081        public static String LOGPROGRESS_UPDATE_QUERY_WHERE_CLAUSE = "' where name = '";
082    
083        public static String LOGPROGRESS_INSERT_QUERY = "insert into OOZIE_SYS (name, data) values ";
084    
085        public static Boolean IS_LOG_PURGING_ENABLED = false;
086    
087        private final XLog log = XLog.getLog(getClass());
088    
089        static class XLogCopyRunnable implements Runnable {
090    
091            private final String hdfsDir;
092    
093            public XLogCopyRunnable(String hdfsDir) {
094                this.hdfsDir = hdfsDir;
095            }
096    
097            public void run() {
098                XLogService xls = Services.get().get(XLogService.class);
099                String oozieLogPath = xls.getOozieLogPath();
100                String oozieLogName = xls.getOozieLogName();
101    
102                XLog.Info.get().clear();
103                XLog log = XLog.getLog(getClass());
104                log.info("hdfs log dir is " + hdfsDir);
105    
106                if (oozieLogPath != null && oozieLogName != null) {
107                    String[] children = new File(oozieLogPath).list();
108                    if (children != null) {
109                        List<String> rolloveredFiles = new ArrayList<String>();
110                        if (oozieLogName.equals(LAST_COMPLETE_LOG_FILE_NAME)) {
111                            //This means no rollover has happened yet before this run.
112                            for (String child: children) {
113                                if (child.startsWith(oozieLogName) && !child.equals(oozieLogName)) {
114                                    rolloveredFiles.add(child);
115                                }
116                            }
117                        }
118                        else {
119                            long lastFileModifyTime = new File(oozieLogPath, LAST_COMPLETE_LOG_FILE_NAME).lastModified();
120                            for (String child: children) {
121                                //Collect files that have been rolloverred.
122                                if (!child.equals(oozieLogName) && child.startsWith(oozieLogName) &&
123                                        lastFileModifyTime < getLastModifiedTime(oozieLogPath, child)) {
124                                    rolloveredFiles.add(child);
125                                }
126                            }
127                        }
128    
129                        if (!rolloveredFiles.isEmpty()) {
130                            String earliestFile = rolloveredFiles.get(0);
131                            long earliestTime = getLastModifiedTime(oozieLogPath, earliestFile);
132                            String latestFile = earliestFile;
133                            long latestTime = earliestTime;
134                            int earliestIndex = 0;
135                            for (int i = 0; i < rolloveredFiles.size(); i++) {
136                                String file = rolloveredFiles.get(i);
137                                if (earliestTime > getLastModifiedTime(oozieLogPath, file)) {
138                                    earliestTime = getLastModifiedTime(oozieLogPath, file);
139                                    earliestFile = file;
140                                    earliestIndex = i;
141                                }
142                                if (latestTime < getLastModifiedTime(oozieLogPath, file)) {
143                                    latestTime = getLastModifiedTime(oozieLogPath, file);
144                                    latestFile = file;
145                                }
146                            }
147    
148                            rolloveredFiles.remove(earliestIndex);
149                            //TODO
150                            //Finish writing the file we processed last run
151                            writeToHdfs(CURRENT_LINE_NUMBER, new File(oozieLogPath, earliestFile));
152    
153                            for (String file : rolloveredFiles) {
154                                //Process all the rolloverred files in the middle if any
155                                //Under normal conditions without interruption of services, this for loop should be empty
156                                writeToHdfs(0, new File(oozieLogPath, file));
157                            }
158    
159                            LAST_COMPLETE_LOG_FILE_NAME = latestFile;
160                            //Process the new oozie.log file
161                            CURRENT_LINE_NUMBER = writeToHdfs(0, new File(oozieLogPath, oozieLogName));
162                        }
163                        else {
164                            //No rollovers since last run, continue processing the current oozie.log file
165                            CURRENT_LINE_NUMBER = writeToHdfs(CURRENT_LINE_NUMBER, new File(oozieLogPath, oozieLogName));
166                        }
167                    }
168                }
169                updateLogProgress(CURRENT_LINE_NUMBER + "," + LAST_COMPLETE_LOG_FILE_NAME);
170            }
171    
172            private int writeToHdfs(int lineNumber, File file){
173                XLog.Info.get().clear();
174                XLog log = XLog.getLog(getClass());
175                HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
176                URI uri = new Path(hdfsDir).toUri();
177                Configuration fsConf = has.createJobConf(uri.getAuthority());
178                FileSystem fs = null;
179                HashMap<String, StringBuilder> logMap = new HashMap<String, StringBuilder>();
180                Pattern splitterPattern = XLogFilter.getSplitterPattern();
181    
182                try {
183                    BufferedReader br = new BufferedReader(new FileReader(file));
184                    String currentLine = "";
185                    Pattern pattern = Pattern.compile("JOB\\[(.*?)\\]");
186    
187                    // Skip lines that have been copied already
188                    for (int i = 0; i < lineNumber; i++) {
189                        br.readLine();
190                    }
191    
192                    String jobId = "";
193                    while ((currentLine = br.readLine()) != null) {
194                        Matcher splitter = splitterPattern.matcher(currentLine);
195                        if (splitter.matches()){
196                            jobId = "";
197                            Matcher matcher = pattern.matcher(currentLine);
198                            while (matcher.find()) {
199                                jobId = matcher.group(1);
200                                break;
201                            }
202                        }
203    
204                        //For efficiency purpose, we first store lines to be copied by jobId
205                        if (!jobId.trim().equals("-") && !jobId.isEmpty()) {
206                            StringBuilder sb = new StringBuilder();
207                            if (logMap.containsKey(jobId)) {
208                                sb = logMap.get(jobId);
209                            }
210                            sb.append(currentLine).append("\n");
211                            logMap.put(jobId, sb);
212                        }
213                        lineNumber++;
214                    }
215    
216                    br.close();
217    
218                    try {
219                        fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf);
220                    }
221                    catch (Exception ex) {
222                        log.error("user has to be specified to access hdfs",
223                                new HadoopAccessorException(ErrorCode.E0902, "user has to be specified to access FileSystem"));
224                    }
225    
226                    for (String id : logMap.keySet()) {
227                        final Path p = new Path(hdfsDir, id + ".log");
228                        FSDataOutputStream os = null;
229                        if (!fs.exists(p)) {
230                            os = fs.create(p);
231                        }
232                        else {
233                            os = fs.append(p);
234                        }
235    
236                        BufferedWriter bw = new BufferedWriter( new OutputStreamWriter(os));
237                        bw.write(logMap.get(id).toString());
238                        bw.close();
239                    }
240    
241                    fs.close();
242                }
243                catch (FileNotFoundException ex) {
244                    log.error(file + " does not exist",
245                            new HadoopAccessorException(ErrorCode.E0902, "cannot locate file: " + file + " on hdfs"));
246                }
247                catch (IOException ex) {
248                    log.warn("log copy failed with exception: " + ex.getMessage());
249                }
250    
251                return lineNumber;
252            }
253        }
254    
255        private static long getLastModifiedTime(String directory, String file) {
256            return new File(directory, file).lastModified();
257        }
258    
259        public void init(Services services) throws ServiceException{
260            Configuration conf = services.getConf();
261            int interval = conf.getInt(CONF_SERVICE_INTERVAL, 300);
262            HDFS_LOG_DIR = conf.get(CONF_HDFS_LOG_DIR);
263            if (HDFS_LOG_DIR == null || HDFS_LOG_DIR.isEmpty()) {
264                throw new ServiceException(ErrorCode.E0307, "hdfs log directory not specified");
265            }
266            IS_LOG_PURGING_ENABLED = conf.getBoolean(CONF_LOG_PURGE, false);
267            OOZIE_INSTANCE_ID = System.getenv("OOZIE_INSTANCE_ID");
268            LOGPROGRESS_GET_QUERY +=  OOZIE_INSTANCE_ID + ".logprogress'";
269            LOGPROGRESS_UPDATE_QUERY_WHERE_CLAUSE +=  OOZIE_INSTANCE_ID + ".logprogress'";
270            LOGPROGRESS_INSERT_QUERY += "('" + OOZIE_INSTANCE_ID + ".logprogress', '"
271                    + CURRENT_LINE_NUMBER + "," + LAST_COMPLETE_LOG_FILE_NAME + "')";
272    
273            jdbcConf.put("driver", conf.get(JPAService.CONF_DRIVER));
274            String url = conf.get(JPAService.CONF_URL);
275            jdbcConf.put("url", url);
276            jdbcConf.put("user", conf.get(JPAService.CONF_USERNAME));
277    
278            jdbcConf.put("password", services.get(HadoopAccessorService.class).getPasswordFromHadoopConf(conf,
279                    JPAService.CONF_PASSWORD));
280            String dbType = url.substring("jdbc:".length());
281            if (dbType.indexOf(":") <= 0) {
282                throw new RuntimeException("Invalid JDBC URL, missing vendor 'jdbc:[VENDOR]:...'");
283            }
284            dbType = dbType.substring(0, dbType.indexOf(":"));
285            jdbcConf.put("dbtype", dbType);
286    
287            initLogProgress();
288    
289            XLogCopyRunnable runnable = new XLogCopyRunnable(new Path(HDFS_LOG_DIR, OOZIE_INSTANCE_ID).toString());
290            services.get(SchedulerService.class).schedule(runnable, 10, interval, SchedulerService.Unit.SEC);
291            log.info("XLogCopyService is initialized");
292        }
293    
294        @Override
295        public Class<? extends Service> getInterface() {
296            return XLogCopyService.class;
297        }
298    
299        @Override
300        public void destroy() {
301            updateLogProgress(CURRENT_LINE_NUMBER + "," + LAST_COMPLETE_LOG_FILE_NAME);
302        }
303    
304        public String getConfHdfsLogDir() {
305            return HDFS_LOG_DIR;
306        }
307    
308        private static void initLogProgress() throws ServiceException{
309            String logProgress = "";
310            XLog log = XLog.getLog(XLogCopyService.class);
311    
312            try {
313                Class.forName(jdbcConf.get("driver")).newInstance();
314                Connection conn = DriverManager.getConnection(jdbcConf.get("url"), jdbcConf.get("user"), jdbcConf.get("password"));
315                try {
316                    log.info("execute query: " + LOGPROGRESS_GET_QUERY);
317                    Statement st = conn.createStatement();
318                    ResultSet rs = st.executeQuery(LOGPROGRESS_GET_QUERY);
319                    if (rs.next()) {
320                        logProgress = rs.getString(1);
321                    }
322    
323                    rs.close();
324                    st.close();
325                }
326                catch (SQLException ex) {
327                    throw new ServiceException(ErrorCode.E0170, "cannot query OOZIE_SYS table", ex);
328                }
329            }
330            catch (SQLException ex) {
331                throw new ServiceException(ErrorCode.E0600, ex);
332            }
333            catch (Exception ex) {
334                throw new ServiceException(ErrorCode.E0307, ex);
335            }
336    
337            if (!logProgress.isEmpty()) {
338                String[] progressRecord = logProgress.trim().split(",");
339                try {
340                    CURRENT_LINE_NUMBER = Integer.parseInt(progressRecord[0].trim());
341                }
342                catch (NumberFormatException ex) {
343                    log.warn("Can not retrieve line number from log progress file, default it to 0");
344                }
345    
346                LAST_COMPLETE_LOG_FILE_NAME = progressRecord[1].trim();
347            }
348            else {
349                try {
350                    Class.forName(jdbcConf.get("driver")).newInstance();
351                    Connection conn = DriverManager.getConnection(jdbcConf.get("url"), jdbcConf.get("user"), jdbcConf.get("password"));
352                    try {
353                        log.info("insert query is " + LOGPROGRESS_INSERT_QUERY);
354                        Statement st = conn.createStatement();
355                        st.executeUpdate(LOGPROGRESS_INSERT_QUERY);
356                        st.close();
357                    }
358                    catch (SQLException ex) {
359                        throw new ServiceException(ErrorCode.E0170, "cannot insert into OOZIE_SYS table", ex);
360                    }
361                }
362                catch (SQLException ex) {
363                    throw new ServiceException(ErrorCode.E0600, ex);
364                }
365                catch (Exception ex) {
366                    throw new ServiceException(ErrorCode.E0307, ex);
367                }
368            }
369        }
370    
371        private static void updateLogProgress(String logProgress) {
372            XLog log = XLog.getLog(XLogCopyService.class);
373            try {
374                Class.forName(jdbcConf.get("driver")).newInstance();
375                Connection conn = DriverManager.getConnection(jdbcConf.get("url"), jdbcConf.get("user"), jdbcConf.get("password"));
376                try {
377                    String updateQuery = LOGPROGRESS_UPDATE_QUERY + logProgress + LOGPROGRESS_UPDATE_QUERY_WHERE_CLAUSE;
378                    log.info("execute query; " + updateQuery);
379                    Statement st = conn.createStatement();
380                    st.executeUpdate(updateQuery);
381                    st.close();
382                }
383                catch (SQLException ex) {
384                    log.error("Cannot update logProgress in OOZIE_SYS table: " + ex.getMessage());
385                }
386            }
387            catch (Exception ex) {
388                log.error("run time error " + ex.getMessage());
389            }
390        }
391    }