001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.service;
019
020 import org.apache.hadoop.fs.FSDataOutputStream;
021 import org.apache.hadoop.fs.FileSystem;
022 import org.apache.hadoop.fs.Path;
023 import org.apache.hadoop.conf.Configuration;
024 import org.apache.oozie.ErrorCode;
025 import org.apache.oozie.util.XLog;
026 import org.apache.oozie.util.XLogFilter;
027
028 import java.io.BufferedReader;
029 import java.io.BufferedWriter;
030 import java.io.File;
031 import java.io.FileNotFoundException;
032 import java.io.FileReader;
033 import java.io.IOException;
034 import java.io.OutputStreamWriter;
035 import java.net.URI;
036 import java.sql.Connection;
037 import java.sql.DriverManager;
038 import java.sql.ResultSet;
039 import java.sql.Statement;
040 import java.sql.SQLException;
041 import java.util.regex.Matcher;
042 import java.util.regex.Pattern;
043 import java.util.ArrayList;
044 import java.util.HashMap;
045 import java.util.List;
046
047 /**
048 * The XLogCopyService copies oozie.log contents onto HDFS. By default,
049 * it runs every 5 minutes.
050 */
051 public class XLogCopyService implements Service {
052
053 public static final String CONF_PREFIX = Service.CONF_PREFIX + "XLogCopyService.";
054
055 /**
056 * Time interval, in minutes, at which the service will be scheduled to run.
057 */
058 public static final String CONF_SERVICE_INTERVAL = CONF_PREFIX + "interval";
059
060 /**
061 * HDFS directory where oozie log will be forward to.
062 */
063 public static final String CONF_HDFS_LOG_DIR = CONF_PREFIX + "hdfs.log.dir";
064
065 public static final String CONF_LOG_PURGE = CONF_PREFIX + "purge.enable";
066
067 public static String HDFS_LOG_DIR;
068
069 public static int CURRENT_LINE_NUMBER = 0;
070
071 public static String LAST_COMPLETE_LOG_FILE_NAME = "oozie.log";
072
073 public static String OOZIE_INSTANCE_ID = "";
074
075 public static HashMap<String, String> jdbcConf = new HashMap<String, String>();
076
077 public static String LOGPROGRESS_GET_QUERY = "select data from OOZIE_SYS where name = '";
078
079 public static String LOGPROGRESS_UPDATE_QUERY = "update OOZIE_SYS set data = '";
080
081 public static String LOGPROGRESS_UPDATE_QUERY_WHERE_CLAUSE = "' where name = '";
082
083 public static String LOGPROGRESS_INSERT_QUERY = "insert into OOZIE_SYS (name, data) values ";
084
085 public static Boolean IS_LOG_PURGING_ENABLED = false;
086
087 private final XLog log = XLog.getLog(getClass());
088
089 static class XLogCopyRunnable implements Runnable {
090
091 private final String hdfsDir;
092
093 public XLogCopyRunnable(String hdfsDir) {
094 this.hdfsDir = hdfsDir;
095 }
096
097 public void run() {
098 XLogService xls = Services.get().get(XLogService.class);
099 String oozieLogPath = xls.getOozieLogPath();
100 String oozieLogName = xls.getOozieLogName();
101
102 XLog.Info.get().clear();
103 XLog log = XLog.getLog(getClass());
104 log.info("hdfs log dir is " + hdfsDir);
105
106 if (oozieLogPath != null && oozieLogName != null) {
107 String[] children = new File(oozieLogPath).list();
108 if (children != null) {
109 List<String> rolloveredFiles = new ArrayList<String>();
110 if (oozieLogName.equals(LAST_COMPLETE_LOG_FILE_NAME)) {
111 //This means no rollover has happened yet before this run.
112 for (String child: children) {
113 if (child.startsWith(oozieLogName) && !child.equals(oozieLogName)) {
114 rolloveredFiles.add(child);
115 }
116 }
117 }
118 else {
119 long lastFileModifyTime = new File(oozieLogPath, LAST_COMPLETE_LOG_FILE_NAME).lastModified();
120 for (String child: children) {
121 //Collect files that have been rolloverred.
122 if (!child.equals(oozieLogName) && child.startsWith(oozieLogName) &&
123 lastFileModifyTime < getLastModifiedTime(oozieLogPath, child)) {
124 rolloveredFiles.add(child);
125 }
126 }
127 }
128
129 if (!rolloveredFiles.isEmpty()) {
130 String earliestFile = rolloveredFiles.get(0);
131 long earliestTime = getLastModifiedTime(oozieLogPath, earliestFile);
132 String latestFile = earliestFile;
133 long latestTime = earliestTime;
134 int earliestIndex = 0;
135 for (int i = 0; i < rolloveredFiles.size(); i++) {
136 String file = rolloveredFiles.get(i);
137 if (earliestTime > getLastModifiedTime(oozieLogPath, file)) {
138 earliestTime = getLastModifiedTime(oozieLogPath, file);
139 earliestFile = file;
140 earliestIndex = i;
141 }
142 if (latestTime < getLastModifiedTime(oozieLogPath, file)) {
143 latestTime = getLastModifiedTime(oozieLogPath, file);
144 latestFile = file;
145 }
146 }
147
148 rolloveredFiles.remove(earliestIndex);
149 //TODO
150 //Finish writing the file we processed last run
151 writeToHdfs(CURRENT_LINE_NUMBER, new File(oozieLogPath, earliestFile));
152
153 for (String file : rolloveredFiles) {
154 //Process all the rolloverred files in the middle if any
155 //Under normal conditions without interruption of services, this for loop should be empty
156 writeToHdfs(0, new File(oozieLogPath, file));
157 }
158
159 LAST_COMPLETE_LOG_FILE_NAME = latestFile;
160 //Process the new oozie.log file
161 CURRENT_LINE_NUMBER = writeToHdfs(0, new File(oozieLogPath, oozieLogName));
162 }
163 else {
164 //No rollovers since last run, continue processing the current oozie.log file
165 CURRENT_LINE_NUMBER = writeToHdfs(CURRENT_LINE_NUMBER, new File(oozieLogPath, oozieLogName));
166 }
167 }
168 }
169 updateLogProgress(CURRENT_LINE_NUMBER + "," + LAST_COMPLETE_LOG_FILE_NAME);
170 }
171
172 private int writeToHdfs(int lineNumber, File file){
173 XLog.Info.get().clear();
174 XLog log = XLog.getLog(getClass());
175 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
176 URI uri = new Path(hdfsDir).toUri();
177 Configuration fsConf = has.createJobConf(uri.getAuthority());
178 FileSystem fs = null;
179 HashMap<String, StringBuilder> logMap = new HashMap<String, StringBuilder>();
180 Pattern splitterPattern = XLogFilter.getSplitterPattern();
181
182 try {
183 BufferedReader br = new BufferedReader(new FileReader(file));
184 String currentLine = "";
185 Pattern pattern = Pattern.compile("JOB\\[(.*?)\\]");
186
187 // Skip lines that have been copied already
188 for (int i = 0; i < lineNumber; i++) {
189 br.readLine();
190 }
191
192 String jobId = "";
193 while ((currentLine = br.readLine()) != null) {
194 Matcher splitter = splitterPattern.matcher(currentLine);
195 if (splitter.matches()){
196 jobId = "";
197 Matcher matcher = pattern.matcher(currentLine);
198 while (matcher.find()) {
199 jobId = matcher.group(1);
200 break;
201 }
202 }
203
204 //For efficiency purpose, we first store lines to be copied by jobId
205 if (!jobId.trim().equals("-") && !jobId.isEmpty()) {
206 StringBuilder sb = new StringBuilder();
207 if (logMap.containsKey(jobId)) {
208 sb = logMap.get(jobId);
209 }
210 sb.append(currentLine).append("\n");
211 logMap.put(jobId, sb);
212 }
213 lineNumber++;
214 }
215
216 br.close();
217
218 try {
219 fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf);
220 }
221 catch (Exception ex) {
222 log.error("user has to be specified to access hdfs",
223 new HadoopAccessorException(ErrorCode.E0902, "user has to be specified to access FileSystem"));
224 }
225
226 for (String id : logMap.keySet()) {
227 final Path p = new Path(hdfsDir, id + ".log");
228 FSDataOutputStream os = null;
229 if (!fs.exists(p)) {
230 os = fs.create(p);
231 }
232 else {
233 os = fs.append(p);
234 }
235
236 BufferedWriter bw = new BufferedWriter( new OutputStreamWriter(os));
237 bw.write(logMap.get(id).toString());
238 bw.close();
239 }
240
241 fs.close();
242 }
243 catch (FileNotFoundException ex) {
244 log.error(file + " does not exist",
245 new HadoopAccessorException(ErrorCode.E0902, "cannot locate file: " + file + " on hdfs"));
246 }
247 catch (IOException ex) {
248 log.warn("log copy failed with exception: " + ex.getMessage());
249 }
250
251 return lineNumber;
252 }
253 }
254
255 private static long getLastModifiedTime(String directory, String file) {
256 return new File(directory, file).lastModified();
257 }
258
259 public void init(Services services) throws ServiceException{
260 Configuration conf = services.getConf();
261 int interval = conf.getInt(CONF_SERVICE_INTERVAL, 300);
262 HDFS_LOG_DIR = conf.get(CONF_HDFS_LOG_DIR);
263 if (HDFS_LOG_DIR == null || HDFS_LOG_DIR.isEmpty()) {
264 throw new ServiceException(ErrorCode.E0307, "hdfs log directory not specified");
265 }
266 IS_LOG_PURGING_ENABLED = conf.getBoolean(CONF_LOG_PURGE, false);
267 OOZIE_INSTANCE_ID = System.getenv("OOZIE_INSTANCE_ID");
268 LOGPROGRESS_GET_QUERY += OOZIE_INSTANCE_ID + ".logprogress'";
269 LOGPROGRESS_UPDATE_QUERY_WHERE_CLAUSE += OOZIE_INSTANCE_ID + ".logprogress'";
270 LOGPROGRESS_INSERT_QUERY += "('" + OOZIE_INSTANCE_ID + ".logprogress', '"
271 + CURRENT_LINE_NUMBER + "," + LAST_COMPLETE_LOG_FILE_NAME + "')";
272
273 jdbcConf.put("driver", conf.get(JPAService.CONF_DRIVER));
274 String url = conf.get(JPAService.CONF_URL);
275 jdbcConf.put("url", url);
276 jdbcConf.put("user", conf.get(JPAService.CONF_USERNAME));
277
278 jdbcConf.put("password", services.get(HadoopAccessorService.class).getPasswordFromHadoopConf(conf,
279 JPAService.CONF_PASSWORD));
280 String dbType = url.substring("jdbc:".length());
281 if (dbType.indexOf(":") <= 0) {
282 throw new RuntimeException("Invalid JDBC URL, missing vendor 'jdbc:[VENDOR]:...'");
283 }
284 dbType = dbType.substring(0, dbType.indexOf(":"));
285 jdbcConf.put("dbtype", dbType);
286
287 initLogProgress();
288
289 XLogCopyRunnable runnable = new XLogCopyRunnable(new Path(HDFS_LOG_DIR, OOZIE_INSTANCE_ID).toString());
290 services.get(SchedulerService.class).schedule(runnable, 10, interval, SchedulerService.Unit.SEC);
291 log.info("XLogCopyService is initialized");
292 }
293
294 @Override
295 public Class<? extends Service> getInterface() {
296 return XLogCopyService.class;
297 }
298
299 @Override
300 public void destroy() {
301 updateLogProgress(CURRENT_LINE_NUMBER + "," + LAST_COMPLETE_LOG_FILE_NAME);
302 }
303
304 public String getConfHdfsLogDir() {
305 return HDFS_LOG_DIR;
306 }
307
308 private static void initLogProgress() throws ServiceException{
309 String logProgress = "";
310 XLog log = XLog.getLog(XLogCopyService.class);
311
312 try {
313 Class.forName(jdbcConf.get("driver")).newInstance();
314 Connection conn = DriverManager.getConnection(jdbcConf.get("url"), jdbcConf.get("user"), jdbcConf.get("password"));
315 try {
316 log.info("execute query: " + LOGPROGRESS_GET_QUERY);
317 Statement st = conn.createStatement();
318 ResultSet rs = st.executeQuery(LOGPROGRESS_GET_QUERY);
319 if (rs.next()) {
320 logProgress = rs.getString(1);
321 }
322
323 rs.close();
324 st.close();
325 }
326 catch (SQLException ex) {
327 throw new ServiceException(ErrorCode.E0170, "cannot query OOZIE_SYS table", ex);
328 }
329 }
330 catch (SQLException ex) {
331 throw new ServiceException(ErrorCode.E0600, ex);
332 }
333 catch (Exception ex) {
334 throw new ServiceException(ErrorCode.E0307, ex);
335 }
336
337 if (!logProgress.isEmpty()) {
338 String[] progressRecord = logProgress.trim().split(",");
339 try {
340 CURRENT_LINE_NUMBER = Integer.parseInt(progressRecord[0].trim());
341 }
342 catch (NumberFormatException ex) {
343 log.warn("Can not retrieve line number from log progress file, default it to 0");
344 }
345
346 LAST_COMPLETE_LOG_FILE_NAME = progressRecord[1].trim();
347 }
348 else {
349 try {
350 Class.forName(jdbcConf.get("driver")).newInstance();
351 Connection conn = DriverManager.getConnection(jdbcConf.get("url"), jdbcConf.get("user"), jdbcConf.get("password"));
352 try {
353 log.info("insert query is " + LOGPROGRESS_INSERT_QUERY);
354 Statement st = conn.createStatement();
355 st.executeUpdate(LOGPROGRESS_INSERT_QUERY);
356 st.close();
357 }
358 catch (SQLException ex) {
359 throw new ServiceException(ErrorCode.E0170, "cannot insert into OOZIE_SYS table", ex);
360 }
361 }
362 catch (SQLException ex) {
363 throw new ServiceException(ErrorCode.E0600, ex);
364 }
365 catch (Exception ex) {
366 throw new ServiceException(ErrorCode.E0307, ex);
367 }
368 }
369 }
370
371 private static void updateLogProgress(String logProgress) {
372 XLog log = XLog.getLog(XLogCopyService.class);
373 try {
374 Class.forName(jdbcConf.get("driver")).newInstance();
375 Connection conn = DriverManager.getConnection(jdbcConf.get("url"), jdbcConf.get("user"), jdbcConf.get("password"));
376 try {
377 String updateQuery = LOGPROGRESS_UPDATE_QUERY + logProgress + LOGPROGRESS_UPDATE_QUERY_WHERE_CLAUSE;
378 log.info("execute query; " + updateQuery);
379 Statement st = conn.createStatement();
380 st.executeUpdate(updateQuery);
381 st.close();
382 }
383 catch (SQLException ex) {
384 log.error("Cannot update logProgress in OOZIE_SYS table: " + ex.getMessage());
385 }
386 }
387 catch (Exception ex) {
388 log.error("run time error " + ex.getMessage());
389 }
390 }
391 }