001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.service;
019
020 import java.io.BufferedReader;
021
022 import org.apache.oozie.util.Instrumentable;
023 import org.apache.oozie.util.Instrumentation;
024 import org.apache.oozie.util.XLogStreamer;
025 import org.apache.hadoop.conf.Configuration;
026 import org.apache.hadoop.fs.Path;
027 import org.apache.hadoop.fs.FileStatus;
028 import org.apache.hadoop.fs.FileSystem;
029 import org.apache.hadoop.fs.FileUtil;
030
031 import java.io.IOException;
032 import java.io.InputStreamReader;
033 import java.io.Writer;
034 import java.util.ArrayList;
035 import java.util.Date;
036 import java.util.List;
037 import java.util.Map;
038 import java.util.HashMap;
039 import java.util.TreeMap;
040 import java.net.URI;
041
042 import org.apache.curator.x.discovery.ServiceInstance;
043 import org.apache.oozie.ErrorCode;
044 import org.apache.oozie.client.OozieClient;
045 import org.apache.oozie.client.rest.RestConstants;
046 import org.apache.oozie.util.AuthUrlClient;
047 import org.apache.oozie.util.XLogFilter;
048 import org.apache.oozie.util.SimpleTimestampedMessageParser;
049 import org.apache.oozie.util.HDFSTimestampedMessageParser;
050 import org.apache.oozie.util.TimestampedMessageParser;
051 import org.apache.oozie.util.XLog;
052 import org.apache.oozie.util.XLogUserFilterParam;
053 import org.apache.oozie.util.ZKUtils;
054 import org.apache.tools.ant.taskdefs.condition.ParserSupports;
055
056 /**
057 * Service that performs streaming of log files over Web Services if enabled in XLogService and collates logs from other Oozie
058 * servers. Requires that a ZooKeeper ensemble is available.
059 */
060 public class ZKXLogStreamingService extends XLogStreamingService implements Service, Instrumentable {
061
062 private ZKUtils zk;
063 private XLog log;
064
065 /**
066 * Initialize the log streaming service.
067 *
068 * @param services services instance.
069 * @throws ServiceException thrown if the log streaming service could not be initialized.
070 */
071 @Override
072 public void init(Services services) throws ServiceException {
073 super.init(services);
074 try {
075 zk = ZKUtils.register(this);
076 }
077 catch (Exception ex) {
078 throw new ServiceException(ErrorCode.E1700, ex.getMessage(), ex);
079 }
080 log = XLog.getLog(this.getClass());
081 }
082
083 /**
084 * Destroy the log streaming service.
085 */
086 @Override
087 public void destroy() {
088 if (zk != null) {
089 zk.unregister(this);
090 }
091 zk = null;
092 super.destroy();
093 }
094
095 /**
096 * Instruments the log streaming service.
097 *
098 * @param instr instrumentation to use.
099 */
100 @Override
101 public void instrument(Instrumentation instr) {
102 super.instrument(instr);
103 }
104
105 /**
106 * Stream the log of a job. It contacts any other running Oozie servers to collate relevant logs while streaming.
107 *
108 * @param filter log streamer filter.
109 * @param startTime start time for log events to filter.
110 * @param endTime end time for log events to filter.
111 * @param writer writer to stream the log to.
112 * @param params additional parameters from the request
113 * @throws IOException thrown if the log cannot be streamed.
114 */
115 @Override
116 public void streamLog(XLogFilter filter, Date startTime, Date endTime, Writer writer, Map<String, String[]> params)
117 throws IOException {
118 XLogService xLogService = Services.get().get(XLogService.class);
119 if (xLogService.getLogOverWS()) {
120 // If ALL_SERVERS_PARAM is set to false, then only stream our log
121 if (!Services.get().get(JobsConcurrencyService.class).isAllServerRequest(params)) {
122 new XLogStreamer(filter, xLogService.getOozieLogPath(), xLogService.getOozieLogName(),
123 xLogService.getOozieLogRotation()).streamLog(writer, startTime, endTime, bufferLen);
124 }
125 // Otherwise, we have to go collate relevant logs from the other Oozie servers
126 else {
127 collateLogs(filter, startTime, endTime, writer, params);
128 }
129 }
130 else {
131 writer.write("Log streaming disabled!!");
132 }
133 }
134
135 /**
136 * Contacts each of the other Oozie servers, gets their logs for the job, collates them, and sends them to the user via the
137 * Writer. It will make sure to not read all of the log messages into memory at the same time to not use up the heap. If there
138 * is a problem talking to one of the other servers, it will ignore that server and prepend a message to the Writer about it.
139 * For getting the logs from this server, it won't use the REST API and instead get them directly to be more efficient.
140 *
141 * @param filter
142 * @param startTime
143 * @param endTime
144 * @param writer
145 * @throws IOException
146 */
147 private void collateLogs(XLogFilter filter, Date startTime, Date endTime, Writer writer,
148 Map<String, String[]> params) throws IOException {
149 XLogService xLogService = Services.get().get(XLogService.class);
150 List<String> badOozies = new ArrayList<String>();
151 List<ServiceInstance<Map>> oozies = null;
152 try {
153 oozies = zk.getAllMetaData();
154 }
155 catch (Exception ex) {
156 throw new IOException("Issue communicating with ZooKeeper: " + ex.getMessage(), ex);
157 }
158 List<TimestampedMessageParser> parsers = new ArrayList<TimestampedMessageParser>(oozies.size());
159 boolean openHDFS = false;
160 FileSystem fs = null;
161 try {
162 // Create a BufferedReader for getting the logs of each server and put them in a TimestampedMessageParser
163 for (ServiceInstance<Map> oozie : oozies) {
164 Map<String, String> oozieMeta = oozie.getPayload();
165 String otherId = oozieMeta.get(ZKUtils.ZKMetadataKeys.OOZIE_ID);
166 // If it's this server, we can just get them directly
167 if (otherId.equals(zk.getZKId())) {
168 BufferedReader reader = new XLogStreamer(filter, xLogService.getOozieLogPath(), xLogService.getOozieLogName(),
169 xLogService.getOozieLogRotation()).makeReader(startTime, endTime);
170 parsers.add(new TimestampedMessageParser(reader, filter));
171 }
172 // If it's another server, we'll have to use the REST API
173 else {
174 String otherUrl = oozieMeta.get(ZKUtils.ZKMetadataKeys.OOZIE_URL);
175 String jobId = filter.getFilterParams().get(DagXLogInfoService.JOB);
176 try {
177 // It's important that we specify ALL_SERVERS_PARAM=false in the GET request to prevent the other Oozie
178 // Server from trying aggregate logs from the other Oozie servers (and creating an infinite recursion)
179 final String url = otherUrl + "/v" + OozieClient.WS_PROTOCOL_VERSION + "/" + RestConstants.JOB
180 + "/" + jobId + "?" + RestConstants.JOB_SHOW_PARAM + "=" + RestConstants.JOB_SHOW_LOG
181 + "&" + RestConstants.ALL_SERVER_REQUEST + "=false" + AuthUrlClient.getQueryParamString(params);
182
183 BufferedReader reader = AuthUrlClient.callServer(url);
184 parsers.add(new SimpleTimestampedMessageParser(reader, filter));
185 }
186 catch(IOException ioe) {
187 log.warn("Failed to retrieve logs for job [" + jobId + "] from Oozie server with ID [" + otherId
188 + "] at [" + otherUrl + "]; log information may be incomplete", ioe);
189 badOozies.add(otherId);
190 }
191 }
192 }
193
194 //If log param debug is set, we need to write start date and end date to outputstream.
195 if(filter.isDebugMode()){
196 writer.write(filter.getDebugMessage());
197 }
198
199 // Add a message about any servers we couldn't contact
200 if (!badOozies.isEmpty()) {
201 writer.write("Unable to contact the following Oozie Servers for logs (log information may be incomplete):\n");
202 for (String badOozie : badOozies) {
203 writer.write(" ");
204 writer.write(badOozie);
205 writer.write("\n");
206 }
207 writer.write("\n");
208 writer.flush();
209 }
210
211 // If it's just the one server (this server), then we don't need to do any more processing and can just copy it directly
212 if (parsers.size() == 1) {
213 TimestampedMessageParser parser = parsers.get(0);
214 parser.processRemaining(writer, bufferLen);
215 }
216 else {
217 // If one oozie server is down, we need to go to hdfs to retrieve the logs
218 if (!badOozies.isEmpty()) {
219 XLogCopyService xls = Services.get().get(XLogCopyService.class);
220 String hdfsDir = xls.getConfHdfsLogDir();
221 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
222 URI uri = new Path(hdfsDir).toUri();
223 Configuration fsConf = has.createJobConf(uri.getAuthority());
224
225 try {
226 fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf);
227 openHDFS = true;
228 }
229 catch (Exception ex) {
230 log.error(System.getProperty("user.name") + " has to be specified to access hdfs",
231 new HadoopAccessorException(ErrorCode.E0902,
232 System.getProperty("user.name") +" has to be specified to access FileSystem"));
233 }
234
235 FileStatus[] fileStatuses = fs.listStatus(new Path(hdfsDir));
236 Path[] paths = FileUtil.stat2Paths(fileStatuses);
237 String jobId = filter.getFilterParams().get("JOB");
238
239 parsers = new ArrayList<TimestampedMessageParser>();
240 for (Path path: paths) {
241 Path logFilePath = new Path(path, jobId + ".log");
242 if (fs.exists(logFilePath)) {
243 BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(logFilePath)));
244 parsers.add(new TimestampedMessageParser(reader, filter));
245 }
246 }
247 }
248
249 // Now that we have a Reader for each server to get the logs from that server, we have to collate them. Within each
250 // server, the logs should already be in the correct order, so we can take advantage of that. We'll use the
251 // BufferedReaders to read the messages from the logs of each server and put them in order without having to bring
252 // every message into memory at the same time.
253 TreeMap<String, TimestampedMessageParser> timestampMap = new TreeMap<String, TimestampedMessageParser>();
254 // populate timestampMap with initial values
255 for (TimestampedMessageParser parser : parsers) {
256 if (parser.increment()) {
257 timestampMap.put(parser.getLastTimestamp(), parser);
258 }
259 }
260 int bytesWritten = 0;
261 while (timestampMap.size() > 1) {
262 // The first entry will be the earliest based on the timestamp (also removes it) from the map
263 TimestampedMessageParser earliestParser = timestampMap.pollFirstEntry().getValue();
264 // Write the message from that parser at that timestamp
265 writer.write(earliestParser.getLastMessage());
266 bytesWritten = earliestParser.getLastMessage().length();
267 if (bytesWritten > bufferLen) {
268 writer.flush();
269 bytesWritten = 0;
270 }
271 // Increment that parser to read the next message
272 if (earliestParser.increment()) {
273 // If it still has messages left, put it back in the map with the new last timestamp for it
274 timestampMap.put(earliestParser.getLastTimestamp(), earliestParser);
275 }
276 }
277 // If there's only one parser left in the map, then we can simply copy the rest of its lines directly to be faster
278 if (timestampMap.size() == 1) {
279 TimestampedMessageParser parser = timestampMap.values().iterator().next();
280 writer.write(parser.getLastMessage()); // don't forget the last message read by the parser
281 parser.processRemaining(writer, bufferLen, bytesWritten + parser.getLastMessage().length());
282 }
283 }
284 }
285 finally {
286 for (TimestampedMessageParser parser : parsers) {
287 parser.closeReader();
288 }
289 writer.flush();
290 if (openHDFS) {
291 fs.close();
292 }
293 }
294 }
295 }