001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import java.io.BufferedReader;
021    
022    import org.apache.oozie.util.Instrumentable;
023    import org.apache.oozie.util.Instrumentation;
024    import org.apache.oozie.util.XLogStreamer;
025    import org.apache.hadoop.conf.Configuration;
026    import org.apache.hadoop.fs.Path;
027    import org.apache.hadoop.fs.FileStatus;
028    import org.apache.hadoop.fs.FileSystem;
029    import org.apache.hadoop.fs.FileUtil;
030    
031    import java.io.IOException;
032    import java.io.InputStreamReader;
033    import java.io.Writer;
034    import java.util.ArrayList;
035    import java.util.Date;
036    import java.util.List;
037    import java.util.Map;
038    import java.util.HashMap;
039    import java.util.TreeMap;
040    import java.net.URI;
041    
042    import org.apache.curator.x.discovery.ServiceInstance;
043    import org.apache.oozie.ErrorCode;
044    import org.apache.oozie.client.OozieClient;
045    import org.apache.oozie.client.rest.RestConstants;
046    import org.apache.oozie.util.AuthUrlClient;
047    import org.apache.oozie.util.XLogFilter;
048    import org.apache.oozie.util.SimpleTimestampedMessageParser;
049    import org.apache.oozie.util.HDFSTimestampedMessageParser;
050    import org.apache.oozie.util.TimestampedMessageParser;
051    import org.apache.oozie.util.XLog;
052    import org.apache.oozie.util.XLogUserFilterParam;
053    import org.apache.oozie.util.ZKUtils;
054    import org.apache.tools.ant.taskdefs.condition.ParserSupports;
055    
056    /**
057     * Service that performs streaming of log files over Web Services if enabled in XLogService and collates logs from other Oozie
058     * servers.  Requires that a ZooKeeper ensemble is available.
059     */
060    public class ZKXLogStreamingService extends XLogStreamingService implements Service, Instrumentable {
061    
062        private ZKUtils zk;
063        private XLog log;
064    
065        /**
066         * Initialize the log streaming service.
067         *
068         * @param services services instance.
069         * @throws ServiceException thrown if the log streaming service could not be initialized.
070         */
071        @Override
072        public void init(Services services) throws ServiceException {
073            super.init(services);
074            try {
075                zk = ZKUtils.register(this);
076            }
077            catch (Exception ex) {
078                throw new ServiceException(ErrorCode.E1700, ex.getMessage(), ex);
079            }
080            log = XLog.getLog(this.getClass());
081        }
082    
083        /**
084         * Destroy the log streaming service.
085         */
086        @Override
087        public void destroy() {
088            if (zk != null) {
089                zk.unregister(this);
090            }
091            zk = null;
092            super.destroy();
093        }
094    
095        /**
096         * Instruments the log streaming service.
097         *
098         * @param instr instrumentation to use.
099         */
100        @Override
101        public void instrument(Instrumentation instr) {
102            super.instrument(instr);
103        }
104    
105        /**
106         * Stream the log of a job.  It contacts any other running Oozie servers to collate relevant logs while streaming.
107         *
108         * @param filter log streamer filter.
109         * @param startTime start time for log events to filter.
110         * @param endTime end time for log events to filter.
111         * @param writer writer to stream the log to.
112         * @param params additional parameters from the request
113         * @throws IOException thrown if the log cannot be streamed.
114         */
115        @Override
116        public void streamLog(XLogFilter filter, Date startTime, Date endTime, Writer writer, Map<String, String[]> params)
117                throws IOException {
118            XLogService xLogService = Services.get().get(XLogService.class);
119            if (xLogService.getLogOverWS()) {
120                // If ALL_SERVERS_PARAM is set to false, then only stream our log
121                if (!Services.get().get(JobsConcurrencyService.class).isAllServerRequest(params)) {
122                    new XLogStreamer(filter, xLogService.getOozieLogPath(), xLogService.getOozieLogName(),
123                            xLogService.getOozieLogRotation()).streamLog(writer, startTime, endTime, bufferLen);
124                }
125                // Otherwise, we have to go collate relevant logs from the other Oozie servers
126                else {
127                    collateLogs(filter, startTime, endTime, writer, params);
128                }
129            }
130            else {
131                writer.write("Log streaming disabled!!");
132            }
133        }
134    
135        /**
136         * Contacts each of the other Oozie servers, gets their logs for the job, collates them, and sends them to the user via the
137         * Writer.  It will make sure to not read all of the log messages into memory at the same time to not use up the heap.  If there
138         * is a problem talking to one of the other servers, it will ignore that server and prepend a message to the Writer about it.
139         * For getting the logs from this server, it won't use the REST API and instead get them directly to be more efficient.
140         *
141         * @param filter
142         * @param startTime
143         * @param endTime
144         * @param writer
145         * @throws IOException
146         */
147        private void collateLogs(XLogFilter filter, Date startTime, Date endTime, Writer writer,
148                Map<String, String[]> params) throws IOException {
149            XLogService xLogService = Services.get().get(XLogService.class);
150            List<String> badOozies = new ArrayList<String>();
151            List<ServiceInstance<Map>> oozies = null;
152            try {
153                oozies = zk.getAllMetaData();
154            }
155            catch (Exception ex) {
156                throw new IOException("Issue communicating with ZooKeeper: " + ex.getMessage(), ex);
157            }
158            List<TimestampedMessageParser> parsers = new ArrayList<TimestampedMessageParser>(oozies.size());
159            boolean openHDFS = false;
160            FileSystem fs = null;
161            try {
162                // Create a BufferedReader for getting the logs of each server and put them in a TimestampedMessageParser
163                for (ServiceInstance<Map> oozie : oozies) {
164                    Map<String, String> oozieMeta = oozie.getPayload();
165                    String otherId = oozieMeta.get(ZKUtils.ZKMetadataKeys.OOZIE_ID);
166                    // If it's this server, we can just get them directly
167                    if (otherId.equals(zk.getZKId())) {
168                        BufferedReader reader = new XLogStreamer(filter, xLogService.getOozieLogPath(), xLogService.getOozieLogName(),
169                                                                 xLogService.getOozieLogRotation()).makeReader(startTime, endTime);
170                        parsers.add(new TimestampedMessageParser(reader, filter));
171                    }
172                    // If it's another server, we'll have to use the REST API
173                    else {
174                        String otherUrl = oozieMeta.get(ZKUtils.ZKMetadataKeys.OOZIE_URL);
175                        String jobId = filter.getFilterParams().get(DagXLogInfoService.JOB);
176                        try {
177                         // It's important that we specify ALL_SERVERS_PARAM=false in the GET request to prevent the other Oozie
178                         // Server from trying aggregate logs from the other Oozie servers (and creating an infinite recursion)
179                            final String url = otherUrl + "/v" + OozieClient.WS_PROTOCOL_VERSION + "/" + RestConstants.JOB
180                                    + "/" + jobId + "?" + RestConstants.JOB_SHOW_PARAM + "=" + RestConstants.JOB_SHOW_LOG
181                                    + "&" + RestConstants.ALL_SERVER_REQUEST + "=false" + AuthUrlClient.getQueryParamString(params);
182    
183                            BufferedReader reader = AuthUrlClient.callServer(url);
184                            parsers.add(new SimpleTimestampedMessageParser(reader, filter));
185                        }
186                        catch(IOException ioe) {
187                            log.warn("Failed to retrieve logs for job [" + jobId + "] from Oozie server with ID [" + otherId
188                                    + "] at [" + otherUrl + "]; log information may be incomplete", ioe);
189                            badOozies.add(otherId);
190                        }
191                    }
192                }
193    
194                //If log param debug is set, we need to write start date and end date to outputstream.
195                if(filter.isDebugMode()){
196                    writer.write(filter.getDebugMessage());
197                }
198    
199                // Add a message about any servers we couldn't contact
200                if (!badOozies.isEmpty()) {
201                    writer.write("Unable to contact the following Oozie Servers for logs (log information may be incomplete):\n");
202                    for (String badOozie : badOozies) {
203                        writer.write("     ");
204                        writer.write(badOozie);
205                        writer.write("\n");
206                    }
207                    writer.write("\n");
208                    writer.flush();
209                }
210    
211                // If it's just the one server (this server), then we don't need to do any more processing and can just copy it directly
212                if (parsers.size() == 1) {
213                    TimestampedMessageParser parser = parsers.get(0);
214                    parser.processRemaining(writer, bufferLen);
215                }
216                else {
217                    // If one oozie server is down, we need to go to hdfs to retrieve the logs
218                    if (!badOozies.isEmpty()) {
219                        XLogCopyService xls = Services.get().get(XLogCopyService.class);
220                        String hdfsDir = xls.getConfHdfsLogDir();
221                        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
222                        URI uri = new Path(hdfsDir).toUri();
223                        Configuration fsConf = has.createJobConf(uri.getAuthority());
224    
225                        try {
226                            fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf);
227                            openHDFS = true;
228                        }
229                        catch (Exception ex) {
230                            log.error(System.getProperty("user.name") + " has to be specified to access hdfs",
231                                    new HadoopAccessorException(ErrorCode.E0902,
232                                            System.getProperty("user.name") +" has to be specified to access FileSystem"));
233                        }
234    
235                        FileStatus[] fileStatuses = fs.listStatus(new Path(hdfsDir));
236                        Path[] paths = FileUtil.stat2Paths(fileStatuses);
237                        String jobId = filter.getFilterParams().get("JOB");
238    
239                        parsers = new ArrayList<TimestampedMessageParser>();
240                        for (Path path: paths) {
241                            Path logFilePath = new Path(path, jobId + ".log");
242                            if (fs.exists(logFilePath)) {
243                                BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(logFilePath)));
244                                parsers.add(new TimestampedMessageParser(reader, filter));
245                            }
246                        }
247                    }
248    
249                    // Now that we have a Reader for each server to get the logs from that server, we have to collate them.  Within each
250                    // server, the logs should already be in the correct order, so we can take advantage of that.  We'll use the
251                    // BufferedReaders to read the messages from the logs of each server and put them in order without having to bring
252                    // every message into memory at the same time.
253                    TreeMap<String, TimestampedMessageParser> timestampMap = new TreeMap<String, TimestampedMessageParser>();
254                    // populate timestampMap with initial values
255                    for (TimestampedMessageParser parser : parsers) {
256                        if (parser.increment()) {
257                            timestampMap.put(parser.getLastTimestamp(), parser);
258                        }
259                    }
260                    int bytesWritten = 0;
261                    while (timestampMap.size() > 1) {
262                        // The first entry will be the earliest based on the timestamp (also removes it) from the map
263                        TimestampedMessageParser earliestParser = timestampMap.pollFirstEntry().getValue();
264                        // Write the message from that parser at that timestamp
265                        writer.write(earliestParser.getLastMessage());
266                        bytesWritten = earliestParser.getLastMessage().length();
267                        if (bytesWritten > bufferLen) {
268                            writer.flush();
269                            bytesWritten = 0;
270                        }
271                        // Increment that parser to read the next message
272                        if (earliestParser.increment()) {
273                            // If it still has messages left, put it back in the map with the new last timestamp for it
274                            timestampMap.put(earliestParser.getLastTimestamp(), earliestParser);
275                        }
276                    }
277                    // If there's only one parser left in the map, then we can simply copy the rest of its lines directly to be faster
278                    if (timestampMap.size() == 1) {
279                        TimestampedMessageParser parser = timestampMap.values().iterator().next();
280                        writer.write(parser.getLastMessage());  // don't forget the last message read by the parser
281                        parser.processRemaining(writer, bufferLen, bytesWritten + parser.getLastMessage().length());
282                    }
283                }
284            }
285            finally {
286                for (TimestampedMessageParser parser : parsers) {
287                    parser.closeReader();
288                }
289                writer.flush();
290                if (openHDFS) {
291                    fs.close();
292                }
293            }
294        }
295    }