001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie;
019
020 import org.apache.oozie.service.XLogService;
021 import org.apache.oozie.service.DagXLogInfoService;
022 import org.apache.hadoop.conf.Configuration;
023 import org.apache.oozie.client.CoordinatorJob;
024 import org.apache.oozie.client.WorkflowJob;
025 import org.apache.oozie.client.OozieClient;
026 import org.apache.oozie.command.CommandException;
027 import org.apache.oozie.command.wf.CompletedActionXCommand;
028 import org.apache.oozie.command.wf.DefinitionXCommand;
029 import org.apache.oozie.command.wf.ExternalIdXCommand;
030 import org.apache.oozie.command.wf.JobXCommand;
031 import org.apache.oozie.command.wf.JobsXCommand;
032 import org.apache.oozie.command.wf.KillXCommand;
033 import org.apache.oozie.command.wf.ReRunXCommand;
034 import org.apache.oozie.command.wf.ResumeXCommand;
035 import org.apache.oozie.command.wf.StartXCommand;
036 import org.apache.oozie.command.wf.SubmitHiveXCommand;
037 import org.apache.oozie.command.wf.SubmitHttpXCommand;
038 import org.apache.oozie.command.wf.SubmitMRXCommand;
039 import org.apache.oozie.command.wf.SubmitPigXCommand;
040 import org.apache.oozie.command.wf.SubmitSqoopXCommand;
041 import org.apache.oozie.command.wf.SubmitXCommand;
042 import org.apache.oozie.command.wf.SuspendXCommand;
043 import org.apache.oozie.command.wf.WorkflowActionInfoXCommand;
044 import org.apache.oozie.executor.jpa.JPAExecutorException;
045 import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor;
046 import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery;
047 import org.apache.oozie.service.Services;
048 import org.apache.oozie.service.CallableQueueService;
049 import org.apache.oozie.util.XLogFilter;
050 import org.apache.oozie.util.XLogUserFilterParam;
051 import org.apache.oozie.util.ParamChecker;
052 import org.apache.oozie.util.XCallable;
053 import org.apache.oozie.util.XConfiguration;
054 import org.apache.oozie.util.XLog;
055 import org.apache.oozie.service.XLogStreamingService;
056
057 import java.io.StringReader;
058 import java.io.Writer;
059 import java.util.Date;
060 import java.util.List;
061 import java.util.Properties;
062 import java.util.Set;
063 import java.util.HashSet;
064 import java.util.StringTokenizer;
065 import java.util.Map;
066 import java.util.HashMap;
067 import java.util.ArrayList;
068 import java.io.IOException;
069
070 /**
071 * The DagEngine provides all the DAG engine functionality for WS calls.
072 */
073 public class DagEngine extends BaseEngine {
074
075 private static final int HIGH_PRIORITY = 2;
076 private static XLog LOG = XLog.getLog(DagEngine.class);
077
078 /**
079 * Create a system Dag engine, with no user and no group.
080 */
081 public DagEngine() {
082 if (Services.get().getConf().getBoolean(USE_XCOMMAND, true) == false) {
083 LOG.debug("Oozie DagEngine is not using XCommands.");
084 }
085 else {
086 LOG.debug("Oozie DagEngine is using XCommands.");
087 }
088 }
089
090 /**
091 * Create a Dag engine to perform operations on behave of a user.
092 *
093 * @param user user name.
094 */
095 public DagEngine(String user) {
096 this();
097
098 this.user = ParamChecker.notEmpty(user, "user");
099 }
100
101 /**
102 * Submit a workflow job. <p/> It validates configuration properties.
103 *
104 * @param conf job configuration.
105 * @param startJob indicates if the job should be started or not.
106 * @return the job Id.
107 * @throws DagEngineException thrown if the job could not be created.
108 */
109 @Override
110 public String submitJob(Configuration conf, boolean startJob) throws DagEngineException {
111 validateSubmitConfiguration(conf);
112
113 try {
114 String jobId;
115 SubmitXCommand submit = new SubmitXCommand(conf);
116 jobId = submit.call();
117 if (startJob) {
118 start(jobId);
119 }
120 return jobId;
121 }
122 catch (CommandException ex) {
123 throw new DagEngineException(ex);
124 }
125 }
126
127 /**
128 * Submit a workflow through a coordinator. It validates configuration properties.
129 * @param conf job conf
130 * @param parentId parent of workflow
131 * @return
132 * @throws DagEngineException
133 */
134 public String submitJobFromCoordinator(Configuration conf, String parentId) throws DagEngineException {
135 validateSubmitConfiguration(conf);
136 try {
137 String jobId;
138 SubmitXCommand submit = new SubmitXCommand(conf, parentId);
139 jobId = submit.call();
140 start(jobId);
141 return jobId;
142 }
143 catch (CommandException ex) {
144 throw new DagEngineException(ex);
145 }
146 }
147
148 /**
149 * Submit a pig/hive/mapreduce job through HTTP.
150 * <p/>
151 * It validates configuration properties.
152 *
153 * @param conf job configuration.
154 * @param jobType job type - can be "pig", "hive", "sqoop" or "mapreduce".
155 * @return the job Id.
156 * @throws DagEngineException thrown if the job could not be created.
157 */
158 public String submitHttpJob(Configuration conf, String jobType) throws DagEngineException {
159 validateSubmitConfiguration(conf);
160
161 try {
162 String jobId;
163 SubmitHttpXCommand submit = null;
164 if (jobType.equals("pig")) {
165 submit = new SubmitPigXCommand(conf);
166 }
167 else if (jobType.equals("mapreduce")) {
168 submit = new SubmitMRXCommand(conf);
169 }
170 else if (jobType.equals("hive")) {
171 submit = new SubmitHiveXCommand(conf);
172 }
173 else if (jobType.equals("sqoop")) {
174 submit = new SubmitSqoopXCommand(conf);
175 }
176
177 jobId = submit.call();
178 start(jobId);
179 return jobId;
180 }
181 catch (CommandException ex) {
182 throw new DagEngineException(ex);
183 }
184 }
185
186 private void validateSubmitConfiguration(Configuration conf) throws DagEngineException {
187 if (conf.get(OozieClient.APP_PATH) == null) {
188 throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH);
189 }
190 }
191
192 /**
193 * Start a job.
194 *
195 * @param jobId job Id.
196 * @throws DagEngineException thrown if the job could not be started.
197 */
198 @Override
199 public void start(String jobId) throws DagEngineException {
200 // Changing to synchronous call from asynchronous queuing to prevent the
201 // loss of command if the queue is full or the queue is lost in case of
202 // failure.
203 try {
204 new StartXCommand(jobId).call();
205 }
206 catch (CommandException e) {
207 throw new DagEngineException(e);
208 }
209 }
210
211 /**
212 * Resume a job.
213 *
214 * @param jobId job Id.
215 * @throws DagEngineException thrown if the job could not be resumed.
216 */
217 @Override
218 public void resume(String jobId) throws DagEngineException {
219 // Changing to synchronous call from asynchronous queuing to prevent the
220 // loss of command if the queue is full or the queue is lost in case of
221 // failure.
222 try {
223 new ResumeXCommand(jobId).call();
224 }
225 catch (CommandException e) {
226 throw new DagEngineException(e);
227 }
228 }
229
230 /**
231 * Suspend a job.
232 *
233 * @param jobId job Id.
234 * @throws DagEngineException thrown if the job could not be suspended.
235 */
236 @Override
237 public void suspend(String jobId) throws DagEngineException {
238 // Changing to synchronous call from asynchronous queuing to prevent the
239 // loss of command if the queue is full or the queue is lost in case of
240 // failure.
241 try {
242 new SuspendXCommand(jobId).call();
243 }
244 catch (CommandException e) {
245 throw new DagEngineException(e);
246 }
247 }
248
249 /**
250 * Kill a job.
251 *
252 * @param jobId job Id.
253 * @throws DagEngineException thrown if the job could not be killed.
254 */
255 @Override
256 public void kill(String jobId) throws DagEngineException {
257 // Changing to synchronous call from asynchronous queuing to prevent the
258 // loss of command if the queue is full or the queue is lost in case of
259 // failure.
260 try {
261 new KillXCommand(jobId).call();
262 LOG.info("User " + user + " killed the WF job " + jobId);
263 }
264 catch (CommandException e) {
265 throw new DagEngineException(e);
266 }
267 }
268
269 /* (non-Javadoc)
270 * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String)
271 */
272 @Override
273 public void change(String jobId, String changeValue) throws DagEngineException {
274 // This code should not be reached.
275 throw new DagEngineException(ErrorCode.E1017);
276 }
277
278 /**
279 * Rerun a job.
280 *
281 * @param jobId job Id to rerun.
282 * @param conf configuration information for the rerun.
283 * @throws DagEngineException thrown if the job could not be rerun.
284 */
285 @Override
286 public void reRun(String jobId, Configuration conf) throws DagEngineException {
287 try {
288 WorkflowJobBean wfBean = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId);
289 Configuration wfConf = new XConfiguration(new StringReader(wfBean.getConf()));
290 XConfiguration.copy(conf, wfConf);
291 validateReRunConfiguration(wfConf);
292 new ReRunXCommand(jobId, wfConf).call();
293 }
294 catch (CommandException ex) {
295 throw new DagEngineException(ex);
296 }
297 catch (JPAExecutorException ex) {
298 throw new DagEngineException(ex);
299 }
300 catch (IOException ex) {
301 throw new DagEngineException(ErrorCode.E0803, ex.getMessage());
302 }
303 }
304
305 private void validateReRunConfiguration(Configuration conf) throws DagEngineException {
306 if (conf.get(OozieClient.APP_PATH) == null) {
307 throw new DagEngineException(ErrorCode.E0401, OozieClient.APP_PATH);
308 }
309 if (conf.get(OozieClient.RERUN_SKIP_NODES) == null && conf.get(OozieClient.RERUN_FAIL_NODES) == null) {
310 throw new DagEngineException(ErrorCode.E0401, OozieClient.RERUN_SKIP_NODES + " OR "
311 + OozieClient.RERUN_FAIL_NODES);
312 }
313 if (conf.get(OozieClient.RERUN_SKIP_NODES) != null && conf.get(OozieClient.RERUN_FAIL_NODES) != null) {
314 throw new DagEngineException(ErrorCode.E0404, OozieClient.RERUN_SKIP_NODES + " OR "
315 + OozieClient.RERUN_FAIL_NODES);
316 }
317 }
318
319 /**
320 * Process an action callback.
321 *
322 * @param actionId the action Id.
323 * @param externalStatus the action external status.
324 * @param actionData the action output data, <code>null</code> if none.
325 * @throws DagEngineException thrown if the callback could not be processed.
326 */
327 public void processCallback(String actionId, String externalStatus, Properties actionData)
328 throws DagEngineException {
329 XLog.Info.get().clearParameter(XLogService.GROUP);
330 XLog.Info.get().clearParameter(XLogService.USER);
331 XCallable<Void> command = null;
332
333 command = new CompletedActionXCommand(actionId, externalStatus,
334 actionData, HIGH_PRIORITY);
335 if (!Services.get().get(CallableQueueService.class).queue(command)) {
336 LOG.warn(XLog.OPS, "queue is full or system is in SAFEMODE, ignoring callback");
337 }
338 }
339
340 /**
341 * Return the info about a job.
342 *
343 * @param jobId job Id.
344 * @return the workflow job info.
345 * @throws DagEngineException thrown if the job info could not be obtained.
346 */
347 @Override
348 public WorkflowJob getJob(String jobId) throws DagEngineException {
349 try {
350 return new JobXCommand(jobId).call();
351 }
352 catch (CommandException ex) {
353 throw new DagEngineException(ex);
354 }
355 }
356
357 /**
358 * Return the info about a job with actions subset.
359 *
360 * @param jobId job Id
361 * @param start starting from this index in the list of actions belonging to the job
362 * @param length number of actions to be returned
363 * @return the workflow job info.
364 * @throws DagEngineException thrown if the job info could not be obtained.
365 */
366 @Override
367 public WorkflowJob getJob(String jobId, int start, int length) throws DagEngineException {
368 try {
369 return new JobXCommand(jobId, start, length).call();
370 }
371 catch (CommandException ex) {
372 throw new DagEngineException(ex);
373 }
374 }
375
376 /**
377 * Return the a job definition.
378 *
379 * @param jobId job Id.
380 * @return the job definition.
381 * @throws DagEngineException thrown if the job definition could no be obtained.
382 */
383 @Override
384 public String getDefinition(String jobId) throws DagEngineException {
385 try {
386 return new DefinitionXCommand(jobId).call();
387 }
388 catch (CommandException ex) {
389 throw new DagEngineException(ex);
390 }
391 }
392
393 /**
394 * Stream the log of a job.
395 *
396 * @param jobId job Id.
397 * @param writer writer to stream the log to.
398 * @param params additional parameters from the request
399 * @throws IOException thrown if the log cannot be streamed.
400 * @throws DagEngineException thrown if there is error in getting the Workflow Information for jobId.
401 */
402 @Override
403 public void streamLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
404 DagEngineException {
405 try {
406 XLogFilter filter = new XLogFilter(new XLogUserFilterParam(params));
407 filter.setParameter(DagXLogInfoService.JOB, jobId);
408 WorkflowJob job = getJob(jobId);
409 Date lastTime = job.getEndTime();
410 if (lastTime == null) {
411 lastTime = job.getLastModifiedTime();
412 }
413 Services.get().get(XLogStreamingService.class).streamLog(filter, job.getCreatedTime(), lastTime, writer, params);
414 }
415 catch (Exception e) {
416 throw new IOException(e);
417 }
418 }
419
420 private static final Set<String> FILTER_NAMES = new HashSet<String>();
421
422 static {
423 FILTER_NAMES.add(OozieClient.FILTER_USER);
424 FILTER_NAMES.add(OozieClient.FILTER_NAME);
425 FILTER_NAMES.add(OozieClient.FILTER_GROUP);
426 FILTER_NAMES.add(OozieClient.FILTER_STATUS);
427 FILTER_NAMES.add(OozieClient.FILTER_ID);
428 }
429
430 /**
431 * Validate a jobs filter.
432 *
433 * @param filter filter to validate.
434 * @return the parsed filter.
435 * @throws DagEngineException thrown if the filter is invalid.
436 */
437 protected Map<String, List<String>> parseFilter(String filter) throws DagEngineException {
438 Map<String, List<String>> map = new HashMap<String, List<String>>();
439 if (filter != null) {
440 StringTokenizer st = new StringTokenizer(filter, ";");
441 while (st.hasMoreTokens()) {
442 String token = st.nextToken();
443 if (token.contains("=")) {
444 String[] pair = token.split("=");
445 if (pair.length != 2) {
446 throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
447 }
448 if (!FILTER_NAMES.contains(pair[0])) {
449 throw new DagEngineException(ErrorCode.E0420, filter, XLog
450 .format("invalid name [{0}]", pair[0]));
451 }
452 if (pair[0].equals("status")) {
453 try {
454 WorkflowJob.Status.valueOf(pair[1]);
455 }
456 catch (IllegalArgumentException ex) {
457 throw new DagEngineException(ErrorCode.E0420, filter, XLog.format("invalid status [{0}]",
458 pair[1]));
459 }
460 }
461 List<String> list = map.get(pair[0]);
462 if (list == null) {
463 list = new ArrayList<String>();
464 map.put(pair[0], list);
465 }
466 list.add(pair[1]);
467 }
468 else {
469 throw new DagEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
470 }
471 }
472 }
473 return map;
474 }
475
476 /**
477 * Return the info about a set of jobs.
478 *
479 * @param filter job filter. Refer to the {@link org.apache.oozie.client.OozieClient} for the filter syntax.
480 * @param start offset, base 1.
481 * @param len number of jobs to return.
482 * @return job info for all matching jobs, the jobs don't contain node action information.
483 * @throws DagEngineException thrown if the jobs info could not be obtained.
484 */
485 public WorkflowsInfo getJobs(String filter, int start, int len) throws DagEngineException {
486 Map<String, List<String>> filterList = parseFilter(filter);
487 try {
488 return new JobsXCommand(filterList, start, len).call();
489 }
490 catch (CommandException dce) {
491 throw new DagEngineException(dce);
492 }
493 }
494
495 /**
496 * Return the workflow Job ID for an external ID. <p/> This is reverse lookup for recovery purposes.
497 *
498 * @param externalId external ID provided at job submission time.
499 * @return the associated workflow job ID if any, <code>null</code> if none.
500 * @throws DagEngineException thrown if the lookup could not be done.
501 */
502 @Override
503 public String getJobIdForExternalId(String externalId) throws DagEngineException {
504 try {
505 return new ExternalIdXCommand(externalId).call();
506 }
507 catch (CommandException dce) {
508 throw new DagEngineException(dce);
509 }
510 }
511
512 @Override
513 public CoordinatorJob getCoordJob(String jobId) throws BaseEngineException {
514 throw new BaseEngineException(new XException(ErrorCode.E0301, "cannot get a coordinator job from DagEngine"));
515 }
516
517 @Override
518 public CoordinatorJob getCoordJob(String jobId, String filter, int start, int length, boolean desc)
519 throws BaseEngineException {
520 throw new BaseEngineException(new XException(ErrorCode.E0301, "cannot get a coordinator job from DagEngine"));
521 }
522
523 public WorkflowActionBean getWorkflowAction(String actionId) throws BaseEngineException {
524 try {
525 return new WorkflowActionInfoXCommand(actionId).call();
526 }
527 catch (CommandException ex) {
528 throw new BaseEngineException(ex);
529 }
530 }
531
532 /* (non-Javadoc)
533 * @see org.apache.oozie.BaseEngine#dryRunSubmit(org.apache.hadoop.conf.Configuration)
534 */
535 @Override
536 public String dryRunSubmit(Configuration conf) throws BaseEngineException {
537 try {
538 SubmitXCommand submit = new SubmitXCommand(true, conf);
539 return submit.call();
540 } catch (CommandException ex) {
541 throw new DagEngineException(ex);
542 }
543 }
544 }