001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.command.wf;
019
020 import java.io.IOException;
021 import java.io.StringReader;
022 import java.net.URI;
023 import java.net.URISyntaxException;
024 import java.util.Date;
025 import java.util.Properties;
026 import java.util.Set;
027
028 import org.apache.hadoop.conf.Configuration;
029 import org.apache.hadoop.fs.FileSystem;
030 import org.apache.hadoop.fs.Path;
031 import org.apache.oozie.DagELFunctions;
032 import org.apache.oozie.ErrorCode;
033 import org.apache.oozie.WorkflowActionBean;
034 import org.apache.oozie.WorkflowJobBean;
035 import org.apache.oozie.action.ActionExecutor;
036 import org.apache.oozie.client.WorkflowAction;
037 import org.apache.oozie.client.WorkflowJob;
038 import org.apache.oozie.command.CommandException;
039 import org.apache.oozie.service.CallbackService;
040 import org.apache.oozie.service.ELService;
041 import org.apache.oozie.service.HadoopAccessorException;
042 import org.apache.oozie.service.HadoopAccessorService;
043 import org.apache.oozie.service.JPAService;
044 import org.apache.oozie.service.LiteWorkflowStoreService;
045 import org.apache.oozie.service.Services;
046 import org.apache.oozie.util.ELEvaluator;
047 import org.apache.oozie.util.InstrumentUtils;
048 import org.apache.oozie.util.Instrumentation;
049 import org.apache.oozie.util.XConfiguration;
050 import org.apache.oozie.workflow.WorkflowException;
051 import org.apache.oozie.workflow.WorkflowInstance;
052 import org.apache.oozie.workflow.lite.LiteWorkflowInstance;
053
054 /**
055 * Base class for Action execution commands. Provides common functionality to handle different types of errors while
056 * attempting to start or end an action.
057 */
058 public abstract class ActionXCommand<T> extends WorkflowXCommand<Void> {
059 private static final String INSTRUMENTATION_GROUP = "action.executors";
060
061 protected static final String RECOVERY_ID_SEPARATOR = "@";
062
063 public ActionXCommand(String name, String type, int priority) {
064 super(name, type, priority);
065 }
066
067 /**
068 * Takes care of Transient failures. Sets the action status to retry and increments the retry count if not enough
069 * attempts have been made. Otherwise returns false.
070 *
071 * @param context the execution context.
072 * @param executor the executor instance being used.
073 * @param status the status to be set for the action.
074 * @return true if the action is scheduled for another retry. false if the number of retries has exceeded the
075 * maximum number of configured retries.
076 * @throws CommandException thrown if unable to handle transient
077 */
078 protected boolean handleTransient(ActionExecutor.Context context, ActionExecutor executor,
079 WorkflowAction.Status status) throws CommandException {
080 LOG.debug("Attempting to retry");
081 ActionExecutorContext aContext = (ActionExecutorContext) context;
082 WorkflowActionBean action = (WorkflowActionBean) aContext.getAction();
083 incrActionErrorCounter(action.getType(), "transient", 1);
084
085 int actionRetryCount = action.getRetries();
086 if (actionRetryCount >= executor.getMaxRetries()) {
087 LOG.warn("Exceeded max retry count [{0}]. Suspending Job", executor.getMaxRetries());
088 return false;
089 }
090 else {
091 action.setStatus(status);
092 action.setPending();
093 action.incRetries();
094 long retryDelayMillis = executor.getRetryInterval() * 1000;
095 action.setPendingAge(new Date(System.currentTimeMillis() + retryDelayMillis));
096 LOG.info("Next Retry, Attempt Number [{0}] in [{1}] milliseconds", actionRetryCount + 1, retryDelayMillis);
097 this.resetUsed();
098 queue(this, retryDelayMillis);
099 return true;
100 }
101 }
102
103 /**
104 * Takes care of non transient failures. The job is suspended, and the state of the action is changed to *MANUAL and
105 * set pending flag of action to false
106 *
107 * @param context the execution context.
108 * @param executor the executor instance being used.
109 * @param status the status to be set for the action.
110 * @throws CommandException thrown if unable to suspend job
111 */
112 protected void handleNonTransient(ActionExecutor.Context context, ActionExecutor executor,
113 WorkflowAction.Status status) throws CommandException {
114 ActionExecutorContext aContext = (ActionExecutorContext) context;
115 WorkflowActionBean action = (WorkflowActionBean) aContext.getAction();
116 incrActionErrorCounter(action.getType(), "nontransient", 1);
117 WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
118 String id = workflow.getId();
119 action.setStatus(status);
120 action.resetPendingOnly();
121 LOG.warn("Suspending Workflow Job id=" + id);
122 try {
123 SuspendXCommand.suspendJob(Services.get().get(JPAService.class), workflow, id, action.getId(), null);
124 }
125 catch (Exception e) {
126 throw new CommandException(ErrorCode.E0727, id, e.getMessage());
127 }
128 finally {
129 updateParentIfNecessary(workflow, 3);
130 }
131 }
132
133 /**
134 * Takes care of errors. </p> For errors while attempting to start the action, the job state is updated and an
135 * {@link ActionEndCommand} is queued. </p> For errors while attempting to end the action, the job state is updated.
136 * </p>
137 *
138 * @param context the execution context.
139 * @param executor the executor instance being used.
140 * @param message
141 * @param isStart whether the error was generated while starting or ending an action.
142 * @param status the status to be set for the action.
143 * @throws CommandException thrown if unable to handle action error
144 */
145 protected void handleError(ActionExecutor.Context context, ActionExecutor executor, String message,
146 boolean isStart, WorkflowAction.Status status) throws CommandException {
147 LOG.warn("Setting Action Status to [{0}]", status);
148 ActionExecutorContext aContext = (ActionExecutorContext) context;
149 WorkflowActionBean action = (WorkflowActionBean) aContext.getAction();
150
151 if (!handleUserRetry(action)) {
152 incrActionErrorCounter(action.getType(), "error", 1);
153 action.setPending();
154 if (isStart) {
155 action.setExecutionData(message, null);
156 queue(new ActionEndXCommand(action.getId(), action.getType()));
157 }
158 else {
159 action.setEndData(status, WorkflowAction.Status.ERROR.toString());
160 }
161 }
162 }
163
164 /**
165 * Fail the job due to failed action
166 *
167 * @param context the execution context.
168 * @throws CommandException thrown if unable to fail job
169 */
170 public void failJob(ActionExecutor.Context context) throws CommandException {
171 ActionExecutorContext aContext = (ActionExecutorContext) context;
172 WorkflowActionBean action = (WorkflowActionBean) aContext.getAction();
173 failJob(context, action);
174 }
175
176 /**
177 * Fail the job due to failed action
178 *
179 * @param context the execution context.
180 * @param action the action that caused the workflow to fail
181 * @throws CommandException thrown if unable to fail job
182 */
183 public void failJob(ActionExecutor.Context context, WorkflowActionBean action) throws CommandException {
184 WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
185 if (!handleUserRetry(action)) {
186 incrActionErrorCounter(action.getType(), "failed", 1);
187 LOG.warn("Failing Job due to failed action [{0}]", action.getName());
188 try {
189 workflow.getWorkflowInstance().fail(action.getName());
190 WorkflowInstance wfInstance = workflow.getWorkflowInstance();
191 ((LiteWorkflowInstance) wfInstance).setStatus(WorkflowInstance.Status.FAILED);
192 workflow.setWorkflowInstance(wfInstance);
193 workflow.setStatus(WorkflowJob.Status.FAILED);
194 action.setStatus(WorkflowAction.Status.FAILED);
195 action.resetPending();
196 queue(new NotificationXCommand(workflow, action));
197 queue(new KillXCommand(workflow.getId()));
198 InstrumentUtils.incrJobCounter(INSTR_FAILED_JOBS_COUNTER_NAME, 1, getInstrumentation());
199 }
200 catch (WorkflowException ex) {
201 throw new CommandException(ex);
202 }
203 }
204 }
205
206 /**
207 * Execute retry for action if this action is eligible for user-retry
208 *
209 * @param context the execution context.
210 * @return true if user-retry has to be handled for this action
211 * @throws CommandException thrown if unable to fail job
212 */
213 public boolean handleUserRetry(WorkflowActionBean action) throws CommandException {
214 String errorCode = action.getErrorCode();
215 Set<String> allowedRetryCode = LiteWorkflowStoreService.getUserRetryErrorCode();
216
217 if (allowedRetryCode.contains(errorCode) && action.getUserRetryCount() < action.getUserRetryMax()) {
218 LOG.info("Preparing retry this action [{0}], errorCode [{1}], userRetryCount [{2}], "
219 + "userRetryMax [{3}], userRetryInterval [{4}]", action.getId(), errorCode, action
220 .getUserRetryCount(), action.getUserRetryMax(), action.getUserRetryInterval());
221 int interval = action.getUserRetryInterval() * 60 * 1000;
222 action.setStatus(WorkflowAction.Status.USER_RETRY);
223 action.incrmentUserRetryCount();
224 action.setPending();
225 queue(new ActionStartXCommand(action.getId(), action.getType()), interval);
226 return true;
227 }
228 return false;
229 }
230
231 /*
232 * In case of action error increment the error count for instrumentation
233 */
234 private void incrActionErrorCounter(String type, String error, int count) {
235 getInstrumentation().incr(INSTRUMENTATION_GROUP, type + "#ex." + error, count);
236 }
237
238 /**
239 * Increment the action counter in the instrumentation log. indicating how
240 * many times the action was executed since the start Oozie server
241 */
242 protected void incrActionCounter(String type, int count) {
243 getInstrumentation().incr(INSTRUMENTATION_GROUP, type + "#" + getName(), count);
244 }
245
246 /**
247 * Adding a cron for the instrumentation time for the given Instrumentation
248 * group
249 */
250 protected void addActionCron(String type, Instrumentation.Cron cron) {
251 getInstrumentation().addCron(INSTRUMENTATION_GROUP, type + "#" + getName(), cron);
252 }
253
254 /**
255 * Workflow action executor context
256 *
257 */
258 public static class ActionExecutorContext implements ActionExecutor.Context {
259 private final WorkflowJobBean workflow;
260 private Configuration protoConf;
261 private final WorkflowActionBean action;
262 private final boolean isRetry;
263 private final boolean isUserRetry;
264 private boolean started;
265 private boolean ended;
266 private boolean executed;
267
268 /**
269 * Constructing the ActionExecutorContext, setting the private members
270 * and constructing the proto configuration
271 */
272 public ActionExecutorContext(WorkflowJobBean workflow, WorkflowActionBean action, boolean isRetry, boolean isUserRetry) {
273 this.workflow = workflow;
274 this.action = action;
275 this.isRetry = isRetry;
276 this.isUserRetry = isUserRetry;
277 try {
278 protoConf = new XConfiguration(new StringReader(workflow.getProtoActionConf()));
279 }
280 catch (IOException ex) {
281 throw new RuntimeException("It should not happen", ex);
282 }
283 }
284
285 /*
286 * (non-Javadoc)
287 * @see org.apache.oozie.action.ActionExecutor.Context#getCallbackUrl(java.lang.String)
288 */
289 public String getCallbackUrl(String externalStatusVar) {
290 return Services.get().get(CallbackService.class).createCallBackUrl(action.getId(), externalStatusVar);
291 }
292
293 /*
294 * (non-Javadoc)
295 * @see org.apache.oozie.action.ActionExecutor.Context#getProtoActionConf()
296 */
297 public Configuration getProtoActionConf() {
298 return protoConf;
299 }
300
301 /*
302 * (non-Javadoc)
303 * @see org.apache.oozie.action.ActionExecutor.Context#getWorkflow()
304 */
305 public WorkflowJob getWorkflow() {
306 return workflow;
307 }
308
309 /**
310 * Returns the workflow action of the given action context
311 *
312 * @return the workflow action of the given action context
313 */
314 public WorkflowAction getAction() {
315 return action;
316 }
317
318 /*
319 * (non-Javadoc)
320 * @see org.apache.oozie.action.ActionExecutor.Context#getELEvaluator()
321 */
322 public ELEvaluator getELEvaluator() {
323 ELEvaluator evaluator = Services.get().get(ELService.class).createEvaluator("workflow");
324 DagELFunctions.configureEvaluator(evaluator, workflow, action);
325 return evaluator;
326 }
327
328 /*
329 * (non-Javadoc)
330 * @see org.apache.oozie.action.ActionExecutor.Context#setVar(java.lang.String, java.lang.String)
331 */
332 public void setVar(String name, String value) {
333 name = action.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + name;
334 WorkflowInstance wfInstance = workflow.getWorkflowInstance();
335 wfInstance.setVar(name, value);
336 workflow.setWorkflowInstance(wfInstance);
337 }
338
339 /*
340 * (non-Javadoc)
341 * @see org.apache.oozie.action.ActionExecutor.Context#getVar(java.lang.String)
342 */
343 public String getVar(String name) {
344 name = action.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + name;
345 return workflow.getWorkflowInstance().getVar(name);
346 }
347
348 /*
349 * (non-Javadoc)
350 * @see org.apache.oozie.action.ActionExecutor.Context#setStartData(java.lang.String, java.lang.String, java.lang.String)
351 */
352 public void setStartData(String externalId, String trackerUri, String consoleUrl) {
353 action.setStartData(externalId, trackerUri, consoleUrl);
354 started = true;
355 }
356
357 /**
358 * Setting the start time of the action
359 */
360 public void setStartTime() {
361 Date now = new Date();
362 action.setStartTime(now);
363 }
364
365 /*
366 * (non-Javadoc)
367 * @see org.apache.oozie.action.ActionExecutor.Context#setExecutionData(java.lang.String, java.util.Properties)
368 */
369 public void setExecutionData(String externalStatus, Properties actionData) {
370 action.setExecutionData(externalStatus, actionData);
371 executed = true;
372 }
373
374 /*
375 * (non-Javadoc)
376 * @see org.apache.oozie.action.ActionExecutor.Context#setExecutionStats(java.lang.String)
377 */
378 public void setExecutionStats(String jsonStats) {
379 action.setExecutionStats(jsonStats);
380 executed = true;
381 }
382
383 /*
384 * (non-Javadoc)
385 * @see org.apache.oozie.action.ActionExecutor.Context#setExternalChildIDs(java.lang.String)
386 */
387 public void setExternalChildIDs(String externalChildIDs) {
388 action.setExternalChildIDs(externalChildIDs);
389 executed = true;
390 }
391
392 /*
393 * (non-Javadoc)
394 * @see org.apache.oozie.action.ActionExecutor.Context#setEndData(org.apache.oozie.client.WorkflowAction.Status, java.lang.String)
395 */
396 public void setEndData(WorkflowAction.Status status, String signalValue) {
397 action.setEndData(status, signalValue);
398 ended = true;
399 }
400
401 /*
402 * (non-Javadoc)
403 * @see org.apache.oozie.action.ActionExecutor.Context#isRetry()
404 */
405 public boolean isRetry() {
406 return isRetry;
407 }
408
409 /**
410 * Return if the executor invocation is a user retry or not.
411 *
412 * @return if the executor invocation is a user retry or not.
413 */
414 public boolean isUserRetry() {
415 return isUserRetry;
416 }
417
418 /**
419 * Returns whether setStartData has been called or not.
420 *
421 * @return true if start completion info has been set.
422 */
423 public boolean isStarted() {
424 return started;
425 }
426
427 /**
428 * Returns whether setExecutionData has been called or not.
429 *
430 * @return true if execution completion info has been set, otherwise false.
431 */
432 public boolean isExecuted() {
433 return executed;
434 }
435
436 /**
437 * Returns whether setEndData has been called or not.
438 *
439 * @return true if end completion info has been set.
440 */
441 public boolean isEnded() {
442 return ended;
443 }
444
445 public void setExternalStatus(String externalStatus) {
446 action.setExternalStatus(externalStatus);
447 }
448
449 @Override
450 public String getRecoveryId() {
451 return action.getId() + RECOVERY_ID_SEPARATOR + workflow.getRun();
452 }
453
454 /* (non-Javadoc)
455 * @see org.apache.oozie.action.ActionExecutor.Context#getActionDir()
456 */
457 public Path getActionDir() throws HadoopAccessorException, IOException, URISyntaxException {
458 String name = getWorkflow().getId() + "/" + action.getName() + "--" + action.getType();
459 FileSystem fs = getAppFileSystem();
460 String actionDirPath = Services.get().getSystemId() + "/" + name;
461 Path fqActionDir = new Path(fs.getHomeDirectory(), actionDirPath);
462 return fqActionDir;
463 }
464
465 /* (non-Javadoc)
466 * @see org.apache.oozie.action.ActionExecutor.Context#getAppFileSystem()
467 */
468 public FileSystem getAppFileSystem() throws HadoopAccessorException, IOException, URISyntaxException {
469 WorkflowJob workflow = getWorkflow();
470 URI uri = new URI(getWorkflow().getAppPath());
471 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
472 Configuration fsConf = has.createJobConf(uri.getAuthority());
473 return has.createFileSystem(workflow.getUser(), uri, fsConf);
474
475 }
476
477 /* (non-Javadoc)
478 * @see org.apache.oozie.action.ActionExecutor.Context#setErrorInfo(java.lang.String, java.lang.String)
479 */
480 @Override
481 public void setErrorInfo(String str, String exMsg) {
482 action.setErrorInfo(str, exMsg);
483 }
484 }
485
486 }