001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.service;
019
020 import java.util.ArrayList;
021 import java.util.Date;
022 import java.util.HashMap;
023 import java.util.HashSet;
024 import java.util.List;
025 import java.util.Set;
026 import java.util.TreeSet;
027 import java.util.Comparator;
028
029 import org.apache.hadoop.conf.Configuration;
030 import org.apache.oozie.BundleActionBean;
031 import org.apache.oozie.BundleJobBean;
032 import org.apache.oozie.CoordinatorActionBean;
033 import org.apache.oozie.CoordinatorJobBean;
034 import org.apache.oozie.ErrorCode;
035 import org.apache.oozie.client.CoordinatorAction;
036 import org.apache.oozie.client.Job;
037 import org.apache.oozie.command.CommandException;
038 import org.apache.oozie.command.bundle.BundleKillXCommand;
039 import org.apache.oozie.command.bundle.BundleStatusUpdateXCommand;
040 import org.apache.oozie.executor.jpa.BundleJobQueryExecutor;
041 import org.apache.oozie.executor.jpa.BundleActionQueryExecutor.BundleActionQuery;
042 import org.apache.oozie.executor.jpa.BundleJobQueryExecutor.BundleJobQuery;
043 import org.apache.oozie.executor.jpa.BundleActionQueryExecutor;
044 import org.apache.oozie.executor.jpa.BundleJobsGetRunningOrPendingJPAExecutor;
045 import org.apache.oozie.executor.jpa.CoordActionQueryExecutor;
046 import org.apache.oozie.executor.jpa.CoordActionQueryExecutor.CoordActionQuery;
047 import org.apache.oozie.executor.jpa.CoordJobGetActionsStatusJPAExecutor;
048 import org.apache.oozie.executor.jpa.CoordJobGetPendingActionsCountJPAExecutor;
049 import org.apache.oozie.executor.jpa.CoordJobQueryExecutor;
050 import org.apache.oozie.executor.jpa.CoordJobQueryExecutor.CoordJobQuery;
051 import org.apache.oozie.executor.jpa.CoordJobsGetPendingJPAExecutor;
052 import org.apache.oozie.executor.jpa.JPAExecutorException;
053 import org.apache.oozie.util.DateUtils;
054 import org.apache.oozie.lock.LockToken;
055 import org.apache.oozie.util.StatusUtils;
056 import org.apache.oozie.util.XLog;
057
058 /**
059 * StateTransitService is scheduled to run at the configured interval.
060 * <p/>
061 * It is to update job's status according to its child actions' status. If all child actions' pending flag equals 0 (job
062 * done), we reset the job's pending flag to 0. If all child actions are succeeded, we set the job's status to
063 * SUCCEEDED.
064 */
065 public class StatusTransitService implements Service {
066 public static final String CONF_PREFIX = Service.CONF_PREFIX + "StatusTransitService.";
067 public static final String CONF_STATUSTRANSIT_INTERVAL = CONF_PREFIX + "statusTransit.interval";
068 public static final String CONF_BACKWARD_SUPPORT_FOR_COORD_STATUS = CONF_PREFIX + "backward.support.for.coord.status";
069 public static final String CONF_BACKWARD_SUPPORT_FOR_STATES_WITHOUT_ERROR = CONF_PREFIX + "backward.support.for.states.without.error";
070 private static int limit = -1;
071 private static Date lastInstanceStartTime = null;
072 private final static XLog LOG = XLog.getLog(StatusTransitRunnable.class);
073
074 /**
075 * StateTransitRunnable is the runnable which is scheduled to run at the configured interval.
076 * <p/>
077 * It is to update job's status according to its child actions' status. If all child actions' pending flag equals 0
078 * (job done), we reset the job's pending flag to 0. If all child actions are succeeded, we set the job's status to
079 * SUCCEEDED.
080 */
081 public static class StatusTransitRunnable implements Runnable {
082 private JPAService jpaService = null;
083 private LockToken lock;
084
085 public StatusTransitRunnable() {
086 jpaService = Services.get().get(JPAService.class);
087 if (jpaService == null) {
088 LOG.error("Missing JPAService");
089 }
090 }
091
092 @Override
093 public void run() {
094 try {
095 Date curDate = new Date(); // records the start time of this service run;
096
097 // first check if there is some other instance running;
098 lock = Services.get().get(MemoryLocksService.class).getWriteLock(StatusTransitService.class.getName(),
099 lockTimeout);
100 if (lock == null) {
101 LOG.info("This StatusTransitService instance"
102 + " will not run since there is already an instance running");
103 }
104 else {
105 LOG.info("Acquired lock for [{0}]", StatusTransitService.class.getName());
106 // running coord jobs transit service
107 coordTransit();
108 // running bundle jobs transit service
109 bundleTransit();
110
111 lastInstanceStartTime = curDate;
112 }
113 }
114 catch (Exception ex) {
115 LOG.warn("Exception happened during StatusTransitRunnable ", ex);
116 }
117 finally {
118 // release lock;
119 if (lock != null) {
120 lock.release();
121 LOG.info("Released lock for [{0}]", StatusTransitService.class.getName());
122 }
123 }
124 }
125
126 public List<BundleJobBean> removeDuplicates(List<BundleJobBean> pendingJobList) {
127 Set<BundleJobBean> s = new TreeSet<BundleJobBean>(new Comparator<BundleJobBean>() {
128 @Override
129 public int compare(BundleJobBean b1, BundleJobBean b2) {
130 if (b1.getId().equals(b2.getId())) {
131 return 0;
132 }
133 else {
134 return 1;
135 }
136 }
137 });
138 s.addAll(pendingJobList);
139 return new ArrayList<BundleJobBean>(s);
140 }
141
142 /**
143 * Aggregate bundle actions' status to bundle jobs
144 *
145 * @throws JPAExecutorException thrown if failed in db updates or retrievals
146 * @throws CommandException thrown if failed to run commands
147 */
148 private void bundleTransit() throws JPAExecutorException, CommandException {
149 List<BundleJobBean> pendingJobCheckList = null;
150
151 if (lastInstanceStartTime == null) {
152 LOG.info("Running bundle status service first instance");
153 // this is the first instance, we need to check for all pending or running jobs;
154 pendingJobCheckList = jpaService.execute(new BundleJobsGetRunningOrPendingJPAExecutor(limit));
155 }
156 else {
157 LOG.info("Running bundle status service from last instance time = "
158 + DateUtils.formatDateOozieTZ(lastInstanceStartTime));
159 // this is not the first instance, we should only check jobs that have actions been
160 // updated >= start time of last service run;
161 List<BundleActionBean> actionList = BundleActionQueryExecutor.getInstance().getList(
162 BundleActionQuery.GET_BUNDLE_ACTIONS_BY_LAST_MODIFIED_TIME, lastInstanceStartTime);
163 Set<String> bundleIds = new HashSet<String>();
164 for (BundleActionBean action : actionList) {
165 bundleIds.add(action.getBundleId());
166 }
167 pendingJobCheckList = new ArrayList<BundleJobBean>();
168 for (String bundleId : bundleIds.toArray(new String[bundleIds.size()])) {
169 BundleJobBean bundle = BundleJobQueryExecutor.getInstance().get(
170 BundleJobQuery.GET_BUNDLE_JOB_ID_STATUS_PENDING_MODTIME, bundleId);
171 // Running bundle job might have pending false
172 if (bundle.isPending() || bundle.getStatus().equals(Job.Status.RUNNING)
173 || bundle.getStatus().equals(Job.Status.RUNNINGWITHERROR)
174 || bundle.getStatus().equals(Job.Status.PAUSED)
175 || bundle.getStatus().equals(Job.Status.PAUSEDWITHERROR)) {
176 pendingJobCheckList.add(bundle);
177 }
178 }
179 }
180 aggregateBundleJobsStatus(pendingJobCheckList);
181 }
182
183 private void aggregateBundleJobsStatus(List<BundleJobBean> bundleLists) throws JPAExecutorException,
184 CommandException {
185 if (bundleLists != null) {
186 for (BundleJobBean bundleJob : bundleLists) {
187 try {
188 String jobId = bundleJob.getId();
189 Job.Status[] bundleStatus = new Job.Status[1];
190 bundleStatus[0] = bundleJob.getStatus();
191 List<BundleActionBean> bundleActions = BundleActionQueryExecutor.getInstance().getList(
192 BundleActionQuery.GET_BUNDLE_UNIGNORED_ACTION_STATUS_PENDING_FOR_BUNDLE, jobId);
193 HashMap<Job.Status, Integer> bundleActionStatus = new HashMap<Job.Status, Integer>();
194 boolean foundPending = false;
195 for (BundleActionBean bAction : bundleActions) {
196 int counter = 0;
197 if (bundleActionStatus.containsKey(bAction.getStatus())) {
198 counter = bundleActionStatus.get(bAction.getStatus()) + 1;
199 }
200 else {
201 ++counter;
202 }
203 bundleActionStatus.put(bAction.getStatus(), counter);
204 if (bAction.getCoordId() == null
205 && (bAction.getStatus() == Job.Status.FAILED || bAction.getStatus() == Job.Status.KILLED)) {
206 (new BundleKillXCommand(jobId)).call();
207 LOG.info("Bundle job ["+ jobId
208 + "] has been killed since one of its coordinator job failed submission.");
209 }
210
211 if (bAction.isPending()) {
212 foundPending = true;
213 }
214 }
215
216 if (!foundPending && checkTerminalStatus(bundleActionStatus, bundleActions, bundleStatus)) {
217 LOG.info("Set bundle job [" + jobId + "] status to '" + bundleStatus[0].toString()
218 + "' from '" + bundleJob.getStatus() + "'");
219 updateBundleJob(foundPending, bundleJob, bundleStatus[0]);
220 }
221 else if (!foundPending && checkPrepStatus(bundleActionStatus, bundleActions, bundleStatus)) {
222 LOG.info("Set bundle job [" + jobId + "] status to '" + bundleStatus[0].toString()
223 + "' from '" + bundleJob.getStatus() + "'");
224 updateBundleJob(foundPending, bundleJob, bundleStatus[0]);
225 }
226 else if (checkPausedStatus(bundleActionStatus, bundleActions, bundleStatus)) {
227 LOG.info("Set bundle job [" + jobId + "] status to '" + bundleStatus[0].toString()
228 + "' from '" + bundleJob.getStatus() + "'");
229 updateBundleJob(foundPending, bundleJob, bundleStatus[0]);
230 }
231 else if (checkSuspendStatus(bundleActionStatus, bundleActions, bundleStatus, foundPending)) {
232 LOG.info("Set bundle job [" + jobId + "] status to '" + bundleStatus[0].toString()
233 + "' from '" + bundleJob.getStatus() + "'");
234 updateBundleJob(foundPending, bundleJob, bundleStatus[0]);
235 }
236 else if (checkRunningStatus(bundleActionStatus, bundleActions, bundleStatus)) {
237 LOG.info("Set bundle job [" + jobId + "] status to '" + bundleStatus[0].toString()
238 + "' from '" + bundleJob.getStatus() + "'");
239 updateBundleJob(foundPending, bundleJob, bundleStatus[0]);
240 }
241 }
242 catch (Exception ex) {
243 LOG.error("Exception happened during aggregate bundle job's status, job = "
244 + bundleJob.getId(), ex);
245 }
246 }
247 }
248
249 }
250
251 private void aggregateCoordJobsStatus(List<CoordinatorJobBean> CoordList) throws JPAExecutorException,
252 CommandException {
253 if (CoordList != null) {
254 Configuration conf = Services.get().getConf();
255 boolean backwardSupportForCoordStatus = conf.getBoolean(CONF_BACKWARD_SUPPORT_FOR_COORD_STATUS, false);
256
257 for (CoordinatorJobBean coordJob : CoordList) {
258 try {
259 // if namespace 0.1 is used and backward support is true, then ignore this coord job
260 if (backwardSupportForCoordStatus == true && coordJob.getAppNamespace() != null
261 && coordJob.getAppNamespace().equals(SchemaService.COORDINATOR_NAMESPACE_URI_1)) {
262 continue;
263 }
264 String jobId = coordJob.getId();
265 Job.Status[] coordStatus = new Job.Status[1];
266 coordStatus[0] = coordJob.getStatus();
267 //Get count of Coordinator actions with pending true
268 boolean isPending = false;
269 int count = jpaService.execute(new CoordJobGetPendingActionsCountJPAExecutor(jobId));
270 if (count > 0) {
271 isPending = true;
272 }
273 // Get status of Coordinator actions
274 List<CoordinatorAction.Status> coordActionStatusList = jpaService
275 .execute(new CoordJobGetActionsStatusJPAExecutor(jobId));
276 HashMap<CoordinatorAction.Status, Integer> coordActionStatus = new HashMap<CoordinatorAction.Status, Integer>();
277
278 for (CoordinatorAction.Status status : coordActionStatusList) {
279 int counter = 0;
280 if (coordActionStatus.containsKey(status)) {
281 counter = coordActionStatus.get(status) + 1;
282 }
283 else {
284 ++counter;
285 }
286 coordActionStatus.put(status, counter);
287 }
288
289 int nonPendingCoordActionsCount = coordActionStatusList.size();
290 boolean isDoneMaterialization = coordJob.isDoneMaterialization();
291 if ((isDoneMaterialization || coordStatus[0] == Job.Status.FAILED || coordStatus[0] == Job.Status.KILLED)
292 && checkCoordTerminalStatus(coordActionStatus, nonPendingCoordActionsCount,
293 coordStatus, isDoneMaterialization)) {
294 updateCoordJob(isPending, coordJob, coordStatus[0]);
295 }
296 else if (checkCoordPausedStatus(coordActionStatus, nonPendingCoordActionsCount, coordStatus)) {
297 updateCoordJob(isPending, coordJob, coordStatus[0]);
298 }
299 else if(checkCoordSuspendStatus( coordActionStatus, nonPendingCoordActionsCount, coordStatus, coordJob.isDoneMaterialization(), isPending)) {
300 updateCoordJob(isPending, coordJob, coordStatus[0]);
301 }
302 else if (checkCoordRunningStatus(coordActionStatus, nonPendingCoordActionsCount, coordStatus)) {
303 updateCoordJob(isPending, coordJob, coordStatus[0]);
304 }
305 else {
306 checkCoordPending(isPending, coordJob, true);
307 }
308 }
309 catch (Exception ex) {
310 LOG.error("Exception happened during aggregate coordinator job's status, job = "
311 + coordJob.getId(), ex);
312 }
313 }
314
315 }
316 }
317
318 private boolean checkTerminalStatus(HashMap<Job.Status, Integer> bundleActionStatus,
319 List<BundleActionBean> bundleActions, Job.Status[] bundleStatus) {
320 boolean ret = false;
321 int totalValuesSucceed = 0;
322 if (bundleActionStatus.containsKey(Job.Status.SUCCEEDED)) {
323 totalValuesSucceed = bundleActionStatus.get(Job.Status.SUCCEEDED);
324 }
325 int totalValuesFailed = 0;
326 if (bundleActionStatus.containsKey(Job.Status.FAILED)) {
327 totalValuesFailed = bundleActionStatus.get(Job.Status.FAILED);
328 }
329 int totalValuesKilled = 0;
330 if (bundleActionStatus.containsKey(Job.Status.KILLED)) {
331 totalValuesKilled = bundleActionStatus.get(Job.Status.KILLED);
332 }
333
334 int totalValuesDoneWithError = 0;
335 if (bundleActionStatus.containsKey(Job.Status.DONEWITHERROR)) {
336 totalValuesDoneWithError = bundleActionStatus.get(Job.Status.DONEWITHERROR);
337 }
338
339 if (bundleActions.size() == (totalValuesSucceed + totalValuesFailed + totalValuesKilled + totalValuesDoneWithError)) {
340 // If all bundle action is done and bundle is killed, then don't change the status.
341 if (bundleStatus[0].equals(Job.Status.KILLED)) {
342 bundleStatus[0] = Job.Status.KILLED;
343 return true;
344 }
345 // If all the bundle actions are succeeded then bundle job should be succeeded.
346 if (bundleActions.size() == totalValuesSucceed) {
347 bundleStatus[0] = Job.Status.SUCCEEDED;
348 ret = true;
349 }
350 else if (bundleActions.size() == totalValuesKilled) {
351 // If all the bundle actions are KILLED then bundle job should be KILLED.
352 bundleStatus[0] = Job.Status.KILLED;
353 ret = true;
354 }
355 else if (bundleActions.size() == totalValuesFailed) {
356 // If all the bundle actions are FAILED then bundle job should be FAILED.
357 bundleStatus[0] = Job.Status.FAILED;
358 ret = true;
359 }
360 else if (bundleActions.size() == totalValuesKilled + totalValuesFailed) {
361 // If all the bundle actions are KILLED then bundle job should be KILLED.
362 bundleStatus[0] = Job.Status.KILLED;
363 ret = true;
364 }
365 else {
366 bundleStatus[0] = Job.Status.DONEWITHERROR;
367 ret = true;
368 }
369 }
370 return ret;
371 }
372
373 private boolean checkCoordTerminalStatus(HashMap<CoordinatorAction.Status, Integer> coordActionStatus,
374 int coordActionsCount, Job.Status[] coordStatus, boolean isDoneMaterialization) {
375 boolean ret = false;
376 int totalValuesSucceed = 0;
377 if (coordActionStatus.containsKey(CoordinatorAction.Status.SUCCEEDED)) {
378 totalValuesSucceed = coordActionStatus.get(CoordinatorAction.Status.SUCCEEDED);
379 }
380 int totalValuesFailed = 0;
381 if (coordActionStatus.containsKey(CoordinatorAction.Status.FAILED)) {
382 totalValuesFailed = coordActionStatus.get(CoordinatorAction.Status.FAILED);
383 }
384 int totalValuesKilled = 0;
385 if (coordActionStatus.containsKey(CoordinatorAction.Status.KILLED)) {
386 totalValuesKilled = coordActionStatus.get(CoordinatorAction.Status.KILLED);
387 }
388 int totalValuesTimeOut = 0;
389 if (coordActionStatus.containsKey(CoordinatorAction.Status.TIMEDOUT)) {
390 totalValuesTimeOut = coordActionStatus.get(CoordinatorAction.Status.TIMEDOUT);
391 }
392 int totalValuesSkipped = 0;
393 if (coordActionStatus.containsKey(CoordinatorAction.Status.SKIPPED)) {
394 totalValuesSkipped = coordActionStatus.get(CoordinatorAction.Status.SKIPPED);
395 }
396
397 if (coordActionsCount ==
398 (totalValuesSucceed + totalValuesFailed + totalValuesKilled + totalValuesTimeOut + totalValuesSkipped)) {
399
400 // If all coord action is done and coord is killed, then don't change the status.
401 if (coordStatus[0].equals(Job.Status.KILLED)) {
402 coordStatus[0] = Job.Status.KILLED;
403 return true;
404 }
405 // If all the coordinator actions are succeeded then coordinator job should be succeeded.
406 if (coordActionsCount == (totalValuesSucceed + totalValuesSkipped) && isDoneMaterialization) {
407 coordStatus[0] = Job.Status.SUCCEEDED;
408 ret = true;
409 }
410 else if (coordActionsCount == totalValuesKilled) {
411 // If all the coordinator actions are KILLED then coordinator job should be KILLED.
412 coordStatus[0] = Job.Status.KILLED;
413 ret = true;
414 }
415 else if (coordActionsCount == totalValuesFailed) {
416 // If all the coordinator actions are FAILED then coordinator job should be FAILED.
417 coordStatus[0] = Job.Status.FAILED;
418 ret = true;
419 }
420 else {
421 coordStatus[0] = Job.Status.DONEWITHERROR;
422 ret = true;
423 }
424 }
425 return ret;
426 }
427
428 private boolean checkPrepStatus(HashMap<Job.Status, Integer> bundleActionStatus,
429 List<BundleActionBean> bundleActions, Job.Status[] bundleStatus) {
430 boolean ret = false;
431 if (bundleActionStatus.containsKey(Job.Status.PREP)) {
432 // If all the bundle actions are PREP then bundle job should be RUNNING.
433 if (bundleActions.size() > bundleActionStatus.get(Job.Status.PREP)) {
434 bundleStatus[0] = getRunningStatus(bundleActionStatus);
435 ret = true;
436 }
437 }
438 return ret;
439 }
440
441 private boolean checkPausedStatus(HashMap<Job.Status, Integer> bundleActionStatus,
442 List<BundleActionBean> bundleActions, Job.Status[] bundleJobStatus) {
443 boolean ret = false;
444
445 // TODO - When bottom up cmds are allowed to change the status of parent job,
446 // if none of the bundle actions are in paused or pausedwitherror, the function should return
447 // false
448
449 // top down
450 // If the bundle job is PAUSED or PAUSEDINERROR and no children are in error
451 // state, then job should be PAUSED otherwise it should be pausedwitherror
452 if (bundleJobStatus[0] == Job.Status.PAUSED || bundleJobStatus[0] == Job.Status.PAUSEDWITHERROR) {
453 if (bundleActionStatus.containsKey(Job.Status.KILLED)
454 || bundleActionStatus.containsKey(Job.Status.FAILED)
455 || bundleActionStatus.containsKey(Job.Status.DONEWITHERROR)
456 || bundleActionStatus.containsKey(Job.Status.SUSPENDEDWITHERROR)
457 || bundleActionStatus.containsKey(Job.Status.RUNNINGWITHERROR)
458 || bundleActionStatus.containsKey(Job.Status.PAUSEDWITHERROR)) {
459 bundleJobStatus[0] = Job.Status.PAUSEDWITHERROR;
460 }
461 else {
462 bundleJobStatus[0] = Job.Status.PAUSED;
463 }
464 ret = true;
465 }
466
467 // bottom up; check the status of parent through their children
468 else if (bundleActionStatus.containsKey(Job.Status.PAUSED)
469 && (bundleActions.size() == bundleActionStatus.get(Job.Status.PAUSED))) {
470 bundleJobStatus[0] = Job.Status.PAUSED;
471 ret = true;
472 }
473 else if (bundleActionStatus.containsKey(Job.Status.PAUSEDWITHERROR)) {
474 int pausedActions = bundleActionStatus.containsKey(Job.Status.PAUSED) ? bundleActionStatus
475 .get(Job.Status.PAUSED) : 0;
476 if (bundleActions.size() == pausedActions + bundleActionStatus.get(Job.Status.PAUSEDWITHERROR)) {
477 bundleJobStatus[0] = Job.Status.PAUSEDWITHERROR;
478 ret = true;
479 }
480 }
481 else {
482 ret = false;
483 }
484 return ret;
485 }
486
487
488 private boolean checkSuspendStatus(HashMap<Job.Status, Integer> bundleActionStatus,
489 List<BundleActionBean> bundleActions, Job.Status[] bundleStatus, boolean isPending) {
490 boolean ret = false;
491
492 // TODO - When bottom up cmds are allowed to change the status of parent job,
493 // if none of the bundle actions are in suspended or suspendedwitherror, the function should return
494 // false
495
496 // top down
497 // if job is suspended
498 if (bundleStatus[0] == Job.Status.SUSPENDED
499 || bundleStatus[0] == Job.Status.SUSPENDEDWITHERROR) {
500 if (bundleActionStatus.containsKey(Job.Status.KILLED)
501 || bundleActionStatus.containsKey(Job.Status.FAILED)
502 || bundleActionStatus.containsKey(Job.Status.DONEWITHERROR)
503 || bundleActionStatus.containsKey(Job.Status.SUSPENDEDWITHERROR)
504 || bundleActionStatus.containsKey(Job.Status.PAUSEDWITHERROR)) {
505 bundleStatus[0] = Job.Status.SUSPENDEDWITHERROR;
506 }
507 else {
508 bundleStatus[0] = Job.Status.SUSPENDED;
509 }
510 ret =true;
511 }
512
513 // bottom up
514 // Update status of parent from the status of its children
515 else if (!isPending && bundleActionStatus.containsKey(Job.Status.SUSPENDED)
516 || bundleActionStatus.containsKey(Job.Status.SUSPENDEDWITHERROR)) {
517 int succeededActions = bundleActionStatus.containsKey(Job.Status.SUCCEEDED) ? bundleActionStatus
518 .get(Job.Status.SUCCEEDED) : 0;
519 int killedActions = bundleActionStatus.containsKey(Job.Status.KILLED) ? bundleActionStatus
520 .get(Job.Status.KILLED) : 0;
521 int failedActions = bundleActionStatus.containsKey(Job.Status.FAILED) ? bundleActionStatus
522 .get(Job.Status.FAILED) : 0;
523 int doneWithErrorActions = bundleActionStatus.containsKey(Job.Status.DONEWITHERROR) ? bundleActionStatus
524 .get(Job.Status.DONEWITHERROR) : 0;
525
526 if (bundleActions.size() == bundleActionStatus.get(Job.Status.SUSPENDED) + succeededActions) {
527 bundleStatus[0] = Job.Status.SUSPENDED;
528 ret = true;
529 }
530 else if (bundleActions.size() == bundleActionStatus.get(Job.Status.SUSPENDEDWITHERROR)
531 + bundleActionStatus.get(Job.Status.SUSPENDED) + succeededActions + killedActions + failedActions + doneWithErrorActions) {
532 bundleStatus[0] = Job.Status.SUSPENDEDWITHERROR;
533 ret = true;
534 }
535 }
536 return ret;
537
538 }
539
540 private boolean checkCoordPausedStatus(HashMap<CoordinatorAction.Status, Integer> coordActionStatus,
541 int coordActionsCount, Job.Status[] coordStatus){
542 boolean ret = false;
543 if (coordStatus[0].equals(Job.Status.PAUSED) || coordStatus[0].equals(Job.Status.PAUSEDWITHERROR)) {
544 if (coordActionStatus.containsKey(CoordinatorAction.Status.KILLED)
545 || coordActionStatus.containsKey(CoordinatorAction.Status.FAILED)
546 || coordActionStatus.containsKey(CoordinatorAction.Status.TIMEDOUT)) {
547 coordStatus[0] = Job.Status.PAUSEDWITHERROR;
548 }
549 else {
550 coordStatus[0] = Job.Status.PAUSED;
551 }
552 ret = true;
553 }
554 return ret;
555 }
556 private boolean checkCoordSuspendStatus(HashMap<CoordinatorAction.Status, Integer> coordActionStatus,
557 int coordActionsCount, Job.Status[] coordStatus, boolean isDoneMaterialization, boolean isPending) {
558 boolean ret = false;
559
560 // TODO - When bottom up cmds are allowed to change the status of parent job
561 //if none of the coord actions are in suspended or suspendedwitherror and materialization done is false
562 //,then the function should return
563 // false
564
565 // top down
566 // check for children only when parent is suspended
567 if (coordStatus[0] == Job.Status.SUSPENDED || coordStatus[0] == Job.Status.SUSPENDEDWITHERROR) {
568
569 if (coordActionStatus.containsKey(CoordinatorAction.Status.KILLED)
570 || coordActionStatus.containsKey(CoordinatorAction.Status.FAILED)
571 || coordActionStatus.containsKey(CoordinatorAction.Status.TIMEDOUT)) {
572 coordStatus[0] = Job.Status.SUSPENDEDWITHERROR;
573 }
574 else {
575 coordStatus[0] = Job.Status.SUSPENDED;
576 }
577 ret = true;
578 }
579 // bottom up
580 // look for children to check the parent's status only if materialization is
581 // done and all actions are non-pending
582 else if (isDoneMaterialization && !isPending && coordActionStatus.containsKey(CoordinatorAction.Status.SUSPENDED)) {
583 int succeededActions = coordActionStatus.containsKey(CoordinatorAction.Status.SUCCEEDED) ? coordActionStatus
584 .get(CoordinatorAction.Status.SUCCEEDED) : 0;
585 int killedActions = coordActionStatus.containsKey(CoordinatorAction.Status.KILLED) ? coordActionStatus
586 .get(CoordinatorAction.Status.KILLED) : 0;
587 int failedActions = coordActionStatus.containsKey(CoordinatorAction.Status.FAILED) ? coordActionStatus
588 .get(CoordinatorAction.Status.FAILED) : 0;
589 int timedoutActions = coordActionStatus.containsKey(CoordinatorAction.Status.TIMEDOUT) ? coordActionStatus
590 .get(CoordinatorAction.Status.TIMEDOUT) : 0;
591
592 if (coordActionsCount == coordActionStatus.get(CoordinatorAction.Status.SUSPENDED) + succeededActions) {
593 coordStatus[0] = Job.Status.SUSPENDED;
594 ret = true;
595 }
596 else if (coordActionsCount == coordActionStatus.get(CoordinatorAction.Status.SUSPENDED)
597 + succeededActions + killedActions + failedActions + timedoutActions) {
598 coordStatus[0] = Job.Status.SUSPENDEDWITHERROR;
599 ret = true;
600 }
601 }
602 return ret;
603 }
604
605 private boolean checkCoordRunningStatus(HashMap<CoordinatorAction.Status, Integer> coordActionStatus,
606 int coordActionsCount, Job.Status[] coordStatus) {
607 boolean ret = false;
608 if (coordStatus[0] != Job.Status.PREP
609 && coordStatus[0] != Job.Status.PREPSUSPENDED
610 && coordStatus[0] != Job.Status.PREPPAUSED) {
611 if (coordActionStatus.containsKey(CoordinatorAction.Status.KILLED)
612 || coordActionStatus.containsKey(CoordinatorAction.Status.FAILED)
613 || coordActionStatus.containsKey(CoordinatorAction.Status.TIMEDOUT)) {
614 coordStatus[0] = Job.Status.RUNNINGWITHERROR;
615 }
616 else {
617 coordStatus[0] = Job.Status.RUNNING;
618 }
619 ret = true;
620 }
621 return ret;
622 }
623
624 private boolean checkRunningStatus(HashMap<Job.Status, Integer> bundleActionStatus,
625 List<BundleActionBean> bundleActions, Job.Status[] bundleStatus) {
626 boolean ret = false;
627 if (bundleStatus[0] != Job.Status.PREP) {
628 bundleStatus[0] = getRunningStatus(bundleActionStatus);
629 ret = true;
630 }
631 return ret;
632
633 }
634
635 private Job.Status getRunningStatus(HashMap<Job.Status, Integer> bundleActionStatus) {
636 if (bundleActionStatus.containsKey(Job.Status.FAILED)
637 || bundleActionStatus.containsKey(Job.Status.KILLED)
638 || bundleActionStatus.containsKey(Job.Status.DONEWITHERROR)
639 || bundleActionStatus.containsKey(Job.Status.RUNNINGWITHERROR)) {
640 return Job.Status.RUNNINGWITHERROR;
641 }
642 else {
643 return Job.Status.RUNNING;
644 }
645 }
646
647 private void updateBundleJob(boolean isPending, BundleJobBean bundleJob, Job.Status bundleStatus)
648 throws JPAExecutorException {
649 String jobId = bundleJob.getId();
650 // Update the Bundle Job
651 // Check for backward support when RUNNINGWITHERROR, SUSPENDEDWITHERROR and
652 // PAUSEDWITHERROR is not supported
653 bundleJob.setStatus(StatusUtils.getStatusIfBackwardSupportTrue(bundleStatus));
654 if (isPending) {
655 bundleJob.setPending();
656 LOG.info("Bundle job [" + jobId + "] Pending set to TRUE");
657 }
658 else {
659 bundleJob.resetPending();
660 LOG.info("Bundle job [" + jobId + "] Pending set to FALSE");
661 }
662 BundleJobQueryExecutor.getInstance().executeUpdate(BundleJobQuery.UPDATE_BUNDLE_JOB_STATUS_PENDING_MODTIME,
663 bundleJob);
664 }
665
666 private void updateCoordJob(boolean isPending, CoordinatorJobBean coordJob, Job.Status coordStatus)
667 throws JPAExecutorException, CommandException {
668 Job.Status prevStatus = coordJob.getStatus();
669 // Update the Coord Job
670 if (coordJob.getStatus() == Job.Status.SUCCEEDED || coordJob.getStatus() == Job.Status.FAILED
671 || coordJob.getStatus() == Job.Status.KILLED || coordJob.getStatus() == Job.Status.DONEWITHERROR) {
672 if (coordStatus == Job.Status.SUSPENDED || coordStatus == Job.Status.SUSPENDEDWITHERROR) {
673 LOG.info("Coord Job [" + coordJob.getId()
674 + "] status to "+ coordStatus +" can not be updated as its already in Terminal state");
675 return;
676 }
677 }
678
679 boolean isPendingStateChanged = checkCoordPending(isPending, coordJob, false);
680 // Check for backward support when RUNNINGWITHERROR, SUSPENDEDWITHERROR and PAUSEDWITHERROR is
681 // not supported
682 coordJob.setStatus(StatusUtils.getStatusIfBackwardSupportTrue(coordStatus));
683 // Backward support when coordinator namespace is 0.1
684 coordJob.setStatus(StatusUtils.getStatus(coordJob));
685 if (coordJob.getStatus() != prevStatus || isPendingStateChanged) {
686 LOG.info("Set coordinator job [" + coordJob.getId() + "] status to '" + coordJob.getStatus() + "' from '"
687 + prevStatus + "'");
688 coordJob.setLastModifiedTime(new Date());
689 CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB_STATUS_PENDING_MODTIME, coordJob);
690 }
691 // update bundle action only when status changes in coord job
692 if (coordJob.getBundleId() != null) {
693 if (!prevStatus.equals(coordJob.getStatus())) {
694 BundleStatusUpdateXCommand bundleStatusUpdate = new BundleStatusUpdateXCommand(coordJob, prevStatus);
695 bundleStatusUpdate.call();
696 }
697 }
698 }
699
700 private boolean checkCoordPending(boolean isPending, CoordinatorJobBean coordJob, boolean saveToDB)
701 throws JPAExecutorException {
702 // Checking the coordinator pending should be updated or not
703 boolean prevPending = coordJob.isPending();
704 if (isPending) {
705 coordJob.setPending();
706 }
707 else {
708 coordJob.resetPending();
709 }
710 boolean hasChange = prevPending != coordJob.isPending();
711 if (saveToDB && hasChange) {
712 LOG.info("Change coordinator job [" + coordJob.getId() + "] pending to '" + coordJob.isPending()
713 + "' from '" + prevPending + "'");
714 CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB_STATUS_PENDING_MODTIME, coordJob);
715 }
716 return hasChange;
717
718 }
719
720 /**
721 * Aggregate coordinator actions' status to coordinator jobs
722 *
723 * @throws JPAExecutorException thrown if failed in db updates or retrievals
724 * @throws CommandException thrown if failed to run commands
725 */
726 private void coordTransit() throws JPAExecutorException, CommandException {
727 List<CoordinatorJobBean> pendingJobCheckList = null;
728 if (lastInstanceStartTime == null) {
729 LOG.info("Running coordinator status service first instance");
730 // this is the first instance, we need to check for all pending jobs;
731 pendingJobCheckList = jpaService.execute(new CoordJobsGetPendingJPAExecutor(limit));
732 }
733 else {
734 LOG.info("Running coordinator status service from last instance time = "
735 + DateUtils.formatDateOozieTZ(lastInstanceStartTime));
736 // this is not the first instance, we should only check jobs
737 // that have actions or jobs been
738 // updated >= start time of last service run;
739 List<CoordinatorActionBean> actionsList = CoordActionQueryExecutor.getInstance().getList(
740 CoordActionQuery.GET_COORD_ACTIONS_BY_LAST_MODIFIED_TIME, lastInstanceStartTime);
741 Set<String> coordIds = new HashSet<String>();
742 for (CoordinatorActionBean action : actionsList) {
743 coordIds.add(action.getJobId());
744 }
745
746 pendingJobCheckList = new ArrayList<CoordinatorJobBean>();
747 for (String coordId : coordIds.toArray(new String[coordIds.size()])) {
748 CoordinatorJobBean coordJob;
749 try {
750 coordJob = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, coordId);
751 }
752 catch (JPAExecutorException jpaee) {
753 if (jpaee.getErrorCode().equals(ErrorCode.E0604)) {
754 LOG.warn("Exception happened during StatusTransitRunnable; Coordinator Job doesn't exist", jpaee);
755 continue;
756 } else {
757 throw jpaee;
758 }
759 }
760 // Running coord job might have pending false
761 Job.Status coordJobStatus = coordJob.getStatus();
762 if ((coordJob.isPending() || coordJobStatus.equals(Job.Status.PAUSED)
763 || coordJobStatus.equals(Job.Status.RUNNING)
764 || coordJobStatus.equals(Job.Status.RUNNINGWITHERROR)
765 || coordJobStatus.equals(Job.Status.PAUSEDWITHERROR))
766 && !coordJobStatus.equals(Job.Status.IGNORED)) {
767 pendingJobCheckList.add(coordJob);
768 }
769 }
770 pendingJobCheckList.addAll(CoordJobQueryExecutor.getInstance().getList(
771 CoordJobQuery.GET_COORD_JOBS_CHANGED, lastInstanceStartTime));
772 }
773 aggregateCoordJobsStatus(pendingJobCheckList);
774 }
775 }
776
777 /**
778 * Initializes the {@link StatusTransitService}.
779 *
780 * @param services services instance.
781 */
782 @Override
783 public void init(Services services) {
784 Configuration conf = services.getConf();
785 Runnable stateTransitRunnable = new StatusTransitRunnable();
786 services.get(SchedulerService.class).schedule(stateTransitRunnable, 10,
787 conf.getInt(CONF_STATUSTRANSIT_INTERVAL, 60), SchedulerService.Unit.SEC);
788 }
789
790 /**
791 * Destroy the StateTransit Jobs Service.
792 */
793 @Override
794 public void destroy() {
795 }
796
797 /**
798 * Return the public interface for the purge jobs service.
799 *
800 * @return {@link StatusTransitService}.
801 */
802 @Override
803 public Class<? extends Service> getInterface() {
804 return StatusTransitService.class;
805 }
806 }
807