001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import java.util.ArrayList;
021    import java.util.Date;
022    import java.util.HashMap;
023    import java.util.HashSet;
024    import java.util.List;
025    import java.util.Set;
026    import java.util.TreeSet;
027    import java.util.Comparator;
028    
029    import org.apache.hadoop.conf.Configuration;
030    import org.apache.oozie.BundleActionBean;
031    import org.apache.oozie.BundleJobBean;
032    import org.apache.oozie.CoordinatorActionBean;
033    import org.apache.oozie.CoordinatorJobBean;
034    import org.apache.oozie.ErrorCode;
035    import org.apache.oozie.client.CoordinatorAction;
036    import org.apache.oozie.client.Job;
037    import org.apache.oozie.command.CommandException;
038    import org.apache.oozie.command.bundle.BundleKillXCommand;
039    import org.apache.oozie.command.bundle.BundleStatusUpdateXCommand;
040    import org.apache.oozie.executor.jpa.BundleJobQueryExecutor;
041    import org.apache.oozie.executor.jpa.BundleActionQueryExecutor.BundleActionQuery;
042    import org.apache.oozie.executor.jpa.BundleJobQueryExecutor.BundleJobQuery;
043    import org.apache.oozie.executor.jpa.BundleActionQueryExecutor;
044    import org.apache.oozie.executor.jpa.BundleJobsGetRunningOrPendingJPAExecutor;
045    import org.apache.oozie.executor.jpa.CoordActionQueryExecutor;
046    import org.apache.oozie.executor.jpa.CoordActionQueryExecutor.CoordActionQuery;
047    import org.apache.oozie.executor.jpa.CoordJobGetActionsStatusJPAExecutor;
048    import org.apache.oozie.executor.jpa.CoordJobGetPendingActionsCountJPAExecutor;
049    import org.apache.oozie.executor.jpa.CoordJobQueryExecutor;
050    import org.apache.oozie.executor.jpa.CoordJobQueryExecutor.CoordJobQuery;
051    import org.apache.oozie.executor.jpa.CoordJobsGetPendingJPAExecutor;
052    import org.apache.oozie.executor.jpa.JPAExecutorException;
053    import org.apache.oozie.util.DateUtils;
054    import org.apache.oozie.lock.LockToken;
055    import org.apache.oozie.util.StatusUtils;
056    import org.apache.oozie.util.XLog;
057    
058    /**
059     * StateTransitService is scheduled to run at the configured interval.
060     * <p/>
061     * It is to update job's status according to its child actions' status. If all child actions' pending flag equals 0 (job
062     * done), we reset the job's pending flag to 0. If all child actions are succeeded, we set the job's status to
063     * SUCCEEDED.
064     */
065    public class StatusTransitService implements Service {
066        public static final String CONF_PREFIX = Service.CONF_PREFIX + "StatusTransitService.";
067        public static final String CONF_STATUSTRANSIT_INTERVAL = CONF_PREFIX + "statusTransit.interval";
068        public static final String CONF_BACKWARD_SUPPORT_FOR_COORD_STATUS = CONF_PREFIX + "backward.support.for.coord.status";
069        public static final String CONF_BACKWARD_SUPPORT_FOR_STATES_WITHOUT_ERROR = CONF_PREFIX + "backward.support.for.states.without.error";
070        private static int limit = -1;
071        private static Date lastInstanceStartTime = null;
072        private final static XLog LOG = XLog.getLog(StatusTransitRunnable.class);
073    
074        /**
075         * StateTransitRunnable is the runnable which is scheduled to run at the configured interval.
076         * <p/>
077         * It is to update job's status according to its child actions' status. If all child actions' pending flag equals 0
078         * (job done), we reset the job's pending flag to 0. If all child actions are succeeded, we set the job's status to
079         * SUCCEEDED.
080         */
081        public static class StatusTransitRunnable implements Runnable {
082            private JPAService jpaService = null;
083            private LockToken lock;
084    
085            public StatusTransitRunnable() {
086                jpaService = Services.get().get(JPAService.class);
087                if (jpaService == null) {
088                    LOG.error("Missing JPAService");
089                }
090            }
091    
092            @Override
093            public void run() {
094                try {
095                    Date curDate = new Date(); // records the start time of this service run;
096    
097                    // first check if there is some other instance running;
098                    lock = Services.get().get(MemoryLocksService.class).getWriteLock(StatusTransitService.class.getName(),
099                            lockTimeout);
100                    if (lock == null) {
101                        LOG.info("This StatusTransitService instance"
102                                + " will not run since there is already an instance running");
103                    }
104                    else {
105                        LOG.info("Acquired lock for [{0}]", StatusTransitService.class.getName());
106                        // running coord jobs transit service
107                        coordTransit();
108                        // running bundle jobs transit service
109                        bundleTransit();
110    
111                        lastInstanceStartTime = curDate;
112                    }
113                }
114                catch (Exception ex) {
115                    LOG.warn("Exception happened during StatusTransitRunnable ", ex);
116                }
117                finally {
118                    // release lock;
119                    if (lock != null) {
120                        lock.release();
121                        LOG.info("Released lock for [{0}]", StatusTransitService.class.getName());
122                    }
123                }
124            }
125    
126            public List<BundleJobBean> removeDuplicates(List<BundleJobBean> pendingJobList) {
127                Set<BundleJobBean> s = new TreeSet<BundleJobBean>(new Comparator<BundleJobBean>() {
128                    @Override
129                    public int compare(BundleJobBean b1, BundleJobBean b2) {
130                        if (b1.getId().equals(b2.getId())) {
131                            return 0;
132                        }
133                        else {
134                            return 1;
135                        }
136                    }
137                });
138                s.addAll(pendingJobList);
139                return new ArrayList<BundleJobBean>(s);
140            }
141    
142            /**
143             * Aggregate bundle actions' status to bundle jobs
144             *
145             * @throws JPAExecutorException thrown if failed in db updates or retrievals
146             * @throws CommandException thrown if failed to run commands
147             */
148            private void bundleTransit() throws JPAExecutorException, CommandException {
149                List<BundleJobBean> pendingJobCheckList = null;
150    
151                if (lastInstanceStartTime == null) {
152                    LOG.info("Running bundle status service first instance");
153                    // this is the first instance, we need to check for all pending or running jobs;
154                    pendingJobCheckList = jpaService.execute(new BundleJobsGetRunningOrPendingJPAExecutor(limit));
155                }
156                else {
157                    LOG.info("Running bundle status service from last instance time =  "
158                            + DateUtils.formatDateOozieTZ(lastInstanceStartTime));
159                    // this is not the first instance, we should only check jobs that have actions been
160                    // updated >= start time of last service run;
161                    List<BundleActionBean> actionList = BundleActionQueryExecutor.getInstance().getList(
162                            BundleActionQuery.GET_BUNDLE_ACTIONS_BY_LAST_MODIFIED_TIME, lastInstanceStartTime);
163                    Set<String> bundleIds = new HashSet<String>();
164                    for (BundleActionBean action : actionList) {
165                        bundleIds.add(action.getBundleId());
166                    }
167                    pendingJobCheckList = new ArrayList<BundleJobBean>();
168                    for (String bundleId : bundleIds.toArray(new String[bundleIds.size()])) {
169                        BundleJobBean bundle = BundleJobQueryExecutor.getInstance().get(
170                                BundleJobQuery.GET_BUNDLE_JOB_ID_STATUS_PENDING_MODTIME, bundleId);
171                        // Running bundle job might have pending false
172                        if (bundle.isPending() || bundle.getStatus().equals(Job.Status.RUNNING)
173                                || bundle.getStatus().equals(Job.Status.RUNNINGWITHERROR)
174                                || bundle.getStatus().equals(Job.Status.PAUSED)
175                                || bundle.getStatus().equals(Job.Status.PAUSEDWITHERROR)) {
176                            pendingJobCheckList.add(bundle);
177                        }
178                    }
179                }
180                aggregateBundleJobsStatus(pendingJobCheckList);
181            }
182    
183            private void aggregateBundleJobsStatus(List<BundleJobBean> bundleLists) throws JPAExecutorException,
184                    CommandException {
185                if (bundleLists != null) {
186                        for (BundleJobBean bundleJob : bundleLists) {
187                            try {
188                                String jobId = bundleJob.getId();
189                                Job.Status[] bundleStatus = new Job.Status[1];
190                                bundleStatus[0] = bundleJob.getStatus();
191                                List<BundleActionBean> bundleActions = BundleActionQueryExecutor.getInstance().getList(
192                                    BundleActionQuery.GET_BUNDLE_UNIGNORED_ACTION_STATUS_PENDING_FOR_BUNDLE, jobId);
193                                HashMap<Job.Status, Integer> bundleActionStatus = new HashMap<Job.Status, Integer>();
194                                boolean foundPending = false;
195                                for (BundleActionBean bAction : bundleActions) {
196                                        int counter = 0;
197                                        if (bundleActionStatus.containsKey(bAction.getStatus())) {
198                                            counter = bundleActionStatus.get(bAction.getStatus()) + 1;
199                                        }
200                                        else {
201                                            ++counter;
202                                        }
203                                        bundleActionStatus.put(bAction.getStatus(), counter);
204                                        if (bAction.getCoordId() == null
205                                                && (bAction.getStatus() == Job.Status.FAILED || bAction.getStatus() == Job.Status.KILLED)) {
206                                            (new BundleKillXCommand(jobId)).call();
207                                            LOG.info("Bundle job ["+ jobId
208                                                            + "] has been killed since one of its coordinator job failed submission.");
209                                        }
210    
211                                     if (bAction.isPending()) {
212                                        foundPending = true;
213                                    }
214                                }
215    
216                                if (!foundPending && checkTerminalStatus(bundleActionStatus, bundleActions, bundleStatus)) {
217                                    LOG.info("Set bundle job [" + jobId + "] status to '" + bundleStatus[0].toString()
218                                            + "' from '" + bundleJob.getStatus() + "'");
219                                    updateBundleJob(foundPending, bundleJob, bundleStatus[0]);
220                                }
221                                else if (!foundPending && checkPrepStatus(bundleActionStatus, bundleActions, bundleStatus)) {
222                                    LOG.info("Set bundle job [" + jobId + "] status to '" + bundleStatus[0].toString()
223                                            + "' from '" + bundleJob.getStatus() + "'");
224                                    updateBundleJob(foundPending, bundleJob, bundleStatus[0]);
225                                }
226                                else if (checkPausedStatus(bundleActionStatus, bundleActions, bundleStatus)) {
227                                    LOG.info("Set bundle job [" + jobId + "] status to '" + bundleStatus[0].toString()
228                                            + "' from '" + bundleJob.getStatus() + "'");
229                                    updateBundleJob(foundPending, bundleJob, bundleStatus[0]);
230                                }
231                                else if (checkSuspendStatus(bundleActionStatus, bundleActions, bundleStatus, foundPending)) {
232                                    LOG.info("Set bundle job [" + jobId + "] status to '" + bundleStatus[0].toString()
233                                            + "' from '" + bundleJob.getStatus() + "'");
234                                    updateBundleJob(foundPending, bundleJob, bundleStatus[0]);
235                                }
236                                else if (checkRunningStatus(bundleActionStatus, bundleActions, bundleStatus)) {
237                                    LOG.info("Set bundle job [" + jobId + "] status to '" + bundleStatus[0].toString()
238                                            + "' from '" + bundleJob.getStatus() + "'");
239                                    updateBundleJob(foundPending, bundleJob, bundleStatus[0]);
240                                }
241                            }
242                            catch (Exception ex) {
243                                LOG.error("Exception happened during aggregate bundle job's status, job = "
244                                        + bundleJob.getId(), ex);
245                            }
246                        }
247                    }
248    
249            }
250    
251            private void aggregateCoordJobsStatus(List<CoordinatorJobBean> CoordList) throws JPAExecutorException,
252                    CommandException {
253                if (CoordList != null) {
254                    Configuration conf = Services.get().getConf();
255                    boolean backwardSupportForCoordStatus = conf.getBoolean(CONF_BACKWARD_SUPPORT_FOR_COORD_STATUS, false);
256    
257                    for (CoordinatorJobBean coordJob : CoordList) {
258                        try {
259                            // if namespace 0.1 is used and backward support is true, then ignore this coord job
260                            if (backwardSupportForCoordStatus == true && coordJob.getAppNamespace() != null
261                                    && coordJob.getAppNamespace().equals(SchemaService.COORDINATOR_NAMESPACE_URI_1)) {
262                                continue;
263                            }
264                            String jobId = coordJob.getId();
265                            Job.Status[] coordStatus = new Job.Status[1];
266                            coordStatus[0] = coordJob.getStatus();
267                            //Get count of Coordinator actions with pending true
268                            boolean isPending = false;
269                            int count = jpaService.execute(new CoordJobGetPendingActionsCountJPAExecutor(jobId));
270                            if (count > 0) {
271                                 isPending = true;
272                            }
273                            // Get status of Coordinator actions
274                            List<CoordinatorAction.Status> coordActionStatusList = jpaService
275                                    .execute(new CoordJobGetActionsStatusJPAExecutor(jobId));
276                            HashMap<CoordinatorAction.Status, Integer> coordActionStatus = new HashMap<CoordinatorAction.Status, Integer>();
277    
278                            for (CoordinatorAction.Status status : coordActionStatusList) {
279                                int counter = 0;
280                                if (coordActionStatus.containsKey(status)) {
281                                    counter = coordActionStatus.get(status) + 1;
282                                }
283                                else {
284                                    ++counter;
285                                }
286                                coordActionStatus.put(status, counter);
287                            }
288    
289                            int nonPendingCoordActionsCount = coordActionStatusList.size();
290                            boolean isDoneMaterialization = coordJob.isDoneMaterialization();
291                            if ((isDoneMaterialization || coordStatus[0] == Job.Status.FAILED || coordStatus[0] == Job.Status.KILLED)
292                                    && checkCoordTerminalStatus(coordActionStatus, nonPendingCoordActionsCount,
293                                            coordStatus, isDoneMaterialization)) {
294                                updateCoordJob(isPending, coordJob, coordStatus[0]);
295                            }
296                            else if (checkCoordPausedStatus(coordActionStatus, nonPendingCoordActionsCount, coordStatus)) {
297                                updateCoordJob(isPending, coordJob, coordStatus[0]);
298                            }
299                            else if(checkCoordSuspendStatus( coordActionStatus, nonPendingCoordActionsCount, coordStatus, coordJob.isDoneMaterialization(), isPending)) {
300                                updateCoordJob(isPending, coordJob, coordStatus[0]);
301                            }
302                            else if (checkCoordRunningStatus(coordActionStatus, nonPendingCoordActionsCount, coordStatus)) {
303                                updateCoordJob(isPending, coordJob, coordStatus[0]);
304                            }
305                            else {
306                                checkCoordPending(isPending, coordJob, true);
307                            }
308                        }
309                        catch (Exception ex) {
310                            LOG.error("Exception happened during aggregate coordinator job's status, job = "
311                                    + coordJob.getId(), ex);
312                        }
313                    }
314    
315                }
316            }
317    
318            private boolean checkTerminalStatus(HashMap<Job.Status, Integer> bundleActionStatus,
319                    List<BundleActionBean> bundleActions, Job.Status[] bundleStatus) {
320                boolean ret = false;
321                int totalValuesSucceed = 0;
322                if (bundleActionStatus.containsKey(Job.Status.SUCCEEDED)) {
323                    totalValuesSucceed = bundleActionStatus.get(Job.Status.SUCCEEDED);
324                }
325                int totalValuesFailed = 0;
326                if (bundleActionStatus.containsKey(Job.Status.FAILED)) {
327                    totalValuesFailed = bundleActionStatus.get(Job.Status.FAILED);
328                }
329                int totalValuesKilled = 0;
330                if (bundleActionStatus.containsKey(Job.Status.KILLED)) {
331                    totalValuesKilled = bundleActionStatus.get(Job.Status.KILLED);
332                }
333    
334                int totalValuesDoneWithError = 0;
335                if (bundleActionStatus.containsKey(Job.Status.DONEWITHERROR)) {
336                    totalValuesDoneWithError = bundleActionStatus.get(Job.Status.DONEWITHERROR);
337                }
338    
339                if (bundleActions.size() == (totalValuesSucceed + totalValuesFailed + totalValuesKilled + totalValuesDoneWithError)) {
340                    // If all bundle action is done and bundle is killed, then don't change the status.
341                    if (bundleStatus[0].equals(Job.Status.KILLED)) {
342                        bundleStatus[0] = Job.Status.KILLED;
343                        return true;
344                    }
345                    // If all the bundle actions are succeeded then bundle job should be succeeded.
346                    if (bundleActions.size() == totalValuesSucceed) {
347                        bundleStatus[0] = Job.Status.SUCCEEDED;
348                        ret = true;
349                    }
350                    else if (bundleActions.size() == totalValuesKilled) {
351                        // If all the bundle actions are KILLED then bundle job should be KILLED.
352                        bundleStatus[0] = Job.Status.KILLED;
353                        ret = true;
354                    }
355                    else if (bundleActions.size() == totalValuesFailed) {
356                        // If all the bundle actions are FAILED then bundle job should be FAILED.
357                        bundleStatus[0] = Job.Status.FAILED;
358                        ret = true;
359                    }
360                    else if (bundleActions.size() == totalValuesKilled + totalValuesFailed) {
361                        // If all the bundle actions are KILLED then bundle job should be KILLED.
362                        bundleStatus[0] = Job.Status.KILLED;
363                        ret = true;
364                    }
365                    else {
366                        bundleStatus[0] = Job.Status.DONEWITHERROR;
367                        ret = true;
368                    }
369                }
370                return ret;
371            }
372    
373            private boolean checkCoordTerminalStatus(HashMap<CoordinatorAction.Status, Integer> coordActionStatus,
374                    int coordActionsCount, Job.Status[] coordStatus, boolean isDoneMaterialization) {
375                boolean ret = false;
376                int totalValuesSucceed = 0;
377                if (coordActionStatus.containsKey(CoordinatorAction.Status.SUCCEEDED)) {
378                    totalValuesSucceed = coordActionStatus.get(CoordinatorAction.Status.SUCCEEDED);
379                }
380                int totalValuesFailed = 0;
381                if (coordActionStatus.containsKey(CoordinatorAction.Status.FAILED)) {
382                    totalValuesFailed = coordActionStatus.get(CoordinatorAction.Status.FAILED);
383                }
384                int totalValuesKilled = 0;
385                if (coordActionStatus.containsKey(CoordinatorAction.Status.KILLED)) {
386                    totalValuesKilled = coordActionStatus.get(CoordinatorAction.Status.KILLED);
387                }
388                int totalValuesTimeOut = 0;
389                if (coordActionStatus.containsKey(CoordinatorAction.Status.TIMEDOUT)) {
390                    totalValuesTimeOut = coordActionStatus.get(CoordinatorAction.Status.TIMEDOUT);
391                }
392                int totalValuesSkipped = 0;
393                if (coordActionStatus.containsKey(CoordinatorAction.Status.SKIPPED)) {
394                    totalValuesSkipped = coordActionStatus.get(CoordinatorAction.Status.SKIPPED);
395                }
396    
397                if (coordActionsCount ==
398                        (totalValuesSucceed + totalValuesFailed + totalValuesKilled + totalValuesTimeOut + totalValuesSkipped)) {
399    
400                    // If all coord action is done and coord is killed, then don't change the status.
401                    if (coordStatus[0].equals(Job.Status.KILLED)) {
402                        coordStatus[0] = Job.Status.KILLED;
403                        return true;
404                    }
405                    // If all the coordinator actions are succeeded then coordinator job should be succeeded.
406                    if (coordActionsCount == (totalValuesSucceed + totalValuesSkipped) && isDoneMaterialization) {
407                        coordStatus[0] = Job.Status.SUCCEEDED;
408                        ret = true;
409                    }
410                    else if (coordActionsCount == totalValuesKilled) {
411                        // If all the coordinator actions are KILLED then coordinator job should be KILLED.
412                        coordStatus[0] = Job.Status.KILLED;
413                        ret = true;
414                    }
415                    else if (coordActionsCount == totalValuesFailed) {
416                        // If all the coordinator actions are FAILED then coordinator job should be FAILED.
417                        coordStatus[0] = Job.Status.FAILED;
418                        ret = true;
419                    }
420                    else {
421                        coordStatus[0] = Job.Status.DONEWITHERROR;
422                        ret = true;
423                    }
424                }
425                return ret;
426            }
427    
428            private boolean checkPrepStatus(HashMap<Job.Status, Integer> bundleActionStatus,
429                    List<BundleActionBean> bundleActions, Job.Status[] bundleStatus) {
430                boolean ret = false;
431                if (bundleActionStatus.containsKey(Job.Status.PREP)) {
432                    // If all the bundle actions are PREP then bundle job should be RUNNING.
433                    if (bundleActions.size() > bundleActionStatus.get(Job.Status.PREP)) {
434                        bundleStatus[0] = getRunningStatus(bundleActionStatus);
435                        ret = true;
436                    }
437                }
438                return ret;
439            }
440    
441            private boolean checkPausedStatus(HashMap<Job.Status, Integer> bundleActionStatus,
442                    List<BundleActionBean> bundleActions, Job.Status[] bundleJobStatus) {
443                boolean ret = false;
444    
445                // TODO - When bottom up cmds are allowed to change the status of parent job,
446                // if none of the bundle actions are in paused or pausedwitherror, the function should return
447                // false
448    
449                // top down
450                // If the bundle job is PAUSED or PAUSEDINERROR and no children are in error
451                // state, then job should be PAUSED otherwise it should be pausedwitherror
452                if (bundleJobStatus[0] == Job.Status.PAUSED || bundleJobStatus[0] == Job.Status.PAUSEDWITHERROR) {
453                    if (bundleActionStatus.containsKey(Job.Status.KILLED)
454                            || bundleActionStatus.containsKey(Job.Status.FAILED)
455                            || bundleActionStatus.containsKey(Job.Status.DONEWITHERROR)
456                            || bundleActionStatus.containsKey(Job.Status.SUSPENDEDWITHERROR)
457                            || bundleActionStatus.containsKey(Job.Status.RUNNINGWITHERROR)
458                            || bundleActionStatus.containsKey(Job.Status.PAUSEDWITHERROR)) {
459                        bundleJobStatus[0] = Job.Status.PAUSEDWITHERROR;
460                    }
461                    else {
462                        bundleJobStatus[0] = Job.Status.PAUSED;
463                    }
464                    ret = true;
465                }
466    
467                // bottom up; check the status of parent through their children
468                else if (bundleActionStatus.containsKey(Job.Status.PAUSED)
469                        && (bundleActions.size() == bundleActionStatus.get(Job.Status.PAUSED))) {
470                    bundleJobStatus[0] = Job.Status.PAUSED;
471                    ret = true;
472                }
473                else if (bundleActionStatus.containsKey(Job.Status.PAUSEDWITHERROR)) {
474                    int pausedActions = bundleActionStatus.containsKey(Job.Status.PAUSED) ? bundleActionStatus
475                            .get(Job.Status.PAUSED) : 0;
476                    if (bundleActions.size() == pausedActions + bundleActionStatus.get(Job.Status.PAUSEDWITHERROR)) {
477                        bundleJobStatus[0] = Job.Status.PAUSEDWITHERROR;
478                        ret = true;
479                    }
480                }
481                else {
482                    ret = false;
483                }
484                return ret;
485            }
486    
487    
488            private boolean checkSuspendStatus(HashMap<Job.Status, Integer> bundleActionStatus,
489                    List<BundleActionBean> bundleActions, Job.Status[] bundleStatus, boolean isPending) {
490                boolean ret = false;
491    
492                // TODO - When bottom up cmds are allowed to change the status of parent job,
493                // if none of the bundle actions are in suspended or suspendedwitherror, the function should return
494                // false
495    
496                // top down
497                // if job is suspended
498                if (bundleStatus[0] == Job.Status.SUSPENDED
499                        || bundleStatus[0] == Job.Status.SUSPENDEDWITHERROR) {
500                    if (bundleActionStatus.containsKey(Job.Status.KILLED)
501                            || bundleActionStatus.containsKey(Job.Status.FAILED)
502                            || bundleActionStatus.containsKey(Job.Status.DONEWITHERROR)
503                            || bundleActionStatus.containsKey(Job.Status.SUSPENDEDWITHERROR)
504                            || bundleActionStatus.containsKey(Job.Status.PAUSEDWITHERROR)) {
505                        bundleStatus[0] = Job.Status.SUSPENDEDWITHERROR;
506                    }
507                    else {
508                        bundleStatus[0] = Job.Status.SUSPENDED;
509                    }
510                    ret =true;
511                }
512    
513                // bottom up
514                // Update status of parent from the status of its children
515                else if (!isPending && bundleActionStatus.containsKey(Job.Status.SUSPENDED)
516                        || bundleActionStatus.containsKey(Job.Status.SUSPENDEDWITHERROR)) {
517                    int succeededActions = bundleActionStatus.containsKey(Job.Status.SUCCEEDED) ? bundleActionStatus
518                            .get(Job.Status.SUCCEEDED) : 0;
519                    int killedActions = bundleActionStatus.containsKey(Job.Status.KILLED) ? bundleActionStatus
520                            .get(Job.Status.KILLED) : 0;
521                    int failedActions = bundleActionStatus.containsKey(Job.Status.FAILED) ? bundleActionStatus
522                            .get(Job.Status.FAILED) : 0;
523                    int doneWithErrorActions = bundleActionStatus.containsKey(Job.Status.DONEWITHERROR) ? bundleActionStatus
524                            .get(Job.Status.DONEWITHERROR) : 0;
525    
526                    if (bundleActions.size() == bundleActionStatus.get(Job.Status.SUSPENDED) + succeededActions) {
527                        bundleStatus[0] = Job.Status.SUSPENDED;
528                        ret = true;
529                    }
530                    else if (bundleActions.size()  == bundleActionStatus.get(Job.Status.SUSPENDEDWITHERROR)
531                            + bundleActionStatus.get(Job.Status.SUSPENDED) + succeededActions + killedActions + failedActions + doneWithErrorActions) {
532                        bundleStatus[0] = Job.Status.SUSPENDEDWITHERROR;
533                        ret = true;
534                    }
535                }
536                return ret;
537    
538            }
539    
540            private boolean checkCoordPausedStatus(HashMap<CoordinatorAction.Status, Integer> coordActionStatus,
541                    int coordActionsCount, Job.Status[] coordStatus){
542                boolean ret = false;
543                if (coordStatus[0].equals(Job.Status.PAUSED) || coordStatus[0].equals(Job.Status.PAUSEDWITHERROR)) {
544                    if (coordActionStatus.containsKey(CoordinatorAction.Status.KILLED)
545                            || coordActionStatus.containsKey(CoordinatorAction.Status.FAILED)
546                            || coordActionStatus.containsKey(CoordinatorAction.Status.TIMEDOUT)) {
547                        coordStatus[0] = Job.Status.PAUSEDWITHERROR;
548                    }
549                    else {
550                        coordStatus[0] = Job.Status.PAUSED;
551                    }
552                    ret = true;
553                }
554                return ret;
555            }
556            private boolean checkCoordSuspendStatus(HashMap<CoordinatorAction.Status, Integer> coordActionStatus,
557                    int coordActionsCount, Job.Status[] coordStatus, boolean isDoneMaterialization, boolean isPending) {
558                boolean ret = false;
559    
560                // TODO - When bottom up cmds are allowed to change the status of parent job
561                //if none of the coord actions are in suspended or suspendedwitherror and materialization done is false
562                //,then the function should return
563                // false
564    
565                // top down
566                // check for children only when parent is suspended
567                if (coordStatus[0] == Job.Status.SUSPENDED || coordStatus[0] == Job.Status.SUSPENDEDWITHERROR) {
568    
569                    if (coordActionStatus.containsKey(CoordinatorAction.Status.KILLED)
570                            || coordActionStatus.containsKey(CoordinatorAction.Status.FAILED)
571                            || coordActionStatus.containsKey(CoordinatorAction.Status.TIMEDOUT)) {
572                        coordStatus[0] = Job.Status.SUSPENDEDWITHERROR;
573                    }
574                    else {
575                        coordStatus[0] = Job.Status.SUSPENDED;
576                    }
577                    ret = true;
578                }
579                // bottom up
580                // look for children to check the parent's status only if materialization is
581                // done and all actions are non-pending
582                else if (isDoneMaterialization && !isPending && coordActionStatus.containsKey(CoordinatorAction.Status.SUSPENDED)) {
583                    int succeededActions = coordActionStatus.containsKey(CoordinatorAction.Status.SUCCEEDED) ? coordActionStatus
584                           .get(CoordinatorAction.Status.SUCCEEDED) : 0;
585                    int killedActions = coordActionStatus.containsKey(CoordinatorAction.Status.KILLED) ? coordActionStatus
586                            .get(CoordinatorAction.Status.KILLED) : 0;
587                    int failedActions = coordActionStatus.containsKey(CoordinatorAction.Status.FAILED) ? coordActionStatus
588                            .get(CoordinatorAction.Status.FAILED) : 0;
589                    int timedoutActions = coordActionStatus.containsKey(CoordinatorAction.Status.TIMEDOUT) ? coordActionStatus
590                            .get(CoordinatorAction.Status.TIMEDOUT) : 0;
591    
592                    if (coordActionsCount == coordActionStatus.get(CoordinatorAction.Status.SUSPENDED) + succeededActions) {
593                        coordStatus[0] = Job.Status.SUSPENDED;
594                        ret = true;
595                    }
596                    else if (coordActionsCount == coordActionStatus.get(CoordinatorAction.Status.SUSPENDED)
597                            + succeededActions + killedActions + failedActions + timedoutActions) {
598                        coordStatus[0] = Job.Status.SUSPENDEDWITHERROR;
599                        ret = true;
600                    }
601                }
602                return ret;
603            }
604    
605            private boolean checkCoordRunningStatus(HashMap<CoordinatorAction.Status, Integer> coordActionStatus,
606                    int coordActionsCount, Job.Status[] coordStatus) {
607                boolean ret = false;
608                if (coordStatus[0] != Job.Status.PREP
609                        && coordStatus[0] != Job.Status.PREPSUSPENDED
610                        && coordStatus[0] !=  Job.Status.PREPPAUSED) {
611                    if (coordActionStatus.containsKey(CoordinatorAction.Status.KILLED)
612                            || coordActionStatus.containsKey(CoordinatorAction.Status.FAILED)
613                            || coordActionStatus.containsKey(CoordinatorAction.Status.TIMEDOUT)) {
614                        coordStatus[0] = Job.Status.RUNNINGWITHERROR;
615                    }
616                    else {
617                        coordStatus[0] = Job.Status.RUNNING;
618                    }
619                    ret = true;
620                }
621                return ret;
622            }
623    
624            private boolean checkRunningStatus(HashMap<Job.Status, Integer> bundleActionStatus,
625                    List<BundleActionBean> bundleActions, Job.Status[] bundleStatus) {
626                boolean ret = false;
627                if (bundleStatus[0] != Job.Status.PREP) {
628                    bundleStatus[0] = getRunningStatus(bundleActionStatus);
629                    ret = true;
630                }
631                return ret;
632    
633            }
634    
635            private Job.Status getRunningStatus(HashMap<Job.Status, Integer> bundleActionStatus) {
636                if (bundleActionStatus.containsKey(Job.Status.FAILED)
637                        || bundleActionStatus.containsKey(Job.Status.KILLED)
638                        || bundleActionStatus.containsKey(Job.Status.DONEWITHERROR)
639                        || bundleActionStatus.containsKey(Job.Status.RUNNINGWITHERROR)) {
640                    return Job.Status.RUNNINGWITHERROR;
641                }
642                else {
643                    return Job.Status.RUNNING;
644                }
645            }
646    
647            private void updateBundleJob(boolean isPending, BundleJobBean bundleJob, Job.Status bundleStatus)
648                    throws JPAExecutorException {
649                String jobId = bundleJob.getId();
650                // Update the Bundle Job
651                // Check for backward support when RUNNINGWITHERROR, SUSPENDEDWITHERROR and
652                // PAUSEDWITHERROR is not supported
653                bundleJob.setStatus(StatusUtils.getStatusIfBackwardSupportTrue(bundleStatus));
654                if (isPending) {
655                    bundleJob.setPending();
656                    LOG.info("Bundle job [" + jobId + "] Pending set to TRUE");
657                }
658                else {
659                    bundleJob.resetPending();
660                    LOG.info("Bundle job [" + jobId + "] Pending set to FALSE");
661                }
662                BundleJobQueryExecutor.getInstance().executeUpdate(BundleJobQuery.UPDATE_BUNDLE_JOB_STATUS_PENDING_MODTIME,
663                        bundleJob);
664            }
665    
666            private void updateCoordJob(boolean isPending, CoordinatorJobBean coordJob, Job.Status coordStatus)
667                    throws JPAExecutorException, CommandException {
668                Job.Status prevStatus = coordJob.getStatus();
669                // Update the Coord Job
670                if (coordJob.getStatus() == Job.Status.SUCCEEDED || coordJob.getStatus() == Job.Status.FAILED
671                        || coordJob.getStatus() == Job.Status.KILLED || coordJob.getStatus() == Job.Status.DONEWITHERROR) {
672                    if (coordStatus == Job.Status.SUSPENDED || coordStatus == Job.Status.SUSPENDEDWITHERROR) {
673                        LOG.info("Coord Job [" + coordJob.getId()
674                                + "] status to "+ coordStatus +" can not be updated as its already in Terminal state");
675                        return;
676                    }
677                }
678    
679                boolean isPendingStateChanged = checkCoordPending(isPending, coordJob, false);
680                // Check for backward support when RUNNINGWITHERROR, SUSPENDEDWITHERROR and PAUSEDWITHERROR is
681                // not supported
682                coordJob.setStatus(StatusUtils.getStatusIfBackwardSupportTrue(coordStatus));
683                // Backward support when coordinator namespace is 0.1
684                coordJob.setStatus(StatusUtils.getStatus(coordJob));
685                if (coordJob.getStatus() != prevStatus || isPendingStateChanged) {
686                    LOG.info("Set coordinator job [" + coordJob.getId() + "] status to '" + coordJob.getStatus() + "' from '"
687                            + prevStatus + "'");
688                    coordJob.setLastModifiedTime(new Date());
689                    CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB_STATUS_PENDING_MODTIME, coordJob);
690                }
691                // update bundle action only when status changes in coord job
692                if (coordJob.getBundleId() != null) {
693                    if (!prevStatus.equals(coordJob.getStatus())) {
694                        BundleStatusUpdateXCommand bundleStatusUpdate = new BundleStatusUpdateXCommand(coordJob, prevStatus);
695                        bundleStatusUpdate.call();
696                    }
697                }
698            }
699    
700            private boolean checkCoordPending(boolean isPending, CoordinatorJobBean coordJob, boolean saveToDB)
701                    throws JPAExecutorException {
702                // Checking the coordinator pending should be updated or not
703                boolean prevPending = coordJob.isPending();
704                if (isPending) {
705                    coordJob.setPending();
706                }
707                else {
708                    coordJob.resetPending();
709                }
710                boolean hasChange = prevPending != coordJob.isPending();
711                if (saveToDB && hasChange) {
712                    LOG.info("Change coordinator job [" + coordJob.getId() + "] pending to '" + coordJob.isPending()
713                            + "' from '" + prevPending + "'");
714                    CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB_STATUS_PENDING_MODTIME, coordJob);
715                }
716                return hasChange;
717    
718            }
719    
720            /**
721             * Aggregate coordinator actions' status to coordinator jobs
722             *
723             * @throws JPAExecutorException thrown if failed in db updates or retrievals
724             * @throws CommandException thrown if failed to run commands
725             */
726            private void coordTransit() throws JPAExecutorException, CommandException {
727                List<CoordinatorJobBean> pendingJobCheckList = null;
728                if (lastInstanceStartTime == null) {
729                    LOG.info("Running coordinator status service first instance");
730                    // this is the first instance, we need to check for all pending jobs;
731                    pendingJobCheckList = jpaService.execute(new CoordJobsGetPendingJPAExecutor(limit));
732                }
733                else {
734                    LOG.info("Running coordinator status service from last instance time =  "
735                            + DateUtils.formatDateOozieTZ(lastInstanceStartTime));
736                    // this is not the first instance, we should only check jobs
737                    // that have actions or jobs been
738                    // updated >= start time of last service run;
739                    List<CoordinatorActionBean> actionsList = CoordActionQueryExecutor.getInstance().getList(
740                            CoordActionQuery.GET_COORD_ACTIONS_BY_LAST_MODIFIED_TIME, lastInstanceStartTime);
741                    Set<String> coordIds = new HashSet<String>();
742                    for (CoordinatorActionBean action : actionsList) {
743                        coordIds.add(action.getJobId());
744                    }
745    
746                    pendingJobCheckList = new ArrayList<CoordinatorJobBean>();
747                    for (String coordId : coordIds.toArray(new String[coordIds.size()])) {
748                        CoordinatorJobBean coordJob;
749                        try {
750                            coordJob = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, coordId);
751                        }
752                        catch (JPAExecutorException jpaee) {
753                            if (jpaee.getErrorCode().equals(ErrorCode.E0604)) {
754                                LOG.warn("Exception happened during StatusTransitRunnable; Coordinator Job doesn't exist", jpaee);
755                                continue;
756                            } else {
757                                throw jpaee;
758                            }
759                        }
760                        // Running coord job might have pending false
761                        Job.Status coordJobStatus = coordJob.getStatus();
762                        if ((coordJob.isPending() || coordJobStatus.equals(Job.Status.PAUSED)
763                                || coordJobStatus.equals(Job.Status.RUNNING)
764                                || coordJobStatus.equals(Job.Status.RUNNINGWITHERROR)
765                                || coordJobStatus.equals(Job.Status.PAUSEDWITHERROR))
766                                && !coordJobStatus.equals(Job.Status.IGNORED)) {
767                            pendingJobCheckList.add(coordJob);
768                        }
769                    }
770                    pendingJobCheckList.addAll(CoordJobQueryExecutor.getInstance().getList(
771                            CoordJobQuery.GET_COORD_JOBS_CHANGED, lastInstanceStartTime));
772                }
773                aggregateCoordJobsStatus(pendingJobCheckList);
774            }
775        }
776    
777        /**
778         * Initializes the {@link StatusTransitService}.
779         *
780         * @param services services instance.
781         */
782        @Override
783        public void init(Services services) {
784            Configuration conf = services.getConf();
785            Runnable stateTransitRunnable = new StatusTransitRunnable();
786            services.get(SchedulerService.class).schedule(stateTransitRunnable, 10,
787                    conf.getInt(CONF_STATUSTRANSIT_INTERVAL, 60), SchedulerService.Unit.SEC);
788        }
789    
790        /**
791         * Destroy the StateTransit Jobs Service.
792         */
793        @Override
794        public void destroy() {
795        }
796    
797        /**
798         * Return the public interface for the purge jobs service.
799         *
800         * @return {@link StatusTransitService}.
801         */
802        @Override
803        public Class<? extends Service> getInterface() {
804            return StatusTransitService.class;
805        }
806    }
807