001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import java.io.IOException;
021    import java.io.StringReader;
022    import java.util.ArrayList;
023    import java.util.Date;
024    import java.util.List;
025    
026    import org.apache.hadoop.conf.Configuration;
027    import org.apache.oozie.BundleActionBean;
028    import org.apache.oozie.BundleJobBean;
029    import org.apache.oozie.CoordinatorActionBean;
030    import org.apache.oozie.CoordinatorJobBean;
031    import org.apache.oozie.ErrorCode;
032    import org.apache.oozie.WorkflowActionBean;
033    import org.apache.oozie.client.Job;
034    import org.apache.oozie.client.OozieClient;
035    import org.apache.oozie.command.CommandException;
036    import org.apache.oozie.command.coord.CoordActionInputCheckXCommand;
037    import org.apache.oozie.command.coord.CoordActionReadyXCommand;
038    import org.apache.oozie.command.coord.CoordActionStartXCommand;
039    import org.apache.oozie.command.coord.CoordKillXCommand;
040    import org.apache.oozie.command.coord.CoordPushDependencyCheckXCommand;
041    import org.apache.oozie.command.coord.CoordResumeXCommand;
042    import org.apache.oozie.command.coord.CoordSubmitXCommand;
043    import org.apache.oozie.command.coord.CoordSuspendXCommand;
044    import org.apache.oozie.command.wf.ActionEndXCommand;
045    import org.apache.oozie.command.wf.ActionStartXCommand;
046    import org.apache.oozie.command.wf.KillXCommand;
047    import org.apache.oozie.command.wf.ResumeXCommand;
048    import org.apache.oozie.command.wf.SignalXCommand;
049    import org.apache.oozie.command.wf.SuspendXCommand;
050    import org.apache.oozie.executor.jpa.BundleActionQueryExecutor;
051    import org.apache.oozie.executor.jpa.BundleJobQueryExecutor;
052    import org.apache.oozie.executor.jpa.CoordActionsGetForRecoveryJPAExecutor;
053    import org.apache.oozie.executor.jpa.CoordActionsGetReadyGroupbyJobIDJPAExecutor;
054    import org.apache.oozie.executor.jpa.CoordJobQueryExecutor;
055    import org.apache.oozie.executor.jpa.BundleActionQueryExecutor.BundleActionQuery;
056    import org.apache.oozie.executor.jpa.BundleJobQueryExecutor.BundleJobQuery;
057    import org.apache.oozie.executor.jpa.CoordJobQueryExecutor.CoordJobQuery;
058    import org.apache.oozie.executor.jpa.JPAExecutorException;
059    import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor;
060    import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor.WorkflowActionQuery;
061    import org.apache.oozie.util.JobUtils;
062    import org.apache.oozie.util.XCallable;
063    import org.apache.oozie.util.XConfiguration;
064    import org.apache.oozie.util.XLog;
065    import org.apache.oozie.util.XmlUtils;
066    import org.jdom.Attribute;
067    import org.jdom.Element;
068    
069    /**
070     * The Recovery Service checks for pending actions and premater coordinator jobs older than a configured age and then
071     * queues them for execution.
072     */
073    public class RecoveryService implements Service {
074    
075        public static final String CONF_PREFIX = Service.CONF_PREFIX + "RecoveryService.";
076        public static final String CONF_PREFIX_WF_ACTIONS = Service.CONF_PREFIX + "wf.actions.";
077        public static final String CONF_PREFIX_COORD = Service.CONF_PREFIX + "coord.";
078        public static final String CONF_PREFIX_BUNDLE = Service.CONF_PREFIX + "bundle.";
079        /**
080         * Time interval, in seconds, at which the recovery service will be scheduled to run.
081         */
082        public static final String CONF_SERVICE_INTERVAL = CONF_PREFIX + "interval";
083        /**
084         * The number of callables to be queued in a batch.
085         */
086        public static final String CONF_CALLABLE_BATCH_SIZE = CONF_PREFIX + "callable.batch.size";
087    
088        /**
089         * Delay for the push missing dependencies in milliseconds.
090         */
091        public static final String CONF_PUSH_DEPENDENCY_INTERVAL = CONF_PREFIX + "push.dependency.interval";
092    
093        /**
094         * Age of actions to queue, in seconds.
095         */
096        public static final String CONF_WF_ACTIONS_OLDER_THAN = CONF_PREFIX_WF_ACTIONS + "older.than";
097        /**
098         * Age of coordinator jobs to recover, in seconds.
099         */
100        public static final String CONF_COORD_OLDER_THAN = CONF_PREFIX_COORD + "older.than";
101    
102        /**
103         * Age of Bundle jobs to recover, in seconds.
104         */
105        public static final String CONF_BUNDLE_OLDER_THAN = CONF_PREFIX_BUNDLE + "older.than";
106    
107        private static final String INSTRUMENTATION_GROUP = "recovery";
108        private static final String INSTR_RECOVERED_ACTIONS_COUNTER = "actions";
109        private static final String INSTR_RECOVERED_COORD_ACTIONS_COUNTER = "coord_actions";
110        private static final String INSTR_RECOVERED_BUNDLE_ACTIONS_COUNTER = "bundle_actions";
111    
112    
113        /**
114         * RecoveryRunnable is the Runnable which is scheduled to run with the configured interval, and takes care of the
115         * queuing of commands.
116         */
117        static class RecoveryRunnable implements Runnable {
118            private final long olderThan;
119            private final long coordOlderThan;
120            private final long bundleOlderThan;
121            private long delay = 0;
122            private List<XCallable<?>> callables;
123            private List<XCallable<?>> delayedCallables;
124            private StringBuilder msg = null;
125            private JPAService jpaService = null;
126    
127            public RecoveryRunnable(long olderThan, long coordOlderThan,long bundleOlderThan) {
128                this.olderThan = olderThan;
129                this.coordOlderThan = coordOlderThan;
130                this.bundleOlderThan = bundleOlderThan;
131            }
132    
133            public void run() {
134                XLog.Info.get().clear();
135                XLog log = XLog.getLog(getClass());
136                msg = new StringBuilder();
137                jpaService = Services.get().get(JPAService.class);
138                runWFRecovery();
139                runCoordActionRecovery();
140                runCoordActionRecoveryForReady();
141                runBundleRecovery();
142                log.debug("QUEUING [{0}] for potential recovery", msg.toString());
143                boolean ret = false;
144                if (null != callables) {
145                    ret = Services.get().get(CallableQueueService.class).queueSerial(callables);
146                    if (ret == false) {
147                        log.warn("Unable to queue the callables commands for RecoveryService. "
148                                + "Most possibly command queue is full. Queue size is :"
149                                + Services.get().get(CallableQueueService.class).queueSize());
150                    }
151                    callables = null;
152                }
153                if (null != delayedCallables) {
154                    ret = Services.get().get(CallableQueueService.class).queueSerial(delayedCallables, this.delay);
155                    if (ret == false) {
156                        log.warn("Unable to queue the delayedCallables commands for RecoveryService. "
157                                + "Most possibly Callable queue is full. Queue size is :"
158                                + Services.get().get(CallableQueueService.class).queueSize());
159                    }
160                    delayedCallables = null;
161                    this.delay = 0;
162                }
163            }
164    
165            private void runBundleRecovery(){
166                XLog.Info.get().clear();
167                XLog log = XLog.getLog(getClass());
168                List<BundleActionBean> bactions = null;
169                try {
170                    bactions = BundleActionQueryExecutor.getInstance().getList(
171                            BundleActionQuery.GET_BUNDLE_WAITING_ACTIONS_OLDER_THAN, bundleOlderThan);
172                }
173                catch (JPAExecutorException ex) {
174                    log.warn("Error reading bundle actions from database", ex);
175                    return;
176                }
177                msg.append(", BUNDLE_ACTIONS : " + bactions.size());
178                for (BundleActionBean baction : bactions) {
179                    try {
180                        Services.get().get(InstrumentationService.class).get()
181                                .incr(INSTRUMENTATION_GROUP, INSTR_RECOVERED_BUNDLE_ACTIONS_COUNTER, 1);
182                        if (baction.getCoordId() == null && baction.getStatus() != Job.Status.PREP) {
183                            log.error("CoordId is null for Bundle action " + baction.getBundleActionId());
184                            continue;
185                        }
186                        if (Services.get().get(JobsConcurrencyService.class).isJobIdForThisServer(baction.getBundleId())) {
187                            if (baction.getStatus() == Job.Status.PREP && baction.getCoordId() == null) {
188                                BundleJobBean bundleJob = null;
189                                if (jpaService != null) {
190                                    bundleJob = BundleJobQueryExecutor.getInstance().get(
191                                            BundleJobQuery.GET_BUNDLE_JOB_ID_JOBXML_CONF, baction.getBundleId());
192                                }
193                                Element bAppXml = XmlUtils.parseXml(bundleJob.getJobXml());
194                                List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace());
195                                for (Element coordElem : coordElems) {
196                                    Attribute name = coordElem.getAttribute("name");
197                                    if (name.getValue().equals(baction.getCoordName())) {
198                                        Configuration coordConf = mergeConfig(coordElem, bundleJob);
199                                        coordConf.set(OozieClient.BUNDLE_ID, baction.getBundleId());
200                                        queueCallable(new CoordSubmitXCommand(coordConf,
201                                                bundleJob.getId(), name.getValue()));
202                                    }
203                                }
204                            }
205                            else if (baction.getStatus() == Job.Status.KILLED) {
206                                queueCallable(new CoordKillXCommand(baction.getCoordId()));
207                            }
208                            else if (baction.getStatus() == Job.Status.SUSPENDED
209                                    || baction.getStatus() == Job.Status.SUSPENDEDWITHERROR) {
210                                queueCallable(new CoordSuspendXCommand(baction.getCoordId()));
211                            }
212                            else if (baction.getStatus() == Job.Status.RUNNING
213                                    || baction.getStatus() == Job.Status.RUNNINGWITHERROR) {
214                                queueCallable(new CoordResumeXCommand(baction.getCoordId()));
215                            }
216                        }
217                    }
218                    catch (Exception ex) {
219                        log.error("Exception, {0}", ex.getMessage(), ex);
220                    }
221                }
222    
223            }
224    
225            /**
226             * Recover coordinator actions that are staying in WAITING or SUBMITTED too long
227             */
228            private void runCoordActionRecovery() {
229                XLog.Info.get().clear();
230                XLog log = XLog.getLog(getClass());
231                long pushMissingDepInterval = Services.get().getConf().getLong(CONF_PUSH_DEPENDENCY_INTERVAL, 200);
232                long pushMissingDepDelay = pushMissingDepInterval;
233                List<CoordinatorActionBean> cactions = null;
234                try {
235                    cactions = jpaService.execute(new CoordActionsGetForRecoveryJPAExecutor(coordOlderThan));
236                }
237                catch (JPAExecutorException ex) {
238                    log.warn("Error reading coord actions from database", ex);
239                    return;
240                }
241                msg.append(", COORD_ACTIONS : " + cactions.size());
242                for (CoordinatorActionBean caction : cactions) {
243                    try {
244                        if (Services.get().get(JobsConcurrencyService.class).isJobIdForThisServer(caction.getId())) {
245                            Services.get().get(InstrumentationService.class).get()
246                                    .incr(INSTRUMENTATION_GROUP, INSTR_RECOVERED_COORD_ACTIONS_COUNTER, 1);
247                            if (caction.getStatus() == CoordinatorActionBean.Status.WAITING) {
248                                queueCallable(new CoordActionInputCheckXCommand(caction.getId(), caction.getJobId()));
249                                log.info("Recover a WAITING coord action and resubmit CoordActionInputCheckXCommand :"
250                                        + caction.getId());
251                                if (caction.getPushMissingDependencies() != null
252                                        && caction.getPushMissingDependencies().length() != 0) {
253                                    queueCallable(new CoordPushDependencyCheckXCommand(caction.getId(), true, true),
254                                            pushMissingDepDelay);
255                                    pushMissingDepDelay = pushMissingDepDelay + pushMissingDepInterval;
256                                    log.info("Recover a WAITING coord action and resubmit CoordPushDependencyCheckX :"
257                                            + caction.getId());
258                                }
259                            }
260                            else if (caction.getStatus() == CoordinatorActionBean.Status.SUBMITTED) {
261                                CoordinatorJobBean coordJob = CoordJobQueryExecutor.getInstance().get(
262                                        CoordJobQuery.GET_COORD_JOB_USER_APPNAME, caction.getJobId());
263                                queueCallable(new CoordActionStartXCommand(caction.getId(), coordJob.getUser(),
264                                        coordJob.getAppName(), caction.getJobId()));
265    
266                                log.info("Recover a SUBMITTED coord action and resubmit CoordActionStartCommand :"
267                                        + caction.getId());
268                            }
269                            else if (caction.getStatus() == CoordinatorActionBean.Status.SUSPENDED) {
270                                if (caction.getExternalId() != null && caction.getPending() > 1) {
271                                    queueCallable(new SuspendXCommand(caction.getExternalId()));
272                                    log.debug("Recover a SUSPENDED coord action and resubmit SuspendXCommand :"
273                                            + caction.getId());
274                                }
275                            }
276                            else if (caction.getStatus() == CoordinatorActionBean.Status.KILLED) {
277                                if (caction.getExternalId() != null) {
278                                    queueCallable(new KillXCommand(caction.getExternalId()));
279                                    log.debug("Recover a KILLED coord action and resubmit KillXCommand :" + caction.getId());
280                                }
281                            }
282                            else if (caction.getStatus() == CoordinatorActionBean.Status.RUNNING) {
283                                if (caction.getExternalId() != null) {
284                                    queueCallable(new ResumeXCommand(caction.getExternalId()));
285                                    log.debug("Recover a RUNNING coord action and resubmit ResumeXCommand :" + caction.getId());
286                                }
287                            }
288                        }
289                    }
290                    catch (Exception ex) {
291                        log.error("Exception, {0}", ex.getMessage(), ex);
292                    }
293                }
294    
295    
296            }
297    
298            /**
299             * Recover coordinator actions that are staying in READY too long
300             */
301            private void runCoordActionRecoveryForReady() {
302                XLog.Info.get().clear();
303                XLog log = XLog.getLog(getClass());
304    
305                try {
306                    List<String> jobids = jpaService.execute(new CoordActionsGetReadyGroupbyJobIDJPAExecutor(coordOlderThan));
307                    jobids = Services.get().get(JobsConcurrencyService.class).getJobIdsForThisServer(jobids);
308                    msg.append(", COORD_READY_JOBS : " + jobids.size());
309                    for (String jobid : jobids) {
310                            queueCallable(new CoordActionReadyXCommand(jobid));
311    
312                        log.info("Recover READY coord actions for jobid :" + jobid);
313                    }
314                }
315                catch (Exception ex) {
316                    log.error("Exception, {0}", ex.getMessage(), ex);
317                }
318            }
319    
320            /**
321             * Recover wf actions
322             */
323            private void runWFRecovery() {
324                XLog.Info.get().clear();
325                XLog log = XLog.getLog(getClass());
326                // queue command for action recovery
327                List<WorkflowActionBean> actions = null;
328                try {
329                    actions = WorkflowActionQueryExecutor.getInstance().getList(WorkflowActionQuery.GET_PENDING_ACTIONS,
330                            olderThan);
331                }
332                catch (JPAExecutorException ex) {
333                    log.warn("Exception while reading pending actions from storage", ex);
334                    return;
335                }
336                // log.debug("QUEUING[{0}] pending wf actions for potential recovery",
337                // actions.size());
338                msg.append(" WF_ACTIONS " + actions.size());
339    
340                for (WorkflowActionBean action : actions) {
341                    try {
342                        if (Services.get().get(JobsConcurrencyService.class).isJobIdForThisServer(action.getId())) {
343                            Services.get().get(InstrumentationService.class).get()
344                                    .incr(INSTRUMENTATION_GROUP, INSTR_RECOVERED_ACTIONS_COUNTER, 1);
345                            if (action.getStatus() == WorkflowActionBean.Status.PREP
346                                    || action.getStatus() == WorkflowActionBean.Status.START_MANUAL) {
347                                queueCallable(new ActionStartXCommand(action.getId(), action.getType()));
348                            }
349                            else if (action.getStatus() == WorkflowActionBean.Status.START_RETRY) {
350                                Date nextRunTime = action.getPendingAge();
351                                queueCallable(new ActionStartXCommand(action.getId(), action.getType()), nextRunTime.getTime()
352                                        - System.currentTimeMillis());
353                            }
354                            else if (action.getStatus() == WorkflowActionBean.Status.DONE
355                                    || action.getStatus() == WorkflowActionBean.Status.END_MANUAL) {
356                                queueCallable(new ActionEndXCommand(action.getId(), action.getType()));
357                            }
358                            else if (action.getStatus() == WorkflowActionBean.Status.END_RETRY) {
359                                Date nextRunTime = action.getPendingAge();
360                                queueCallable(new ActionEndXCommand(action.getId(), action.getType()), nextRunTime.getTime()
361                                        - System.currentTimeMillis());
362    
363                            }
364                            else if (action.getStatus() == WorkflowActionBean.Status.OK
365                                    || action.getStatus() == WorkflowActionBean.Status.ERROR) {
366                                queueCallable(new SignalXCommand(action.getJobId(), action.getId()));
367                            }
368                            else if (action.getStatus() == WorkflowActionBean.Status.USER_RETRY) {
369                                queueCallable(new ActionStartXCommand(action.getId(), action.getType()));
370                            }
371                        }
372                    }
373                    catch (Exception ex) {
374                        log.error("Exception, {0}", ex.getMessage(), ex);
375                    }
376                }
377    
378            }
379    
380            /**
381             * Adds callables to a list. If the number of callables in the list reaches {@link
382             * RecoveryService#CONF_CALLABLE_BATCH_SIZE}, the entire batch is queued and the callables list is reset.
383             *
384             * @param callable the callable to queue.
385             */
386            private void queueCallable(XCallable<?> callable) {
387                if (callables == null) {
388                    callables = new ArrayList<XCallable<?>>();
389                }
390                callables.add(callable);
391                if (callables.size() == Services.get().getConf().getInt(CONF_CALLABLE_BATCH_SIZE, 10)) {
392                    boolean ret = Services.get().get(CallableQueueService.class).queueSerial(callables);
393                    if (ret == false) {
394                        XLog.getLog(getClass()).warn(
395                                "Unable to queue the callables commands for RecoveryService. "
396                                        + "Most possibly command queue is full. Queue size is :"
397                                        + Services.get().get(CallableQueueService.class).queueSize());
398                    }
399                    callables = new ArrayList<XCallable<?>>();
400                }
401            }
402    
403            /**
404             * Adds callables to a list. If the number of callables in the list reaches {@link
405             * RecoveryService#CONF_CALLABLE_BATCH_SIZE}, the entire batch is queued with the delay set to the maximum delay
406             * of the callables in the list. The callables list and the delay is reset.
407             *
408             * @param callable the callable to queue.
409             * @param delay the delay for the callable.
410             */
411            private void queueCallable(XCallable<?> callable, long delay) {
412                if (delayedCallables == null) {
413                    delayedCallables = new ArrayList<XCallable<?>>();
414                }
415                this.delay = Math.max(this.delay, delay);
416                delayedCallables.add(callable);
417                if (delayedCallables.size() == Services.get().getConf().getInt(CONF_CALLABLE_BATCH_SIZE, 10)) {
418                    boolean ret = Services.get().get(CallableQueueService.class).queueSerial(delayedCallables, this.delay);
419                    if (ret == false) {
420                        XLog.getLog(getClass()).warn("Unable to queue the delayedCallables commands for RecoveryService. "
421                                + "Most possibly Callable queue is full. Queue size is :"
422                                + Services.get().get(CallableQueueService.class).queueSize());
423                    }
424                    delayedCallables = new ArrayList<XCallable<?>>();
425                    this.delay = 0;
426                }
427            }
428        }
429    
430        /**
431         * Initializes the RecoveryService.
432         *
433         * @param services services instance.
434         */
435        @Override
436        public void init(Services services) {
437            Configuration conf = services.getConf();
438            Runnable recoveryRunnable = new RecoveryRunnable(conf.getInt(CONF_WF_ACTIONS_OLDER_THAN, 120), conf.getInt(
439                    CONF_COORD_OLDER_THAN, 600),conf.getInt(CONF_BUNDLE_OLDER_THAN, 600));
440            services.get(SchedulerService.class).schedule(recoveryRunnable, 10, getRecoveryServiceInterval(conf),
441                                                          SchedulerService.Unit.SEC);
442        }
443    
444        public int getRecoveryServiceInterval(Configuration conf){
445            return conf.getInt(CONF_SERVICE_INTERVAL, 60);
446        }
447    
448        /**
449         * Destroy the Recovery Service.
450         */
451        @Override
452        public void destroy() {
453        }
454    
455        /**
456         * Return the public interface for the Recovery Service.
457         *
458         * @return {@link RecoveryService}.
459         */
460        @Override
461        public Class<? extends Service> getInterface() {
462            return RecoveryService.class;
463        }
464    
465        /**
466         * Merge Bundle job config and the configuration from the coord job to pass
467         * to Coord Engine
468         *
469         * @param coordElem the coordinator configuration
470         * @return Configuration merged configuration
471         * @throws CommandException thrown if failed to merge configuration
472         */
473        private static Configuration mergeConfig(Element coordElem,BundleJobBean bundleJob) throws CommandException {
474            XLog.Info.get().clear();
475            XLog log = XLog.getLog("RecoveryService");
476    
477            String jobConf = bundleJob.getConf();
478            // Step 1: runConf = jobConf
479            Configuration runConf = null;
480            try {
481                runConf = new XConfiguration(new StringReader(jobConf));
482            }
483            catch (IOException e1) {
484                log.warn("Configuration parse error in:" + jobConf);
485                throw new CommandException(ErrorCode.E1306, e1.getMessage(), e1);
486            }
487            // Step 2: Merge local properties into runConf
488            // extract 'property' tags under 'configuration' block in the coordElem
489            // convert Element to XConfiguration
490            Element localConfigElement = coordElem.getChild("configuration", coordElem.getNamespace());
491    
492            if (localConfigElement != null) {
493                String strConfig = XmlUtils.prettyPrint(localConfigElement).toString();
494                Configuration localConf;
495                try {
496                    localConf = new XConfiguration(new StringReader(strConfig));
497                }
498                catch (IOException e1) {
499                    log.warn("Configuration parse error in:" + strConfig);
500                    throw new CommandException(ErrorCode.E1307, e1.getMessage(), e1);
501                }
502    
503                // copy configuration properties in the coordElem to the runConf
504                XConfiguration.copy(localConf, runConf);
505            }
506    
507            // Step 3: Extract value of 'app-path' in coordElem, save it as a
508            // new property called 'oozie.coord.application.path', and normalize.
509            String appPath = coordElem.getChild("app-path", coordElem.getNamespace()).getValue();
510            runConf.set(OozieClient.COORDINATOR_APP_PATH, appPath);
511            // Normalize coordinator appPath here;
512            try {
513                JobUtils.normalizeAppPath(runConf.get(OozieClient.USER_NAME), runConf.get(OozieClient.GROUP_NAME), runConf);
514            }
515            catch (IOException e) {
516                throw new CommandException(ErrorCode.E1001, runConf.get(OozieClient.COORDINATOR_APP_PATH));
517            }
518            return runConf;
519        }
520    }