001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.command.coord;
019    
020    import java.io.StringReader;
021    import java.net.URI;
022    import java.text.ParseException;
023    import java.util.TimeZone;
024    import java.util.Map;
025    import java.util.HashMap;
026    import java.util.List;
027    import java.util.Date;
028    import java.util.Calendar;
029    
030    import org.apache.hadoop.conf.Configuration;
031    import org.apache.oozie.CoordinatorActionBean;
032    import org.apache.oozie.ErrorCode;
033    import org.apache.oozie.client.CoordinatorAction;
034    import org.apache.oozie.command.CommandException;
035    import org.apache.oozie.coord.CoordELEvaluator;
036    import org.apache.oozie.coord.CoordELFunctions;
037    import org.apache.oozie.coord.CoordUtils;
038    import org.apache.oozie.coord.CoordinatorJobException;
039    import org.apache.oozie.coord.SyncCoordAction;
040    import org.apache.oozie.coord.TimeUnit;
041    import org.apache.oozie.dependency.ActionDependency;
042    import org.apache.oozie.dependency.DependencyChecker;
043    import org.apache.oozie.dependency.URIHandler;
044    import org.apache.oozie.dependency.URIHandler.DependencyType;
045    import org.apache.oozie.service.Services;
046    import org.apache.oozie.service.URIHandlerService;
047    import org.apache.oozie.service.UUIDService;
048    import org.apache.oozie.util.DateUtils;
049    import org.apache.oozie.util.ELEvaluator;
050    import org.apache.oozie.util.XConfiguration;
051    import org.apache.oozie.util.XmlUtils;
052    import org.jdom.Element;
053    import org.quartz.CronExpression;
054    import org.apache.commons.lang.StringUtils;
055    import org.apache.oozie.CoordinatorJobBean;
056    
057    public class CoordCommandUtils {
058        public static int CURRENT = 0;
059        public static int LATEST = 1;
060        public static int FUTURE = 2;
061        public static int OFFSET = 3;
062        public static int ABSOLUTE = 4;
063        public static int UNEXPECTED = -1;
064        public static final String RESOLVED_UNRESOLVED_SEPARATOR = "!!";
065        public static final String UNRESOLVED_INST_TAG = "unresolved-instances";
066    
067        /**
068         * parse a function like coord:latest(n)/future() and return the 'n'.
069         * <p/>
070         *
071         * @param function
072         * @param restArg
073         * @return int instanceNumber
074         * @throws Exception
075         */
076        public static int getInstanceNumber(String function, StringBuilder restArg) throws Exception {
077            int funcType = getFuncType(function);
078            if (funcType == ABSOLUTE) {
079                return ABSOLUTE;
080            }
081            if (funcType == CURRENT || funcType == LATEST) {
082                return parseOneArg(function);
083            }
084            else {
085                return parseMoreArgs(function, restArg);
086            }
087        }
088    
089        /**
090         * Evaluates function for coord-action-create-inst tag
091         * @param event
092         * @param appInst
093         * @param conf
094         * @param function
095         * @return evaluation result
096         * @throws Exception
097         */
098        private static String evaluateInstanceFunction(Element event, SyncCoordAction appInst, Configuration conf,
099                String function) throws Exception {
100            ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator("coord-action-create-inst", event, appInst, conf);
101            return CoordELFunctions.evalAndWrap(eval, function);
102        }
103    
104        public static int parseOneArg(String funcName) throws Exception {
105            int firstPos = funcName.indexOf("(");
106            int lastPos = funcName.lastIndexOf(")");
107            if (firstPos >= 0 && lastPos > firstPos) {
108                String tmp = funcName.substring(firstPos + 1, lastPos).trim();
109                if (tmp.length() > 0) {
110                    return (int) Double.parseDouble(tmp);
111                }
112            }
113            throw new RuntimeException("Unformatted function :" + funcName);
114        }
115    
116        public static String parseOneStringArg(String funcName) throws Exception {
117            int firstPos = funcName.indexOf("(");
118            int lastPos = funcName.lastIndexOf(")");
119            if (firstPos >= 0 && lastPos > firstPos) {
120                return funcName.substring(firstPos + 1, lastPos).trim();
121            }
122            throw new RuntimeException("Unformatted function :" + funcName);
123        }
124    
125        private static int parseMoreArgs(String funcName, StringBuilder restArg) throws Exception {
126            int firstPos = funcName.indexOf("(");
127            int secondPos = funcName.lastIndexOf(",");
128            int lastPos = funcName.lastIndexOf(")");
129            if (firstPos >= 0 && secondPos > firstPos) {
130                String tmp = funcName.substring(firstPos + 1, secondPos).trim();
131                if (tmp.length() > 0) {
132                    restArg.append(funcName.substring(secondPos + 1, lastPos).trim());
133                    return (int) Double.parseDouble(tmp);
134                }
135            }
136            throw new RuntimeException("Unformatted function :" + funcName);
137        }
138    
139        /**
140         * @param EL function name
141         * @return type of EL function
142         */
143        public static int getFuncType(String function) {
144            if (function.indexOf("current") >= 0) {
145                return CURRENT;
146            }
147            else if (function.indexOf("latest") >= 0) {
148                return LATEST;
149            }
150            else if (function.indexOf("future") >= 0) {
151                return FUTURE;
152            }
153            else if (function.indexOf("offset") >= 0) {
154                return OFFSET;
155            }
156            else if (function.indexOf("absolute") >= 0) {
157                return ABSOLUTE;
158            }
159            return UNEXPECTED;
160            // throw new RuntimeException("Unexpected instance name "+ function);
161        }
162    
163        /**
164         * @param startInst: EL function name
165         * @param endInst: EL function name
166         * @throws CommandException if both are not the same function
167         */
168        public static void checkIfBothSameType(String startInst, String endInst) throws CommandException {
169            if (getFuncType(startInst) != getFuncType(endInst)) {
170                if (getFuncType(startInst) == ABSOLUTE) {
171                    if (getFuncType(endInst) != CURRENT) {
172                        throw new CommandException(ErrorCode.E1010,
173                                "Only start-instance as absolute and end-instance as current is supported." + " start = "
174                                        + startInst + "  end = " + endInst);
175                    }
176                }
177                else {
178                    throw new CommandException(ErrorCode.E1010,
179                            " start-instance and end-instance both should be either latest or current or future or offset\n"
180                                    + " start " + startInst + " and end " + endInst);
181                }
182            }
183        }
184    
185    
186        /**
187         * Resolve list of <instance> </instance> tags.
188         *
189         * @param event
190         * @param instances
191         * @param actionInst
192         * @param conf
193         * @param eval: ELEvalautor
194         * @throws Exception
195         */
196        public static void resolveInstances(Element event, StringBuilder instances, SyncCoordAction actionInst,
197                Configuration conf, ELEvaluator eval) throws Exception {
198            for (Element eInstance : (List<Element>) event.getChildren("instance", event.getNamespace())) {
199    
200                if (instances.length() > 0) {
201                    instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
202                }
203                instances.append(materializeInstance(event, eInstance.getTextTrim(), actionInst, conf, eval));
204            }
205            event.removeChildren("instance", event.getNamespace());
206        }
207    
208        /**
209         * Resolve <start-instance> <end-insatnce> tag. Don't resolve any
210         * latest()/future()
211         *
212         * @param event
213         * @param instances
214         * @param appInst
215         * @param conf
216         * @param eval: ELEvalautor
217         * @throws Exception
218         */
219        public static void resolveInstanceRange(Element event, StringBuilder instances, SyncCoordAction appInst,
220                Configuration conf, ELEvaluator eval) throws Exception {
221            Element eStartInst = event.getChild("start-instance", event.getNamespace());
222            Element eEndInst = event.getChild("end-instance", event.getNamespace());
223            if (eStartInst != null && eEndInst != null) {
224                String strStart = evaluateInstanceFunction(event, appInst, conf, eStartInst.getTextTrim());
225                String strEnd = evaluateInstanceFunction(event, appInst, conf, eEndInst.getTextTrim());
226                checkIfBothSameType(strStart, strEnd);
227                StringBuilder restArg = new StringBuilder(); // To store rest
228                                                             // arguments for
229                                                             // future
230                                                             // function
231                int startIndex = getInstanceNumber(strStart, restArg);
232                String startRestArg = restArg.toString();
233                restArg.delete(0, restArg.length());
234                int endIndex = getInstanceNumber(strEnd, restArg);
235                String endRestArg = restArg.toString();
236                int funcType = getFuncType(strStart);
237    
238                if (funcType == ABSOLUTE) {
239                    StringBuffer bf = new StringBuffer();
240                    bf.append("${coord:absoluteRange(\"").append(parseOneStringArg(strStart))
241                            .append("\",").append(endIndex).append(")}");
242                    String matInstance = materializeInstance(event, bf.toString(), appInst, conf, eval);
243                    if (matInstance != null && !matInstance.isEmpty()) {
244                        if (instances.length() > 0) {
245                            instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
246                        }
247                        instances.append(matInstance);
248                    }
249                }
250                else {
251                    if (funcType == OFFSET) {
252                        TimeUnit startU = TimeUnit.valueOf(startRestArg);
253                        TimeUnit endU = TimeUnit.valueOf(endRestArg);
254                        if (startU.getCalendarUnit() * startIndex > endU.getCalendarUnit() * endIndex) {
255                            throw new CommandException(ErrorCode.E1010,
256                                    " start-instance should be equal or earlier than the end-instance \n"
257                                            + XmlUtils.prettyPrint(event));
258                        }
259                        Calendar startCal = CoordELFunctions.resolveOffsetRawTime(startIndex, startU, eval);
260                        Calendar endCal = CoordELFunctions.resolveOffsetRawTime(endIndex, endU, eval);
261                        if (startCal != null && endCal != null) {
262                            List<Integer> expandedFreqs = CoordELFunctions.expandOffsetTimes(startCal, endCal, eval);
263                            for (int i = expandedFreqs.size() - 1; i >= 0; i--) {
264                                String matInstance = materializeInstance(event, "${coord:offset(" + expandedFreqs.get(i)
265                                        + ", \"MINUTE\")}", appInst, conf, eval);
266                                if (matInstance == null || matInstance.length() == 0) {
267                                    // Earlier than dataset's initial instance
268                                    break;
269                                }
270                                if (instances.length() > 0) {
271                                    instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
272                                }
273                                instances.append(matInstance);
274                            }
275                        }
276                    }
277                    else {
278                        if (startIndex > endIndex) {
279                            throw new CommandException(ErrorCode.E1010,
280                                    " start-instance should be equal or earlier than the end-instance \n"
281                                            + XmlUtils.prettyPrint(event));
282                        }
283                        if (funcType == CURRENT) {
284                            // Everything could be resolved NOW. no latest() ELs
285                            String matInstance = materializeInstance(event, "${coord:currentRange(" + startIndex + ","
286                                    + endIndex + ")}", appInst, conf, eval);
287                            if (matInstance != null && !matInstance.isEmpty()) {
288                                if (instances.length() > 0) {
289                                    instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
290                                }
291                                instances.append(matInstance);
292                            }
293                        }
294    
295                        else { // latest(n)/future() EL is present
296                            if (funcType == LATEST) {
297                                instances.append("${coord:latestRange(").append(startIndex).append(",").append(endIndex)
298                                .append(")}");
299                            }
300                            else if (funcType == FUTURE) {
301                                instances.append("${coord:futureRange(").append(startIndex).append(",").append(endIndex)
302                                .append(",'").append(endRestArg).append("')}");
303                            }
304                        }
305                    }
306                }
307                // Remove start-instance and end-instances
308                event.removeChild("start-instance", event.getNamespace());
309                event.removeChild("end-instance", event.getNamespace());
310            }
311        }
312    
313        /**
314         * Materialize one instance like current(-2)
315         *
316         * @param event : <data-in>
317         * @param expr : instance like current(-1)
318         * @param appInst : application specific info
319         * @param conf
320         * @param evalInst :ELEvaluator
321         * @return materialized date string
322         * @throws Exception
323         */
324        public static String materializeInstance(Element event, String expr, SyncCoordAction appInst, Configuration conf,
325                ELEvaluator evalInst) throws Exception {
326            if (event == null) {
327                return null;
328            }
329            // ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event,
330            // appInst, conf);
331            return CoordELFunctions.evalAndWrap(evalInst, expr);
332        }
333    
334        /**
335         * Create two new tags with <uris> and <unresolved-instances>.
336         *
337         * @param event
338         * @param instances
339         * @throws Exception
340         */
341        private static String separateResolvedAndUnresolved(Element event, StringBuilder instances)
342                throws Exception {
343            StringBuilder unresolvedInstances = new StringBuilder();
344            StringBuilder urisWithDoneFlag = new StringBuilder();
345            StringBuilder depList = new StringBuilder();
346            String uris = createEarlyURIs(event, instances.toString(), unresolvedInstances, urisWithDoneFlag);
347            if (uris.length() > 0) {
348                Element uriInstance = new Element("uris", event.getNamespace());
349                uriInstance.addContent(uris);
350                event.getContent().add(1, uriInstance);
351                if (depList.length() > 0) {
352                    depList.append(CoordELFunctions.INSTANCE_SEPARATOR);
353                }
354                depList.append(urisWithDoneFlag);
355            }
356            if (unresolvedInstances.length() > 0) {
357                Element elemInstance = new Element(UNRESOLVED_INST_TAG, event.getNamespace());
358                elemInstance.addContent(unresolvedInstances.toString());
359                event.getContent().add(1, elemInstance);
360            }
361            return depList.toString();
362        }
363    
364        /**
365         * The function create a list of URIs separated by "," using the instances
366         * time stamp and URI-template
367         *
368         * @param event : <data-in> event
369         * @param instances : List of time stamp separated by ","
370         * @param unresolvedInstances : list of instance with latest function
371         * @param urisWithDoneFlag : list of URIs with the done flag appended
372         * @return : list of URIs separated by ";" as a string.
373         * @throws Exception
374         */
375        public static String createEarlyURIs(Element event, String instances, StringBuilder unresolvedInstances,
376                StringBuilder urisWithDoneFlag) throws Exception {
377            if (instances == null || instances.length() == 0) {
378                return "";
379            }
380            String[] instanceList = instances.split(CoordELFunctions.INSTANCE_SEPARATOR);
381            StringBuilder uris = new StringBuilder();
382    
383            Element doneFlagElement = event.getChild("dataset", event.getNamespace()).getChild("done-flag",
384                    event.getNamespace());
385            URIHandlerService uriService = Services.get().get(URIHandlerService.class);
386    
387            for (int i = 0; i < instanceList.length; i++) {
388                if (instanceList[i].trim().length() == 0) {
389                    continue;
390                }
391                int funcType = getFuncType(instanceList[i]);
392                if (funcType == LATEST || funcType == FUTURE) {
393                    if (unresolvedInstances.length() > 0) {
394                        unresolvedInstances.append(CoordELFunctions.INSTANCE_SEPARATOR);
395                    }
396                    unresolvedInstances.append(instanceList[i]);
397                    continue;
398                }
399                ELEvaluator eval = CoordELEvaluator.createURIELEvaluator(instanceList[i]);
400                if (uris.length() > 0) {
401                    uris.append(CoordELFunctions.INSTANCE_SEPARATOR);
402                    urisWithDoneFlag.append(CoordELFunctions.INSTANCE_SEPARATOR);
403                }
404    
405                String uriPath = CoordELFunctions.evalAndWrap(eval, event.getChild("dataset", event.getNamespace())
406                        .getChild("uri-template", event.getNamespace()).getTextTrim());
407                URIHandler uriHandler = uriService.getURIHandler(uriPath);
408                uriHandler.validate(uriPath);
409                uris.append(uriPath);
410                urisWithDoneFlag.append(uriHandler.getURIWithDoneFlag(uriPath, CoordUtils.getDoneFlag(doneFlagElement)));
411            }
412            return uris.toString();
413        }
414    
415        /**
416         * @param eAction
417         * @param coordAction
418         * @param conf
419         * @return boolean to determine whether the SLA element is present or not
420         * @throws CoordinatorJobException
421         */
422        public static boolean materializeSLA(Element eAction, CoordinatorActionBean coordAction, Configuration conf)
423                throws CoordinatorJobException {
424            Element eSla = eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla"));
425            if (eSla == null) {
426                // eAppXml.getNamespace("sla"));
427                return false;
428            }
429            try {
430                ELEvaluator evalSla = CoordELEvaluator.createSLAEvaluator(eAction, coordAction, conf);
431                List<Element> elemList = eSla.getChildren();
432                for (Element elem : elemList) {
433                    String updated;
434                    try {
435                        updated = CoordELFunctions.evalAndWrap(evalSla, elem.getText().trim());
436                    }
437                    catch (Exception e) {
438                        throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e);
439                    }
440                    elem.removeContent();
441                    elem.addContent(updated);
442                }
443            }
444            catch (Exception e) {
445                throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e);
446            }
447            return true;
448        }
449    
450        /**
451         * Materialize one instance for specific nominal time. It includes: 1.
452         * Materialize data events (i.e. <data-in> and <data-out>) 2. Materialize
453         * data properties (i.e dataIn(<DS>) and dataOut(<DS>) 3. remove 'start' and
454         * 'end' tag 4. Add 'instance_number' and 'nominal-time' tag
455         *
456         * @param jobId coordinator job id
457         * @param dryrun true if it is dryrun
458         * @param eAction frequency unexploded-job
459         * @param nominalTime materialization time
460         * @param actualTime action actual time
461         * @param instanceCount instance numbers
462         * @param conf job configuration
463         * @param actionBean CoordinatorActionBean to materialize
464         * @return one materialized action for specific nominal time
465         * @throws Exception
466         */
467        @SuppressWarnings("unchecked")
468        public static String materializeOneInstance(String jobId, boolean dryrun, Element eAction, Date nominalTime,
469                Date actualTime, int instanceCount, Configuration conf, CoordinatorActionBean actionBean) throws Exception {
470            String actionId = Services.get().get(UUIDService.class).generateChildId(jobId, instanceCount + "");
471            SyncCoordAction appInst = new SyncCoordAction();
472            appInst.setActionId(actionId);
473            appInst.setName(eAction.getAttributeValue("name"));
474            appInst.setNominalTime(nominalTime);
475            appInst.setActualTime(actualTime);
476            String frequency = eAction.getAttributeValue("frequency");
477            appInst.setFrequency(frequency);
478            appInst.setTimeUnit(TimeUnit.valueOf(eAction.getAttributeValue("freq_timeunit")));
479            appInst.setTimeZone(DateUtils.getTimeZone(eAction.getAttributeValue("timezone")));
480            appInst.setEndOfDuration(TimeUnit.valueOf(eAction.getAttributeValue("end_of_duration")));
481    
482            Map<String, StringBuilder> dependencyMap = null;
483    
484            Element inputList = eAction.getChild("input-events", eAction.getNamespace());
485            List<Element> dataInList = null;
486            if (inputList != null) {
487                dataInList = inputList.getChildren("data-in", eAction.getNamespace());
488                dependencyMap = materializeDataEvents(dataInList, appInst, conf);
489            }
490    
491            Element outputList = eAction.getChild("output-events", eAction.getNamespace());
492            List<Element> dataOutList = null;
493            if (outputList != null) {
494                dataOutList = outputList.getChildren("data-out", eAction.getNamespace());
495                materializeDataEvents(dataOutList, appInst, conf);
496            }
497    
498            eAction.removeAttribute("start");
499            eAction.removeAttribute("end");
500            eAction.setAttribute("instance-number", Integer.toString(instanceCount));
501            eAction.setAttribute("action-nominal-time", DateUtils.formatDateOozieTZ(nominalTime));
502            eAction.setAttribute("action-actual-time", DateUtils.formatDateOozieTZ(actualTime));
503    
504            // Setting up action bean
505            actionBean.setCreatedConf(XmlUtils.prettyPrint(conf).toString());
506            actionBean.setRunConf(XmlUtils.prettyPrint(conf).toString());
507            actionBean.setCreatedTime(actualTime);
508            actionBean.setJobId(jobId);
509            actionBean.setId(actionId);
510            actionBean.setLastModifiedTime(new Date());
511            actionBean.setStatus(CoordinatorAction.Status.WAITING);
512            actionBean.setActionNumber(instanceCount);
513            if (dependencyMap != null) {
514                StringBuilder sbPull = dependencyMap.get(DependencyType.PULL.name());
515                if (sbPull != null) {
516                    actionBean.setMissingDependencies(sbPull.toString());
517                }
518                StringBuilder sbPush = dependencyMap.get(DependencyType.PUSH.name());
519                if (sbPush != null) {
520                    actionBean.setPushMissingDependencies(sbPush.toString());
521                }
522            }
523            actionBean.setNominalTime(nominalTime);
524            boolean isSla = CoordCommandUtils.materializeSLA(eAction, actionBean, conf);
525            if (isSla == true) {
526                actionBean.setSlaXml(XmlUtils.prettyPrint(
527                        eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla")))
528                        .toString());
529            }
530    
531            // actionBean.setTrackerUri(trackerUri);//TOOD:
532            // actionBean.setConsoleUrl(consoleUrl); //TODO:
533            // actionBean.setType(type);//TODO:
534            // actionBean.setErrorInfo(errorCode, errorMessage); //TODO:
535            // actionBean.setExternalStatus(externalStatus);//TODO
536            if (!dryrun) {
537                return XmlUtils.prettyPrint(eAction).toString();
538            }
539            else {
540                return dryRunCoord(eAction, actionBean);
541            }
542        }
543    
544        /**
545         * @param eAction the actionXml related element
546         * @param actionBean the coordinator action bean
547         * @return
548         * @throws Exception
549         */
550        static String dryRunCoord(Element eAction, CoordinatorActionBean actionBean) throws Exception {
551            String action = XmlUtils.prettyPrint(eAction).toString();
552            StringBuilder actionXml = new StringBuilder(action);
553            Configuration actionConf = new XConfiguration(new StringReader(actionBean.getRunConf()));
554    
555            boolean isPushDepAvailable = true;
556            if (actionBean.getPushMissingDependencies() != null) {
557                ActionDependency actionDep = DependencyChecker.checkForAvailability(
558                        actionBean.getPushMissingDependencies(), actionConf, true);
559                if (actionDep.getMissingDependencies().size() != 0) {
560                    isPushDepAvailable = false;
561                }
562    
563            }
564            boolean isPullDepAvailable = true;
565            CoordActionInputCheckXCommand coordActionInput = new CoordActionInputCheckXCommand(actionBean.getId(),
566                    actionBean.getJobId());
567            if (actionBean.getMissingDependencies() != null) {
568                StringBuilder existList = new StringBuilder();
569                StringBuilder nonExistList = new StringBuilder();
570                StringBuilder nonResolvedList = new StringBuilder();
571                getResolvedList(actionBean.getMissingDependencies(), nonExistList, nonResolvedList);
572                isPullDepAvailable = coordActionInput.checkInput(actionXml, existList, nonExistList, actionConf);
573            }
574    
575            if (isPullDepAvailable && isPushDepAvailable) {
576                // Check for latest/future
577                boolean isLatestFutureDepAvailable = coordActionInput.checkUnResolvedInput(actionXml, actionConf);
578                if (isLatestFutureDepAvailable) {
579                    String newActionXml = CoordActionInputCheckXCommand.resolveCoordConfiguration(actionXml, actionConf,
580                            actionBean.getId());
581                    actionXml.replace(0, actionXml.length(), newActionXml);
582                }
583            }
584    
585            return actionXml.toString();
586        }
587    
588        /**
589         * Materialize all <input-events>/<data-in> or <output-events>/<data-out>
590         * tags Create uris for resolved instances. Create unresolved instance for
591         * latest()/future().
592         *
593         * @param events
594         * @param appInst
595         * @param conf
596         * @throws Exception
597         */
598        public static Map<String, StringBuilder> materializeDataEvents(List<Element> events, SyncCoordAction appInst, Configuration conf
599                ) throws Exception {
600    
601            if (events == null) {
602                return null;
603            }
604            StringBuilder unresolvedList = new StringBuilder();
605            Map<String, StringBuilder> dependencyMap = new HashMap<String, StringBuilder>();
606            URIHandlerService uriService = Services.get().get(URIHandlerService.class);
607            StringBuilder pullMissingDep = null;
608            StringBuilder pushMissingDep = null;
609    
610            for (Element event : events) {
611                StringBuilder instances = new StringBuilder();
612                ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf);
613                // Handle list of instance tag
614                resolveInstances(event, instances, appInst, conf, eval);
615                // Handle start-instance and end-instance
616                resolveInstanceRange(event, instances, appInst, conf, eval);
617                // Separate out the unresolved instances
618                String resolvedList = separateResolvedAndUnresolved(event, instances);
619                if (!resolvedList.isEmpty()) {
620                    Element uri = event.getChild("dataset", event.getNamespace()).getChild("uri-template",
621                            event.getNamespace());
622                    String uriTemplate = uri.getText();
623                    URI baseURI = uriService.getAuthorityWithScheme(uriTemplate);
624                    URIHandler handler = uriService.getURIHandler(baseURI);
625                    if (handler.getDependencyType(baseURI).equals(DependencyType.PULL)) {
626                        pullMissingDep = (pullMissingDep == null) ? new StringBuilder(resolvedList) : pullMissingDep.append(
627                                CoordELFunctions.INSTANCE_SEPARATOR).append(resolvedList);
628                    }
629                    else {
630                        pushMissingDep = (pushMissingDep == null) ? new StringBuilder(resolvedList) : pushMissingDep.append(
631                                CoordELFunctions.INSTANCE_SEPARATOR).append(resolvedList);
632                    }
633                }
634    
635                String tmpUnresolved = event.getChildTextTrim(UNRESOLVED_INST_TAG, event.getNamespace());
636                if (tmpUnresolved != null) {
637                    if (unresolvedList.length() > 0) {
638                        unresolvedList.append(CoordELFunctions.INSTANCE_SEPARATOR);
639                    }
640                    unresolvedList.append(tmpUnresolved);
641                }
642            }
643            if (unresolvedList.length() > 0) {
644                if (pullMissingDep == null) {
645                    pullMissingDep = new StringBuilder();
646                }
647                pullMissingDep.append(RESOLVED_UNRESOLVED_SEPARATOR).append(unresolvedList);
648            }
649            dependencyMap.put(DependencyType.PULL.name(), pullMissingDep);
650            dependencyMap.put(DependencyType.PUSH.name(), pushMissingDep);
651            return dependencyMap;
652        }
653    
654        /**
655         * Get resolved string from missDepList
656         *
657         * @param missDepList
658         * @param resolved
659         * @param unresolved
660         * @return resolved string
661         */
662        public static String getResolvedList(String missDepList, StringBuilder resolved, StringBuilder unresolved) {
663            if (missDepList != null) {
664                int index = missDepList.indexOf(RESOLVED_UNRESOLVED_SEPARATOR);
665                if (index < 0) {
666                    resolved.append(missDepList);
667                }
668                else {
669                    resolved.append(missDepList.substring(0, index));
670                    unresolved.append(missDepList.substring(index + RESOLVED_UNRESOLVED_SEPARATOR.length()));
671                }
672            }
673            return resolved.toString();
674        }
675    
676        /**
677         * Get the next action time after a given time
678         *
679         * @param targetDate
680         * @param coordJob
681         * @return the next valid action time
682         */
683        public static Date getNextValidActionTimeForCronFrequency(Date targetDate, CoordinatorJobBean coordJob) throws ParseException {
684    
685            String freq = coordJob.getFrequency();
686            TimeZone tz = DateUtils.getOozieProcessingTimeZone();
687            String[] cronArray = freq.split(" ");
688            Date nextTime = null;
689    
690            // Current CronExpression doesn't support operations
691            // where both date of months and day of weeks are specified.
692            // As a result, we need to split this scenario into two cases
693            // and return the earlier time
694            if (!cronArray[2].trim().equals("?") && !cronArray[4].trim().equals("?")) {
695    
696                // When any one of day of month or day of week fields is a wildcard
697                // we need to replace the wildcard with "?"
698                if (cronArray[2].trim().equals("*") || cronArray[4].trim().equals("*")) {
699                    if (cronArray[2].trim().equals("*")) {
700                        cronArray[2] = "?";
701                    }
702                    else {
703                        cronArray[4] = "?";
704                    }
705                    freq= StringUtils.join(cronArray, " ");
706    
707                    // The cronExpression class takes second
708                    // as the first field where oozie is operating on
709                    // minute basis
710                    CronExpression expr = new CronExpression("0 " + freq);
711                    expr.setTimeZone(tz);
712                    nextTime = expr.getNextValidTimeAfter(targetDate);
713                }
714                // If both fields are specified by non-wildcards,
715                // we need to split it into two expressions
716                else {
717                    String[] cronArray1 = freq.split(" ");
718                    String[] cronArray2 = freq.split(" ");
719    
720                    cronArray1[2] = "?";
721                    cronArray2[4] = "?";
722    
723                    String freq1 = StringUtils.join(cronArray1, " ");
724                    String freq2 = StringUtils.join(cronArray2, " ");
725    
726                    // The cronExpression class takes second
727                    // as the first field where oozie is operating on
728                    // minute basis
729                    CronExpression expr1 = new CronExpression("0 " + freq1);
730                    expr1.setTimeZone(tz);
731                    CronExpression expr2 = new CronExpression("0 " + freq2);
732                    expr2.setTimeZone(tz);
733                    nextTime = expr1.getNextValidTimeAfter(targetDate);
734                    Date nextTime2 = expr2.getNextValidTimeAfter(targetDate);
735                    nextTime = nextTime.compareTo(nextTime2) < 0 ? nextTime: nextTime2;
736                }
737            }
738            else {
739                // The cronExpression class takes second
740                // as the first field where oozie is operating on
741                // minute basis
742                CronExpression expr  = new CronExpression("0 " + freq);
743                expr.setTimeZone(tz);
744                nextTime = expr.getNextValidTimeAfter(targetDate);
745            }
746    
747            return nextTime;
748        }
749    
750    }