001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.coord;
019    
020    import java.util.Calendar;
021    import java.util.Date;
022    import java.util.HashMap;
023    import java.util.Iterator;
024    import java.util.List;
025    import java.util.Map;
026    
027    import org.apache.hadoop.conf.Configuration;
028    import org.apache.oozie.CoordinatorActionBean;
029    import org.apache.oozie.command.coord.CoordCommandUtils;
030    import org.apache.oozie.service.ELService;
031    import org.apache.oozie.service.Services;
032    import org.apache.oozie.util.DateUtils;
033    import org.apache.oozie.util.ELEvaluator;
034    import org.apache.oozie.util.XmlUtils;
035    import org.jdom.Element;
036    
037    /**
038     * This class provide different evaluators required at different stages
039     */
040    public class CoordELEvaluator {
041        public static final Integer MINUTE = 1;
042        public static final Integer HOUR = 60 * MINUTE;
043    
044        /**
045         * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
046         * 1)
047         *
048         * @param conf : Configuration containing property variables
049         * @return configured ELEvaluator
050         */
051        public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
052            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
053            setConfigToEval(eval, conf);
054            return eval;
055        }
056    
057        /**
058         * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
059         *
060         * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
061         * @param appInst : Application Instance related information such as Action creation Time
062         * @param conf :Configuration to substitute any variables
063         * @return configured ELEvaluator
064         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
065         */
066        public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
067                throws Exception {
068            return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
069        }
070    
071        public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
072                                                             Configuration conf) throws Exception {
073            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
074            setConfigToEval(eval, conf);
075            SyncCoordDataset ds = getDSObject(event);
076            CoordELFunctions.configureEvaluator(eval, ds, appInst);
077            return eval;
078        }
079    
080        public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
081                                                               HashMap<String, String> dataNameList) throws Exception {
082            ELEvaluator eval = createELEvaluatorForGroup(conf, group);
083            for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
084                String key = it.next();
085                String value = dataNameList.get(key);
086                eval.setVariable("oozie.dataname." + key, value);
087            }
088            return eval;
089        }
090    
091        /**
092         * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
093         * should be resolved when all other data dependencies are met.
094         *
095         * @param actualTime : Action start time
096         * @param nominalTime : Action creation time
097         * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
098         * @param conf :Configuration to substitute any variables
099         * @return configured ELEvaluator
100         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
101         */
102        public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
103                throws Exception {
104            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
105            setConfigToEval(eval, conf);
106            SyncCoordDataset ds = getDSObject(dEvent);
107            SyncCoordAction appInst = new SyncCoordAction();
108            appInst.setNominalTime(nominalTime);
109            appInst.setActualTime(actualTime);
110            CoordELFunctions.configureEvaluator(eval, ds, appInst);
111            eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
112            return eval;
113        }
114    
115        /**
116         * Create a SLA evaluator to be used during Materialization
117         * @param eAction
118         * @param coordAction
119         * @param conf
120         * @return
121         * @throws Exception
122         */
123        public static ELEvaluator createSLAEvaluator(Element eAction, CoordinatorActionBean coordAction, Configuration conf)
124                throws Exception {
125            ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
126            setConfigToEval(eval, conf);
127            SyncCoordAction appInst = new SyncCoordAction();// TODO:
128            appInst.setNominalTime(coordAction.getNominalTime());
129            appInst.setActualTime(coordAction.getCreatedTime());
130            appInst.setActionId(coordAction.getId());
131            appInst.setName(eAction.getAttributeValue("name"));
132            CoordELFunctions.configureEvaluator(eval, null, appInst);
133    
134            Element events = eAction.getChild("output-events", eAction.getNamespace());
135            if (events != null) {
136                for (Object obj : events.getChildren("data-out", eAction.getNamespace())) {
137                    Element data = (Element) obj;
138                    if (data.getChild("uris", data.getNamespace()) != null) {
139                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
140                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
141                        eval.setVariable(".dataout." + data.getAttributeValue("name"), uris);
142                    }
143                    if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
144                        eval.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true");
145                    }
146                }
147            }
148            return eval;
149        }
150    
151        /**
152         * Create an Evaluator using conf and input/output-data (used for sla)
153         * @param conf
154         * @param group
155         * @param dataNameList
156         * @return
157         * @throws Exception
158         */
159        public static ELEvaluator createELEvaluatorForDataAndConf(Configuration conf, String group,
160                HashMap<String, String> dataNameList) throws Exception {
161            ELEvaluator eval = createELEvaluatorForDataEcho(conf, group, dataNameList);
162            setConfigToEval(eval, conf);
163            return eval;
164        }
165    
166        /**
167         * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
168         *
169         * @param eJob : XML element for the application instance
170         * @param conf :Configuration to substitute any variables
171         * @return configured ELEvaluator
172         * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
173         */
174        public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
175            ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
176            setConfigToEval(e, conf);
177            SyncCoordAction appInst = new SyncCoordAction();
178            String strNominalTime = eJob.getAttributeValue("action-nominal-time");
179            if (strNominalTime != null) {
180                appInst.setNominalTime(DateUtils.parseDateOozieTZ(strNominalTime));
181                appInst.setTimeZone(DateUtils.getTimeZone(eJob.getAttributeValue("timezone")));
182                appInst.setFrequency(eJob.getAttributeValue("frequency"));
183                appInst.setTimeUnit(TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit")));
184                appInst.setActionId(actionId);
185                appInst.setName(eJob.getAttributeValue("name"));
186            }
187            String strActualTime = eJob.getAttributeValue("action-actual-time");
188            if (strActualTime != null) {
189                appInst.setActualTime(DateUtils.parseDateOozieTZ(strActualTime));
190            }
191            CoordELFunctions.configureEvaluator(e, null, appInst);
192            Element events = eJob.getChild("input-events", eJob.getNamespace());
193            if (events != null) {
194                for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
195                    if (data.getChild("uris", data.getNamespace()) != null) {
196                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
197                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
198                        e.setVariable(".datain." + data.getAttributeValue("name"), uris);
199                    }
200                    else {
201                    }
202                    if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
203                        e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
204                        // check
205                        // null
206                    }
207                }
208            }
209            events = eJob.getChild("output-events", eJob.getNamespace());
210            if (events != null) {
211                for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
212                    if (data.getChild("uris", data.getNamespace()) != null) {
213                        String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
214                        uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
215                        e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
216                    }
217                    else {
218                    }// TODO
219                    if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
220                        e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
221                        // check
222                        // null
223                    }
224                }
225            }
226            return e;
227        }
228    
229        /**
230         * Create a new Evaluator to resolve URI temple with time specific constant
231         *
232         * @param strDate : Date-time
233         * @return configured ELEvaluator
234         * @throws Exception If there is any date-time string in wrong format, the exception is thrown
235         */
236        public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
237            ELEvaluator eval = new ELEvaluator();
238            Calendar date = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
239            // always???
240            date.setTime(DateUtils.parseDateOozieTZ(strDate));
241            eval.setVariable("YEAR", date.get(Calendar.YEAR));
242            eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
243            eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
244            eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
245            eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
246            return eval;
247        }
248    
249        /**
250         * Create Dataset object using the Dataset XML information
251         *
252         * @param eData
253         * @return
254         * @throws Exception
255         */
256        private static SyncCoordDataset getDSObject(Element eData) throws Exception {
257            SyncCoordDataset ds = new SyncCoordDataset();
258            Element eDataset = eData.getChild("dataset", eData.getNamespace());
259            // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
260            Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance"));
261            ds.setInitInstance(initInstance);
262            if (eDataset.getAttributeValue("frequency") != null) {
263                int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
264                ds.setFrequency(frequency);
265                ds.setType("SYNC");
266                if (eDataset.getAttributeValue("freq_timeunit") == null) {
267                    throw new RuntimeException("No freq_timeunit defined in data set definition\n"
268                            + XmlUtils.prettyPrint(eDataset));
269                }
270                ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
271                if (eDataset.getAttributeValue("timezone") == null) {
272                    throw new RuntimeException("No timezone defined in data set definition\n"
273                            + XmlUtils.prettyPrint(eDataset));
274                }
275                ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
276                if (eDataset.getAttributeValue("end_of_duration") == null) {
277                    throw new RuntimeException("No end_of_duration defined in data set definition\n"
278                            + XmlUtils.prettyPrint(eDataset));
279                }
280                ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
281    
282                Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
283                String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
284                ds.setDoneFlag(doneFlag);
285            }
286            else {
287                ds.setType("ASYNC");
288            }
289            String name = eDataset.getAttributeValue("name");
290            ds.setName(name);
291            // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
292            // eData.getNamespace()));
293            String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
294            ds.setUriTemplate(uriTemplate);
295            // ds.setTimeUnit(TimeUnit.MINUTES);
296            return ds;
297        }
298    
299        /**
300         * Set all job configurations properties into evaluator.
301         *
302         * @param eval : Evaluator to set variables
303         * @param conf : configurations to set Evaluator
304         */
305        private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
306            for (Map.Entry<String, String> entry : conf) {
307                eval.setVariable(entry.getKey(), entry.getValue().trim());
308            }
309        }
310    
311        /**
312         * make any one digit number to two digit string pre-appending a"0"
313         *
314         * @param num : number to make sting
315         * @return :String of length at least two digit.
316         */
317        private static String make2Digits(int num) {
318            String ret = "" + num;
319            if (num <= 9) {
320                ret = "0" + ret;
321            }
322            return ret;
323        }
324    }