001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.coord;
019
020 import java.util.Calendar;
021 import java.util.Date;
022 import java.util.HashMap;
023 import java.util.Iterator;
024 import java.util.List;
025 import java.util.Map;
026
027 import org.apache.hadoop.conf.Configuration;
028 import org.apache.oozie.CoordinatorActionBean;
029 import org.apache.oozie.command.coord.CoordCommandUtils;
030 import org.apache.oozie.service.ELService;
031 import org.apache.oozie.service.Services;
032 import org.apache.oozie.util.DateUtils;
033 import org.apache.oozie.util.ELEvaluator;
034 import org.apache.oozie.util.XmlUtils;
035 import org.jdom.Element;
036
037 /**
038 * This class provide different evaluators required at different stages
039 */
040 public class CoordELEvaluator {
041 public static final Integer MINUTE = 1;
042 public static final Integer HOUR = 60 * MINUTE;
043
044 /**
045 * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
046 * 1)
047 *
048 * @param conf : Configuration containing property variables
049 * @return configured ELEvaluator
050 */
051 public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
052 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
053 setConfigToEval(eval, conf);
054 return eval;
055 }
056
057 /**
058 * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
059 *
060 * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
061 * @param appInst : Application Instance related information such as Action creation Time
062 * @param conf :Configuration to substitute any variables
063 * @return configured ELEvaluator
064 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
065 */
066 public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
067 throws Exception {
068 return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
069 }
070
071 public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
072 Configuration conf) throws Exception {
073 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
074 setConfigToEval(eval, conf);
075 SyncCoordDataset ds = getDSObject(event);
076 CoordELFunctions.configureEvaluator(eval, ds, appInst);
077 return eval;
078 }
079
080 public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
081 HashMap<String, String> dataNameList) throws Exception {
082 ELEvaluator eval = createELEvaluatorForGroup(conf, group);
083 for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
084 String key = it.next();
085 String value = dataNameList.get(key);
086 eval.setVariable("oozie.dataname." + key, value);
087 }
088 return eval;
089 }
090
091 /**
092 * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
093 * should be resolved when all other data dependencies are met.
094 *
095 * @param actualTime : Action start time
096 * @param nominalTime : Action creation time
097 * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
098 * @param conf :Configuration to substitute any variables
099 * @return configured ELEvaluator
100 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
101 */
102 public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
103 throws Exception {
104 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
105 setConfigToEval(eval, conf);
106 SyncCoordDataset ds = getDSObject(dEvent);
107 SyncCoordAction appInst = new SyncCoordAction();
108 appInst.setNominalTime(nominalTime);
109 appInst.setActualTime(actualTime);
110 CoordELFunctions.configureEvaluator(eval, ds, appInst);
111 eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
112 return eval;
113 }
114
115 /**
116 * Create a SLA evaluator to be used during Materialization
117 * @param eAction
118 * @param coordAction
119 * @param conf
120 * @return
121 * @throws Exception
122 */
123 public static ELEvaluator createSLAEvaluator(Element eAction, CoordinatorActionBean coordAction, Configuration conf)
124 throws Exception {
125 ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
126 setConfigToEval(eval, conf);
127 SyncCoordAction appInst = new SyncCoordAction();// TODO:
128 appInst.setNominalTime(coordAction.getNominalTime());
129 appInst.setActualTime(coordAction.getCreatedTime());
130 appInst.setActionId(coordAction.getId());
131 appInst.setName(eAction.getAttributeValue("name"));
132 CoordELFunctions.configureEvaluator(eval, null, appInst);
133
134 Element events = eAction.getChild("output-events", eAction.getNamespace());
135 if (events != null) {
136 for (Object obj : events.getChildren("data-out", eAction.getNamespace())) {
137 Element data = (Element) obj;
138 if (data.getChild("uris", data.getNamespace()) != null) {
139 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
140 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
141 eval.setVariable(".dataout." + data.getAttributeValue("name"), uris);
142 }
143 if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
144 eval.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true");
145 }
146 }
147 }
148 return eval;
149 }
150
151 /**
152 * Create an Evaluator using conf and input/output-data (used for sla)
153 * @param conf
154 * @param group
155 * @param dataNameList
156 * @return
157 * @throws Exception
158 */
159 public static ELEvaluator createELEvaluatorForDataAndConf(Configuration conf, String group,
160 HashMap<String, String> dataNameList) throws Exception {
161 ELEvaluator eval = createELEvaluatorForDataEcho(conf, group, dataNameList);
162 setConfigToEval(eval, conf);
163 return eval;
164 }
165
166 /**
167 * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
168 *
169 * @param eJob : XML element for the application instance
170 * @param conf :Configuration to substitute any variables
171 * @return configured ELEvaluator
172 * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
173 */
174 public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
175 ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
176 setConfigToEval(e, conf);
177 SyncCoordAction appInst = new SyncCoordAction();
178 String strNominalTime = eJob.getAttributeValue("action-nominal-time");
179 if (strNominalTime != null) {
180 appInst.setNominalTime(DateUtils.parseDateOozieTZ(strNominalTime));
181 appInst.setTimeZone(DateUtils.getTimeZone(eJob.getAttributeValue("timezone")));
182 appInst.setFrequency(eJob.getAttributeValue("frequency"));
183 appInst.setTimeUnit(TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit")));
184 appInst.setActionId(actionId);
185 appInst.setName(eJob.getAttributeValue("name"));
186 }
187 String strActualTime = eJob.getAttributeValue("action-actual-time");
188 if (strActualTime != null) {
189 appInst.setActualTime(DateUtils.parseDateOozieTZ(strActualTime));
190 }
191 CoordELFunctions.configureEvaluator(e, null, appInst);
192 Element events = eJob.getChild("input-events", eJob.getNamespace());
193 if (events != null) {
194 for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
195 if (data.getChild("uris", data.getNamespace()) != null) {
196 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
197 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
198 e.setVariable(".datain." + data.getAttributeValue("name"), uris);
199 }
200 else {
201 }
202 if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
203 e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
204 // check
205 // null
206 }
207 }
208 }
209 events = eJob.getChild("output-events", eJob.getNamespace());
210 if (events != null) {
211 for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
212 if (data.getChild("uris", data.getNamespace()) != null) {
213 String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
214 uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
215 e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
216 }
217 else {
218 }// TODO
219 if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
220 e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
221 // check
222 // null
223 }
224 }
225 }
226 return e;
227 }
228
229 /**
230 * Create a new Evaluator to resolve URI temple with time specific constant
231 *
232 * @param strDate : Date-time
233 * @return configured ELEvaluator
234 * @throws Exception If there is any date-time string in wrong format, the exception is thrown
235 */
236 public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
237 ELEvaluator eval = new ELEvaluator();
238 Calendar date = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
239 // always???
240 date.setTime(DateUtils.parseDateOozieTZ(strDate));
241 eval.setVariable("YEAR", date.get(Calendar.YEAR));
242 eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
243 eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
244 eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
245 eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
246 return eval;
247 }
248
249 /**
250 * Create Dataset object using the Dataset XML information
251 *
252 * @param eData
253 * @return
254 * @throws Exception
255 */
256 private static SyncCoordDataset getDSObject(Element eData) throws Exception {
257 SyncCoordDataset ds = new SyncCoordDataset();
258 Element eDataset = eData.getChild("dataset", eData.getNamespace());
259 // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
260 Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance"));
261 ds.setInitInstance(initInstance);
262 if (eDataset.getAttributeValue("frequency") != null) {
263 int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
264 ds.setFrequency(frequency);
265 ds.setType("SYNC");
266 if (eDataset.getAttributeValue("freq_timeunit") == null) {
267 throw new RuntimeException("No freq_timeunit defined in data set definition\n"
268 + XmlUtils.prettyPrint(eDataset));
269 }
270 ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
271 if (eDataset.getAttributeValue("timezone") == null) {
272 throw new RuntimeException("No timezone defined in data set definition\n"
273 + XmlUtils.prettyPrint(eDataset));
274 }
275 ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
276 if (eDataset.getAttributeValue("end_of_duration") == null) {
277 throw new RuntimeException("No end_of_duration defined in data set definition\n"
278 + XmlUtils.prettyPrint(eDataset));
279 }
280 ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
281
282 Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
283 String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
284 ds.setDoneFlag(doneFlag);
285 }
286 else {
287 ds.setType("ASYNC");
288 }
289 String name = eDataset.getAttributeValue("name");
290 ds.setName(name);
291 // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
292 // eData.getNamespace()));
293 String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
294 ds.setUriTemplate(uriTemplate);
295 // ds.setTimeUnit(TimeUnit.MINUTES);
296 return ds;
297 }
298
299 /**
300 * Set all job configurations properties into evaluator.
301 *
302 * @param eval : Evaluator to set variables
303 * @param conf : configurations to set Evaluator
304 */
305 private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
306 for (Map.Entry<String, String> entry : conf) {
307 eval.setVariable(entry.getKey(), entry.getValue().trim());
308 }
309 }
310
311 /**
312 * make any one digit number to two digit string pre-appending a"0"
313 *
314 * @param num : number to make sting
315 * @return :String of length at least two digit.
316 */
317 private static String make2Digits(int num) {
318 String ret = "" + num;
319 if (num <= 9) {
320 ret = "0" + ret;
321 }
322 return ret;
323 }
324 }