001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.coord;
019
020 import com.google.common.collect.Lists;
021 import org.apache.commons.lang.StringUtils;
022 import org.apache.hadoop.conf.Configuration;
023 import org.apache.oozie.ErrorCode;
024 import org.apache.oozie.client.OozieClient;
025 import org.apache.oozie.command.CommandException;
026 import org.apache.oozie.dependency.URIHandler;
027 import org.apache.oozie.dependency.URIHandler.Context;
028 import org.apache.oozie.service.Services;
029 import org.apache.oozie.service.URIHandlerService;
030 import org.apache.oozie.util.DateUtils;
031 import org.apache.oozie.util.ELEvaluator;
032 import org.apache.oozie.util.ParamChecker;
033 import org.apache.oozie.util.XLog;
034
035 import java.net.URI;
036 import java.util.ArrayList;
037 import java.util.Calendar;
038 import java.util.Date;
039 import java.util.GregorianCalendar;
040 import java.util.List;
041 import java.util.TimeZone;
042
043 /**
044 * This class implements the EL function related to coordinator
045 */
046
047 public class CoordELFunctions {
048 final public static String DATASET = "oozie.coord.el.dataset.bean";
049 final public static String COORD_ACTION = "oozie.coord.el.app.bean";
050 final public static String CONFIGURATION = "oozie.coord.el.conf";
051 final public static String LATEST_EL_USE_CURRENT_TIME = "oozie.service.ELService.latest-el.use-current-time";
052 // INSTANCE_SEPARATOR is used to separate multiple directories into one tag.
053 final public static String INSTANCE_SEPARATOR = "#";
054 final public static String DIR_SEPARATOR = ",";
055 // TODO: in next release, support flexibility
056 private static String END_OF_OPERATION_INDICATOR_FILE = "_SUCCESS";
057
058 public static final long MINUTE_MSEC = 60 * 1000L;
059 public static final long HOUR_MSEC = 60 * MINUTE_MSEC;
060 public static final long DAY_MSEC = 24 * HOUR_MSEC;
061 public static final long MONTH_MSEC = 30 * DAY_MSEC;
062 public static final long YEAR_MSEC = 365 * DAY_MSEC;
063
064 /**
065 * Used in defining the frequency in 'day' unit. <p/> domain: <code> val > 0</code> and should be integer.
066 *
067 * @param val frequency in number of days.
068 * @return number of days and also set the frequency timeunit to "day"
069 */
070 public static int ph1_coord_days(int val) {
071 val = ParamChecker.checkGTZero(val, "n");
072 ELEvaluator eval = ELEvaluator.getCurrent();
073 eval.setVariable("timeunit", TimeUnit.DAY);
074 eval.setVariable("endOfDuration", TimeUnit.NONE);
075 return val;
076 }
077
078 /**
079 * Used in defining the frequency in 'month' unit. <p/> domain: <code> val > 0</code> and should be integer.
080 *
081 * @param val frequency in number of months.
082 * @return number of months and also set the frequency timeunit to "month"
083 */
084 public static int ph1_coord_months(int val) {
085 val = ParamChecker.checkGTZero(val, "n");
086 ELEvaluator eval = ELEvaluator.getCurrent();
087 eval.setVariable("timeunit", TimeUnit.MONTH);
088 eval.setVariable("endOfDuration", TimeUnit.NONE);
089 return val;
090 }
091
092 /**
093 * Used in defining the frequency in 'hour' unit. <p/> parameter value domain: <code> val > 0</code> and should
094 * be integer.
095 *
096 * @param val frequency in number of hours.
097 * @return number of minutes and also set the frequency timeunit to "minute"
098 */
099 public static int ph1_coord_hours(int val) {
100 val = ParamChecker.checkGTZero(val, "n");
101 ELEvaluator eval = ELEvaluator.getCurrent();
102 eval.setVariable("timeunit", TimeUnit.MINUTE);
103 eval.setVariable("endOfDuration", TimeUnit.NONE);
104 return val * 60;
105 }
106
107 /**
108 * Used in defining the frequency in 'minute' unit. <p/> domain: <code> val > 0</code> and should be integer.
109 *
110 * @param val frequency in number of minutes.
111 * @return number of minutes and also set the frequency timeunit to "minute"
112 */
113 public static int ph1_coord_minutes(int val) {
114 val = ParamChecker.checkGTZero(val, "n");
115 ELEvaluator eval = ELEvaluator.getCurrent();
116 eval.setVariable("timeunit", TimeUnit.MINUTE);
117 eval.setVariable("endOfDuration", TimeUnit.NONE);
118 return val;
119 }
120
121 /**
122 * Used in defining the frequency in 'day' unit and specify the "end of day" property. <p/> Every instance will
123 * start at 00:00 hour of each day. <p/> domain: <code> val > 0</code> and should be integer.
124 *
125 * @param val frequency in number of days.
126 * @return number of days and also set the frequency timeunit to "day" and end_of_duration flag to "day"
127 */
128 public static int ph1_coord_endOfDays(int val) {
129 val = ParamChecker.checkGTZero(val, "n");
130 ELEvaluator eval = ELEvaluator.getCurrent();
131 eval.setVariable("timeunit", TimeUnit.DAY);
132 eval.setVariable("endOfDuration", TimeUnit.END_OF_DAY);
133 return val;
134 }
135
136 /**
137 * Used in defining the frequency in 'month' unit and specify the "end of month" property. <p/> Every instance will
138 * start at first day of each month at 00:00 hour. <p/> domain: <code> val > 0</code> and should be integer.
139 *
140 * @param val: frequency in number of months.
141 * @return number of months and also set the frequency timeunit to "month" and end_of_duration flag to "month"
142 */
143 public static int ph1_coord_endOfMonths(int val) {
144 val = ParamChecker.checkGTZero(val, "n");
145 ELEvaluator eval = ELEvaluator.getCurrent();
146 eval.setVariable("timeunit", TimeUnit.MONTH);
147 eval.setVariable("endOfDuration", TimeUnit.END_OF_MONTH);
148 return val;
149 }
150
151 /**
152 * Calculate the difference of timezone offset in minutes between dataset and coordinator job. <p/> Depends on: <p/>
153 * 1. Timezone of both dataset and job <p/> 2. Action creation Time
154 *
155 * @return difference in minutes (DataSet TZ Offset - Application TZ offset)
156 */
157 public static int ph2_coord_tzOffset() {
158 long actionCreationTime = getActionCreationtime().getTime();
159 TimeZone dsTZ = ParamChecker.notNull(getDatasetTZ(), "DatasetTZ");
160 TimeZone jobTZ = ParamChecker.notNull(getJobTZ(), "JobTZ");
161 return (dsTZ.getOffset(actionCreationTime) - jobTZ.getOffset(actionCreationTime)) / (1000 * 60);
162 }
163
164 public static int ph3_coord_tzOffset() {
165 return ph2_coord_tzOffset();
166 }
167
168 /**
169 * Returns the a date string while given a base date in 'strBaseDate',
170 * offset and unit (e.g. DAY, MONTH, HOUR, MINUTE, MONTH).
171 *
172 * @param strBaseDate -- base date
173 * @param offset -- any number
174 * @param unit -- DAY, MONTH, HOUR, MINUTE, MONTH
175 * @return date string
176 * @throws Exception
177 */
178 public static String ph2_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception {
179 Calendar baseCalDate = DateUtils.getCalendar(strBaseDate);
180 StringBuilder buffer = new StringBuilder();
181 baseCalDate.add(TimeUnit.valueOf(unit).getCalendarUnit(), offset);
182 buffer.append(DateUtils.formatDateOozieTZ(baseCalDate));
183 return buffer.toString();
184 }
185
186 public static String ph3_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception {
187 return ph2_coord_dateOffset(strBaseDate, offset, unit);
188 }
189
190 /**
191 * Determine the date-time in Oozie processing timezone of n-th future available dataset instance
192 * from nominal Time but not beyond the instance specified as 'instance.
193 * <p/>
194 * It depends on:
195 * <p/>
196 * 1. Data set frequency
197 * <p/>
198 * 2. Data set Time unit (day, month, minute)
199 * <p/>
200 * 3. Data set Time zone/DST
201 * <p/>
202 * 4. End Day/Month flag
203 * <p/>
204 * 5. Data set initial instance
205 * <p/>
206 * 6. Action Creation Time
207 * <p/>
208 * 7. Existence of dataset's directory
209 *
210 * @param n :instance count
211 * <p/>
212 * domain: n >= 0, n is integer
213 * @param instance: How many future instance it should check? value should
214 * be >=0
215 * @return date-time in Oozie processing timezone of the n-th instance
216 * <p/>
217 * @throws Exception
218 */
219 public static String ph3_coord_future(int n, int instance) throws Exception {
220 ParamChecker.checkGEZero(n, "future:n");
221 ParamChecker.checkGTZero(instance, "future:instance");
222 if (isSyncDataSet()) {// For Sync Dataset
223 return coord_future_sync(n, instance);
224 }
225 else {
226 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
227 }
228 }
229
230 /**
231 * Determine the date-time in Oozie processing timezone of the future available dataset instances
232 * from start to end offsets from nominal Time but not beyond the instance specified as 'instance'.
233 * <p/>
234 * It depends on:
235 * <p/>
236 * 1. Data set frequency
237 * <p/>
238 * 2. Data set Time unit (day, month, minute)
239 * <p/>
240 * 3. Data set Time zone/DST
241 * <p/>
242 * 4. End Day/Month flag
243 * <p/>
244 * 5. Data set initial instance
245 * <p/>
246 * 6. Action Creation Time
247 * <p/>
248 * 7. Existence of dataset's directory
249 *
250 * @param start : start instance offset
251 * <p/>
252 * domain: start >= 0, start is integer
253 * @param end : end instance offset
254 * <p/>
255 * domain: end >= 0, end is integer
256 * @param instance: How many future instance it should check? value should
257 * be >=0
258 * @return date-time in Oozie processing timezone of the instances from start to end offsets
259 * delimited by comma.
260 * <p/>
261 * @throws Exception
262 */
263 public static String ph3_coord_futureRange(int start, int end, int instance) throws Exception {
264 ParamChecker.checkGEZero(start, "future:n");
265 ParamChecker.checkGEZero(end, "future:n");
266 ParamChecker.checkGTZero(instance, "future:instance");
267 if (isSyncDataSet()) {// For Sync Dataset
268 return coord_futureRange_sync(start, end, instance);
269 }
270 else {
271 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
272 }
273 }
274
275 private static String coord_future_sync(int n, int instance) throws Exception {
276 return coord_futureRange_sync(n, n, instance);
277 }
278
279 private static String coord_futureRange_sync(int startOffset, int endOffset, int instance) throws Exception {
280 final XLog LOG = XLog.getLog(CoordELFunctions.class);
281 final Thread currentThread = Thread.currentThread();
282 ELEvaluator eval = ELEvaluator.getCurrent();
283 String retVal = "";
284 int datasetFrequency = (int) getDSFrequency();// in minutes
285 TimeUnit dsTimeUnit = getDSTimeUnit();
286 int[] instCount = new int[1];
287 Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
288 StringBuilder resolvedInstances = new StringBuilder();
289 StringBuilder resolvedURIPaths = new StringBuilder();
290 if (nominalInstanceCal != null) {
291 Calendar initInstance = getInitialInstanceCal();
292 nominalInstanceCal = (Calendar) initInstance.clone();
293 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
294
295 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
296 if (ds == null) {
297 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
298 }
299 String uriTemplate = ds.getUriTemplate();
300 Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
301 if (conf == null) {
302 throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
303 }
304 int available = 0, checkedInstance = 0;
305 boolean resolved = false;
306 String user = ParamChecker
307 .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
308 String doneFlag = ds.getDoneFlag();
309 URIHandlerService uriService = Services.get().get(URIHandlerService.class);
310 URIHandler uriHandler = null;
311 Context uriContext = null;
312 try {
313 while (instance >= checkedInstance && !currentThread.isInterrupted()) {
314 ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
315 String uriPath = uriEval.evaluate(uriTemplate, String.class);
316 if (uriHandler == null) {
317 URI uri = new URI(uriPath);
318 uriHandler = uriService.getURIHandler(uri);
319 uriContext = uriHandler.getContext(uri, conf, user);
320 }
321 String uriWithDoneFlag = uriHandler.getURIWithDoneFlag(uriPath, doneFlag);
322 if (uriHandler.exists(new URI(uriWithDoneFlag), uriContext)) {
323 if (available == endOffset) {
324 LOG.debug("Matched future(" + available + "): " + uriWithDoneFlag);
325 resolved = true;
326 resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal));
327 resolvedURIPaths.append(uriPath);
328 retVal = resolvedInstances.toString();
329 eval.setVariable("resolved_path", resolvedURIPaths.toString());
330 break;
331 }
332 else if (available >= startOffset) {
333 LOG.debug("Matched future(" + available + "): " + uriWithDoneFlag);
334 resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal)).append(
335 INSTANCE_SEPARATOR);
336 resolvedURIPaths.append(uriPath).append(INSTANCE_SEPARATOR);
337 }
338 available++;
339 }
340 // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency);
341 nominalInstanceCal = (Calendar) initInstance.clone();
342 instCount[0]++;
343 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
344 checkedInstance++;
345 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
346 }
347 }
348 finally {
349 if (uriContext != null) {
350 uriContext.destroy();
351 }
352 }
353 if (!resolved) {
354 // return unchanged future function with variable 'is_resolved'
355 // to 'false'
356 eval.setVariable("is_resolved", Boolean.FALSE);
357 if (startOffset == endOffset) {
358 retVal = "${coord:future(" + startOffset + ", " + instance + ")}";
359 }
360 else {
361 retVal = "${coord:futureRange(" + startOffset + ", " + endOffset + ", " + instance + ")}";
362 }
363 }
364 else {
365 eval.setVariable("is_resolved", Boolean.TRUE);
366 }
367 }
368 else {// No feasible nominal time
369 eval.setVariable("is_resolved", Boolean.TRUE);
370 retVal = "";
371 }
372 return retVal;
373 }
374
375 /**
376 * Return nominal time or Action Creation Time.
377 * <p/>
378 *
379 * @return coordinator action creation or materialization date time
380 * @throws Exception if unable to format the Date object to String
381 */
382 public static String ph2_coord_nominalTime() throws Exception {
383 ELEvaluator eval = ELEvaluator.getCurrent();
384 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
385 "Coordinator Action");
386 return DateUtils.formatDateOozieTZ(action.getNominalTime());
387 }
388
389 public static String ph3_coord_nominalTime() throws Exception {
390 return ph2_coord_nominalTime();
391 }
392
393 /**
394 * Convert from standard date-time formatting to a desired format.
395 * <p/>
396 * @param dateTimeStr - A timestamp in standard (ISO8601) format.
397 * @param format - A string representing the desired format.
398 * @return coordinator action creation or materialization date time
399 * @throws Exception if unable to format the Date object to String
400 */
401 public static String ph2_coord_formatTime(String dateTimeStr, String format)
402 throws Exception {
403 Date dateTime = DateUtils.parseDateOozieTZ(dateTimeStr);
404 return DateUtils.formatDateCustom(dateTime, format);
405 }
406
407 public static String ph3_coord_formatTime(String dateTimeStr, String format)
408 throws Exception {
409 return ph2_coord_formatTime(dateTimeStr, format);
410 }
411
412 /**
413 * Return Action Id. <p/>
414 *
415 * @return coordinator action Id
416 */
417 public static String ph2_coord_actionId() throws Exception {
418 ELEvaluator eval = ELEvaluator.getCurrent();
419 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
420 "Coordinator Action");
421 return action.getActionId();
422 }
423
424 public static String ph3_coord_actionId() throws Exception {
425 return ph2_coord_actionId();
426 }
427
428 /**
429 * Return Job Name. <p/>
430 *
431 * @return coordinator name
432 */
433 public static String ph2_coord_name() throws Exception {
434 ELEvaluator eval = ELEvaluator.getCurrent();
435 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
436 "Coordinator Action");
437 return action.getName();
438 }
439
440 public static String ph3_coord_name() throws Exception {
441 return ph2_coord_name();
442 }
443
444 /**
445 * Return Action Start time. <p/>
446 *
447 * @return coordinator action start time
448 * @throws Exception if unable to format the Date object to String
449 */
450 public static String ph2_coord_actualTime() throws Exception {
451 ELEvaluator eval = ELEvaluator.getCurrent();
452 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
453 if (coordAction == null) {
454 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
455 }
456 return DateUtils.formatDateOozieTZ(coordAction.getActualTime());
457 }
458
459 public static String ph3_coord_actualTime() throws Exception {
460 return ph2_coord_actualTime();
461 }
462
463 /**
464 * Used to specify a list of URI's that are used as input dir to the workflow job. <p/> Look for two evaluator-level
465 * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the current list of
466 * URI. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something
467 * unresolved, this function will echo back the original function <p/> otherwise it sends the uris.
468 *
469 * @param dataInName : Datain name
470 * @return the list of URI's separated by INSTANCE_SEPARATOR <p/> if there are unresolved EL function (i.e. latest)
471 * , echo back <p/> the function without resolving the function.
472 */
473 public static String ph3_coord_dataIn(String dataInName) {
474 String uris = "";
475 ELEvaluator eval = ELEvaluator.getCurrent();
476 uris = (String) eval.getVariable(".datain." + dataInName);
477 Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
478 if (unresolved != null && unresolved.booleanValue() == true) {
479 return "${coord:dataIn('" + dataInName + "')}";
480 }
481 return uris;
482 }
483
484 /**
485 * Used to specify a list of URI's that are output dir of the workflow job. <p/> Look for one evaluator-level
486 * variable <p/> dataout.<DATAOUT_NAME> <p/> It defines the current list of URI. <p/> otherwise it sends the uris.
487 *
488 * @param dataOutName : Dataout name
489 * @return the list of URI's separated by INSTANCE_SEPARATOR
490 */
491 public static String ph3_coord_dataOut(String dataOutName) {
492 String uris = "";
493 ELEvaluator eval = ELEvaluator.getCurrent();
494 uris = (String) eval.getVariable(".dataout." + dataOutName);
495 return uris;
496 }
497
498 /**
499 * Determine the date-time in Oozie processing timezone of n-th dataset instance. <p/> It depends on: <p/> 1.
500 * Data set frequency <p/> 2.
501 * Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. Data
502 * set initial instance <p/> 6. Action Creation Time
503 *
504 * @param n instance count domain: n is integer
505 * @return date-time in Oozie processing timezone of the n-th instance returns 'null' means n-th instance is
506 * earlier than Initial-Instance of DS
507 * @throws Exception
508 */
509 public static String ph2_coord_current(int n) throws Exception {
510 if (isSyncDataSet()) { // For Sync Dataset
511 return coord_current_sync(n);
512 }
513 else {
514 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
515 }
516 }
517
518 /**
519 * Determine the date-time in Oozie processing timezone of current dataset instances
520 * from start to end offsets from the nominal time. <p/> It depends
521 * on: <p/> 1. Data set frequency <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST
522 * <p/> 4. End Day/Month flag <p/> 5. Data set initial instance <p/> 6. Action Creation Time
523 *
524 * @param start :start instance offset <p/> domain: start <= 0, start is integer
525 * @param end :end instance offset <p/> domain: end <= 0, end is integer
526 * @return date-time in Oozie processing timezone of the instances from start to end offsets
527 * delimited by comma. <p/> If the current instance time of the dataset based on the Action Creation Time
528 * is earlier than the Initial-Instance of DS an empty string is returned.
529 * If an instance within the range is earlier than Initial-Instance of DS that instance is ignored
530 * @throws Exception
531 */
532 public static String ph2_coord_currentRange(int start, int end) throws Exception {
533 if (isSyncDataSet()) { // For Sync Dataset
534 return coord_currentRange_sync(start, end);
535 }
536 else {
537 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
538 }
539 }
540 /**
541 * Determine the date-time in Oozie processing timezone of the given offset from the dataset effective nominal time. <p/> It
542 * depends on: <p> 1. Data set frequency <p/> 2. Data set Time Unit <p/> 3. Data set Time zone/DST
543 * <p/> 4. Data set initial instance <p/> 5. Action Creation Time
544 *
545 * @param n offset amount (integer)
546 * @param timeUnit TimeUnit for offset n ("MINUTE", "HOUR", "DAY", "MONTH", "YEAR")
547 * @return date-time in Oozie processing timezone of the given offset from the dataset effective nominal time
548 * @throws Exception if there was a problem formatting
549 */
550 public static String ph2_coord_offset(int n, String timeUnit) throws Exception {
551 if (isSyncDataSet()) { // For Sync Dataset
552 return coord_offset_sync(n, timeUnit);
553 }
554 else {
555 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
556 }
557 }
558
559 /**
560 * Determine how many hours is on the date of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency
561 * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5.
562 * Data set initial instance <p/> 6. Action Creation Time
563 *
564 * @param n instance count <p/> domain: n is integer
565 * @return number of hours on that day <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS
566 * @throws Exception
567 */
568 public static int ph2_coord_hoursInDay(int n) throws Exception {
569 int datasetFrequency = (int) getDSFrequency();
570 // /Calendar nominalInstanceCal =
571 // getCurrentInstance(getActionCreationtime());
572 Calendar nominalInstanceCal = getEffectiveNominalTime();
573 if (nominalInstanceCal == null) {
574 return -1;
575 }
576 nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n);
577 /*
578 * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0)
579 * { return -1; }
580 */
581 nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ
582 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
583 return DateUtils.hoursInDay(nominalInstanceCal);
584 }
585
586 public static int ph3_coord_hoursInDay(int n) throws Exception {
587 return ph2_coord_hoursInDay(n);
588 }
589
590 /**
591 * Calculate number of days in one month for n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency .
592 * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5.
593 * Data set initial instance <p/> 6. Action Creation Time
594 *
595 * @param n instance count. domain: n is integer
596 * @return number of days in that month <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS
597 * @throws Exception
598 */
599 public static int ph2_coord_daysInMonth(int n) throws Exception {
600 int datasetFrequency = (int) getDSFrequency();// in minutes
601 // Calendar nominalInstanceCal =
602 // getCurrentInstance(getActionCreationtime());
603 Calendar nominalInstanceCal = getEffectiveNominalTime();
604 if (nominalInstanceCal == null) {
605 return -1;
606 }
607 nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n);
608 /*
609 * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0)
610 * { return -1; }
611 */
612 nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ
613 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
614 return nominalInstanceCal.getActualMaximum(Calendar.DAY_OF_MONTH);
615 }
616
617 public static int ph3_coord_daysInMonth(int n) throws Exception {
618 return ph2_coord_daysInMonth(n);
619 }
620
621 /**
622 * Determine the date-time in Oozie processing timezone of n-th latest available dataset instance. <p/> It depends
623 * on: <p/> 1. Data set frequency <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST
624 * <p/> 4. End Day/Month flag <p/> 5. Data set initial instance <p/> 6. Action Creation Time <p/> 7. Existence of
625 * dataset's directory
626 *
627 * @param n :instance count <p/> domain: n <= 0, n is integer
628 * @return date-time in Oozie processing timezone of the n-th instance <p/> returns 'null' means n-th instance is
629 * earlier than Initial-Instance of DS
630 * @throws Exception
631 */
632 public static String ph3_coord_latest(int n) throws Exception {
633 ParamChecker.checkLEZero(n, "latest:n");
634 if (isSyncDataSet()) {// For Sync Dataset
635 return coord_latest_sync(n);
636 }
637 else {
638 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
639 }
640 }
641
642 /**
643 * Determine the date-time in Oozie processing timezone of latest available dataset instances
644 * from start to end offsets from the nominal time. <p/> It depends
645 * on: <p/> 1. Data set frequency <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST
646 * <p/> 4. End Day/Month flag <p/> 5. Data set initial instance <p/> 6. Action Creation Time <p/> 7. Existence of
647 * dataset's directory
648 *
649 * @param start :start instance offset <p/> domain: start <= 0, start is integer
650 * @param end :end instance offset <p/> domain: end <= 0, end is integer
651 * @return date-time in Oozie processing timezone of the instances from start to end offsets
652 * delimited by comma. <p/> returns 'null' means start offset instance is
653 * earlier than Initial-Instance of DS
654 * @throws Exception
655 */
656 public static String ph3_coord_latestRange(int start, int end) throws Exception {
657 ParamChecker.checkLEZero(start, "latest:n");
658 ParamChecker.checkLEZero(end, "latest:n");
659 if (isSyncDataSet()) {// For Sync Dataset
660 return coord_latestRange_sync(start, end);
661 }
662 else {
663 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
664 }
665 }
666
667 /**
668 * Configure an evaluator with data set and application specific information. <p/> Helper method of associating
669 * dataset and application object
670 *
671 * @param evaluator : to set variables
672 * @param ds : Data Set object
673 * @param coordAction : Application instance
674 */
675 public static void configureEvaluator(ELEvaluator evaluator, SyncCoordDataset ds, SyncCoordAction coordAction) {
676 evaluator.setVariable(COORD_ACTION, coordAction);
677 evaluator.setVariable(DATASET, ds);
678 }
679
680 /**
681 * Helper method to wrap around with "${..}". <p/>
682 *
683 *
684 * @param eval :EL evaluator
685 * @param expr : expression to evaluate
686 * @return Resolved expression or echo back the same expression
687 * @throws Exception
688 */
689 public static String evalAndWrap(ELEvaluator eval, String expr) throws Exception {
690 try {
691 eval.setVariable(".wrap", null);
692 String result = eval.evaluate(expr, String.class);
693 if (eval.getVariable(".wrap") != null) {
694 return "${" + result + "}";
695 }
696 else {
697 return result;
698 }
699 }
700 catch (Exception e) {
701 throw new Exception("Unable to evaluate :" + expr + ":\n", e);
702 }
703 }
704
705 // Set of echo functions
706
707 public static String ph1_coord_current_echo(String n) {
708 return echoUnResolved("current", n);
709 }
710
711 public static String ph1_coord_absolute_echo(String date) {
712 return echoUnResolved("absolute", date);
713 }
714
715 public static String ph1_coord_currentRange_echo(String start, String end) {
716 return echoUnResolved("currentRange", start + ", " + end);
717 }
718
719 public static String ph1_coord_offset_echo(String n, String timeUnit) {
720 return echoUnResolved("offset", n + " , " + timeUnit);
721 }
722
723 public static String ph2_coord_current_echo(String n) {
724 return echoUnResolved("current", n);
725 }
726
727 public static String ph2_coord_currentRange_echo(String start, String end) {
728 return echoUnResolved("currentRange", start + ", " + end);
729 }
730
731 public static String ph2_coord_offset_echo(String n, String timeUnit) {
732 return echoUnResolved("offset", n + " , " + timeUnit);
733 }
734
735 public static String ph2_coord_absolute_echo(String date) {
736 return echoUnResolved("absolute", date);
737 }
738
739 public static String ph2_coord_absolute_range(String startInstance, int end) throws Exception {
740 int[] instanceCount = new int[1];
741
742 // getCurrentInstance() returns null, which means startInstance is less
743 // than initial instance
744 if (getCurrentInstance(DateUtils.getCalendar(startInstance).getTime(), instanceCount) == null) {
745 throw new CommandException(ErrorCode.E1010,
746 "intial-instance should be equal or earlier than the start-instance. intial-instance is "
747 + getInitialInstance() + " and start-instance is " + startInstance);
748 }
749 int[] nominalCount = new int[1];
750 if (getCurrentInstance(getActionCreationtime(), nominalCount) == null) {
751 throw new CommandException(ErrorCode.E1010,
752 "intial-instance should be equal or earlier than the nominal time. intial-instance is "
753 + getInitialInstance() + " and nominal time is " + getActionCreationtime());
754 }
755 // getCurrentInstance return offset relative to initial instance.
756 // start instance offset - nominal offset = start offset relative to
757 // nominal time-stamp.
758 int start = instanceCount[0] - nominalCount[0];
759 if (start > end) {
760 throw new CommandException(ErrorCode.E1010,
761 "start-instance should be equal or earlier than the end-instance. startInstance is "
762 + startInstance + " which is equivalent to current (" + instanceCount[0]
763 + ") but end is specified as current (" + end + ")");
764 }
765 return ph2_coord_currentRange(start, end);
766 }
767
768 public static String ph1_coord_dateOffset_echo(String n, String offset, String unit) {
769 return echoUnResolved("dateOffset", n + " , " + offset + " , " + unit);
770 }
771
772 public static String ph1_coord_formatTime_echo(String dateTime, String format) {
773 // Quote the dateTime value since it would contain a ':'.
774 return echoUnResolved("formatTime", "'"+dateTime+"'" + " , " + format);
775 }
776
777 public static String ph1_coord_latest_echo(String n) {
778 return echoUnResolved("latest", n);
779 }
780
781 public static String ph2_coord_latest_echo(String n) {
782 return ph1_coord_latest_echo(n);
783 }
784
785 public static String ph1_coord_future_echo(String n, String instance) {
786 return echoUnResolved("future", n + ", " + instance + "");
787 }
788
789 public static String ph2_coord_future_echo(String n, String instance) {
790 return ph1_coord_future_echo(n, instance);
791 }
792
793 public static String ph1_coord_latestRange_echo(String start, String end) {
794 return echoUnResolved("latestRange", start + ", " + end);
795 }
796
797 public static String ph2_coord_latestRange_echo(String start, String end) {
798 return ph1_coord_latestRange_echo(start, end);
799 }
800
801 public static String ph1_coord_futureRange_echo(String start, String end, String instance) {
802 return echoUnResolved("futureRange", start + ", " + end + ", " + instance);
803 }
804
805 public static String ph2_coord_futureRange_echo(String start, String end, String instance) {
806 return ph1_coord_futureRange_echo(start, end, instance);
807 }
808
809 public static String ph1_coord_dataIn_echo(String n) {
810 ELEvaluator eval = ELEvaluator.getCurrent();
811 String val = (String) eval.getVariable("oozie.dataname." + n);
812 if (val == null || val.equals("data-in") == false) {
813 XLog.getLog(CoordELFunctions.class).error("data_in_name " + n + " is not valid");
814 throw new RuntimeException("data_in_name " + n + " is not valid");
815 }
816 return echoUnResolved("dataIn", "'" + n + "'");
817 }
818
819 public static String ph1_coord_dataOut_echo(String n) {
820 ELEvaluator eval = ELEvaluator.getCurrent();
821 String val = (String) eval.getVariable("oozie.dataname." + n);
822 if (val == null || val.equals("data-out") == false) {
823 XLog.getLog(CoordELFunctions.class).error("data_out_name " + n + " is not valid");
824 throw new RuntimeException("data_out_name " + n + " is not valid");
825 }
826 return echoUnResolved("dataOut", "'" + n + "'");
827 }
828
829 public static String ph1_coord_nominalTime_echo() {
830 return echoUnResolved("nominalTime", "");
831 }
832
833 public static String ph1_coord_nominalTime_echo_wrap() {
834 // return "${coord:nominalTime()}"; // no resolution
835 return echoUnResolved("nominalTime", "");
836 }
837
838 public static String ph1_coord_nominalTime_echo_fixed() {
839 return "2009-03-06T010:00"; // Dummy resolution
840 }
841
842 public static String ph1_coord_actualTime_echo_wrap() {
843 // return "${coord:actualTime()}"; // no resolution
844 return echoUnResolved("actualTime", "");
845 }
846
847 public static String ph1_coord_actionId_echo() {
848 return echoUnResolved("actionId", "");
849 }
850
851 public static String ph1_coord_name_echo() {
852 return echoUnResolved("name", "");
853 }
854
855 // The following echo functions are not used in any phases yet
856 // They are here for future purpose.
857 public static String coord_minutes_echo(String n) {
858 return echoUnResolved("minutes", n);
859 }
860
861 public static String coord_hours_echo(String n) {
862 return echoUnResolved("hours", n);
863 }
864
865 public static String coord_days_echo(String n) {
866 return echoUnResolved("days", n);
867 }
868
869 public static String coord_endOfDay_echo(String n) {
870 return echoUnResolved("endOfDay", n);
871 }
872
873 public static String coord_months_echo(String n) {
874 return echoUnResolved("months", n);
875 }
876
877 public static String coord_endOfMonth_echo(String n) {
878 return echoUnResolved("endOfMonth", n);
879 }
880
881 public static String coord_actualTime_echo() {
882 return echoUnResolved("actualTime", "");
883 }
884
885 // This echo function will always return "24" for validation only.
886 // This evaluation ****should not**** replace the original XML
887 // Create a temporary string and validate the function
888 // This is **required** for evaluating an expression like
889 // coord:HoursInDay(0) + 3
890 // actual evaluation will happen in phase 2 or phase 3.
891 public static String ph1_coord_hoursInDay_echo(String n) {
892 return "24";
893 // return echoUnResolved("hoursInDay", n);
894 }
895
896 // This echo function will always return "30" for validation only.
897 // This evaluation ****should not**** replace the original XML
898 // Create a temporary string and validate the function
899 // This is **required** for evaluating an expression like
900 // coord:daysInMonth(0) + 3
901 // actual evaluation will happen in phase 2 or phase 3.
902 public static String ph1_coord_daysInMonth_echo(String n) {
903 // return echoUnResolved("daysInMonth", n);
904 return "30";
905 }
906
907 // This echo function will always return "3" for validation only.
908 // This evaluation ****should not**** replace the original XML
909 // Create a temporary string and validate the function
910 // This is **required** for evaluating an expression like coord:tzOffset + 2
911 // actual evaluation will happen in phase 2 or phase 3.
912 public static String ph1_coord_tzOffset_echo() {
913 // return echoUnResolved("tzOffset", "");
914 return "3";
915 }
916
917 // Local methods
918 /**
919 * @param n
920 * @return n-th instance Date-Time from current instance for data-set <p/> return empty string ("") if the
921 * Action_Creation_time or the n-th instance <p/> is earlier than the Initial_Instance of dataset.
922 * @throws Exception
923 */
924 private static String coord_current_sync(int n) throws Exception {
925 return coord_currentRange_sync(n, n);
926 }
927
928 private static String coord_currentRange_sync(int start, int end) throws Exception {
929 final XLog LOG = XLog.getLog(CoordELFunctions.class);
930 int datasetFrequency = getDSFrequency();// in minutes
931 TimeUnit dsTimeUnit = getDSTimeUnit();
932 int[] instCount = new int[1];// used as pass by ref
933 Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
934 if (nominalInstanceCal == null) {
935 LOG.warn("If the initial instance of the dataset is later than the nominal time, an empty string is"
936 + " returned. This means that no data is available at the current-instance specified by the user"
937 + " and the user could try modifying his initial-instance to an earlier time.");
938 return "";
939 } else {
940 Calendar initInstance = getInitialInstanceCal();
941 // Add in the reverse order - newest instance first.
942 nominalInstanceCal = (Calendar) initInstance.clone();
943 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), (instCount[0] + start) * datasetFrequency);
944 List<String> instances = new ArrayList<String>();
945 for (int i = start; i <= end; i++) {
946 if (nominalInstanceCal.compareTo(initInstance) < 0) {
947 LOG.warn("If the initial instance of the dataset is later than the current-instance specified,"
948 + " such as coord:current({0}) in this case, an empty string is returned. This means that"
949 + " no data is available at the current-instance specified by the user and the user could"
950 + " try modifying his initial-instance to an earlier time.", start);
951 }
952 else {
953 instances.add(DateUtils.formatDateOozieTZ(nominalInstanceCal));
954 }
955 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency);
956 }
957 instances = Lists.reverse(instances);
958 return StringUtils.join(instances, CoordELFunctions.INSTANCE_SEPARATOR);
959 }
960 }
961
962 /**
963 *
964 * @param n offset amount (integer)
965 * @param timeUnit TimeUnit for offset n ("MINUTE", "HOUR", "DAY", "MONTH", "YEAR")
966 * @return the offset time from the effective nominal time <p/> return empty string ("") if the Action_Creation_time or the
967 * offset instance <p/> is earlier than the Initial_Instance of dataset.
968 * @throws Exception
969 */
970 private static String coord_offset_sync(int n, String timeUnit) throws Exception {
971 Calendar rawCal = resolveOffsetRawTime(n, TimeUnit.valueOf(timeUnit), null);
972 if (rawCal == null) {
973 // warning already logged by resolveOffsetRawTime()
974 return "";
975 }
976
977 int freq = getDSFrequency();
978 TimeUnit freqUnit = getDSTimeUnit();
979 int freqCount = 0;
980 // We're going to manually turn back/forward cal by decrements/increments of freq and then check that it gives the same
981 // time as rawCal; this is to check that the offset time resolves to a frequency offset of the effective nominal time
982 // In other words, that there exists an integer x, such that coord:offset(n, timeUnit) == coord:current(x) is true
983 // If not, then we'll "rewind" rawCal to the latest instance earlier than rawCal and use that.
984 Calendar cal = getInitialInstanceCal();
985 if (rawCal.before(cal)) {
986 while (cal.after(rawCal)) {
987 cal.add(freqUnit.getCalendarUnit(), -freq);
988 freqCount--;
989 }
990 }
991 else if (rawCal.after(cal)) {
992 while (cal.before(rawCal)) {
993 cal.add(freqUnit.getCalendarUnit(), freq);
994 freqCount++;
995 }
996 }
997 if (cal.before(rawCal)) {
998 rawCal = cal;
999 }
1000 else if (cal.after(rawCal)) {
1001 cal.add(freqUnit.getCalendarUnit(), -freq);
1002 rawCal = cal;
1003 freqCount--;
1004 }
1005 String rawCalStr = DateUtils.formatDateOozieTZ(rawCal);
1006
1007 Calendar nominalInstanceCal = getInitialInstanceCal();
1008 nominalInstanceCal.add(freqUnit.getCalendarUnit(), freq * freqCount);
1009 if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) {
1010 XLog.getLog(CoordELFunctions.class).warn("If the initial instance of the dataset is later than the offset instance"
1011 + " specified, such as coord:offset({0}, {1}) in this case, an empty string is returned. This means that no"
1012 + " data is available at the offset instance specified by the user and the user could try modifying his"
1013 + " initial-instance to an earlier time.", n, timeUnit);
1014 return "";
1015 }
1016 String nominalCalStr = DateUtils.formatDateOozieTZ(nominalInstanceCal);
1017
1018 if (!rawCalStr.equals(nominalCalStr)) {
1019 throw new RuntimeException("Shouldn't happen");
1020 }
1021 return rawCalStr;
1022 }
1023
1024 /**
1025 * @param offset
1026 * @return n-th available latest instance Date-Time for SYNC data-set
1027 * @throws Exception
1028 */
1029 private static String coord_latest_sync(int offset) throws Exception {
1030 return coord_latestRange_sync(offset, offset);
1031 }
1032
1033 private static String coord_latestRange_sync(int startOffset, int endOffset) throws Exception {
1034 final XLog LOG = XLog.getLog(CoordELFunctions.class);
1035 final Thread currentThread = Thread.currentThread();
1036 ELEvaluator eval = ELEvaluator.getCurrent();
1037 String retVal = "";
1038 int datasetFrequency = (int) getDSFrequency();// in minutes
1039 TimeUnit dsTimeUnit = getDSTimeUnit();
1040 int[] instCount = new int[1];
1041 boolean useCurrentTime = Services.get().getConf().getBoolean(LATEST_EL_USE_CURRENT_TIME, false);
1042 Calendar nominalInstanceCal;
1043 if (useCurrentTime) {
1044 nominalInstanceCal = getCurrentInstance(new Date(), instCount);
1045 }
1046 else {
1047 nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
1048 }
1049 StringBuilder resolvedInstances = new StringBuilder();
1050 StringBuilder resolvedURIPaths = new StringBuilder();
1051 if (nominalInstanceCal != null) {
1052 Calendar initInstance = getInitialInstanceCal();
1053 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1054 if (ds == null) {
1055 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1056 }
1057 String uriTemplate = ds.getUriTemplate();
1058 Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
1059 if (conf == null) {
1060 throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
1061 }
1062 int available = 0;
1063 boolean resolved = false;
1064 String user = ParamChecker
1065 .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
1066 String doneFlag = ds.getDoneFlag();
1067 URIHandlerService uriService = Services.get().get(URIHandlerService.class);
1068 URIHandler uriHandler = null;
1069 Context uriContext = null;
1070 try {
1071 while (nominalInstanceCal.compareTo(initInstance) >= 0 && !currentThread.isInterrupted()) {
1072 ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
1073 String uriPath = uriEval.evaluate(uriTemplate, String.class);
1074 if (uriHandler == null) {
1075 URI uri = new URI(uriPath);
1076 uriHandler = uriService.getURIHandler(uri);
1077 uriContext = uriHandler.getContext(uri, conf, user);
1078 }
1079 String uriWithDoneFlag = uriHandler.getURIWithDoneFlag(uriPath, doneFlag);
1080 if (uriHandler.exists(new URI(uriWithDoneFlag), uriContext)) {
1081 XLog.getLog(CoordELFunctions.class)
1082 .debug("Found latest(" + available + "): " + uriWithDoneFlag);
1083 if (available == startOffset) {
1084 LOG.debug("Matched latest(" + available + "): " + uriWithDoneFlag);
1085 resolved = true;
1086 resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal));
1087 resolvedURIPaths.append(uriPath);
1088 retVal = resolvedInstances.toString();
1089 eval.setVariable("resolved_path", resolvedURIPaths.toString());
1090 break;
1091 }
1092 else if (available <= endOffset) {
1093 LOG.debug("Matched latest(" + available + "): " + uriWithDoneFlag);
1094 resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal)).append(
1095 INSTANCE_SEPARATOR);
1096 resolvedURIPaths.append(uriPath).append(INSTANCE_SEPARATOR);
1097 }
1098
1099 available--;
1100 }
1101 // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), -datasetFrequency);
1102 nominalInstanceCal = (Calendar) initInstance.clone();
1103 instCount[0]--;
1104 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
1105 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
1106 }
1107 }
1108 finally {
1109 if (uriContext != null) {
1110 uriContext.destroy();
1111 }
1112 }
1113 if (!resolved) {
1114 // return unchanged latest function with variable 'is_resolved'
1115 // to 'false'
1116 eval.setVariable("is_resolved", Boolean.FALSE);
1117 if (startOffset == endOffset) {
1118 retVal = "${coord:latest(" + startOffset + ")}";
1119 }
1120 else {
1121 retVal = "${coord:latestRange(" + startOffset + "," + endOffset + ")}";
1122 }
1123 }
1124 else {
1125 eval.setVariable("is_resolved", Boolean.TRUE);
1126 }
1127 }
1128 else {// No feasible nominal time
1129 eval.setVariable("is_resolved", Boolean.FALSE);
1130 }
1131 return retVal;
1132 }
1133
1134 /**
1135 * @param tm
1136 * @return a new Evaluator to be used for URI-template evaluation
1137 */
1138 private static ELEvaluator getUriEvaluator(Calendar tm) {
1139 tm.setTimeZone(DateUtils.getOozieProcessingTimeZone());
1140 ELEvaluator retEval = new ELEvaluator();
1141 retEval.setVariable("YEAR", tm.get(Calendar.YEAR));
1142 retEval.setVariable("MONTH", (tm.get(Calendar.MONTH) + 1) < 10 ? "0" + (tm.get(Calendar.MONTH) + 1) : (tm
1143 .get(Calendar.MONTH) + 1));
1144 retEval.setVariable("DAY", tm.get(Calendar.DAY_OF_MONTH) < 10 ? "0" + tm.get(Calendar.DAY_OF_MONTH) : tm
1145 .get(Calendar.DAY_OF_MONTH));
1146 retEval.setVariable("HOUR", tm.get(Calendar.HOUR_OF_DAY) < 10 ? "0" + tm.get(Calendar.HOUR_OF_DAY) : tm
1147 .get(Calendar.HOUR_OF_DAY));
1148 retEval.setVariable("MINUTE", tm.get(Calendar.MINUTE) < 10 ? "0" + tm.get(Calendar.MINUTE) : tm
1149 .get(Calendar.MINUTE));
1150 return retEval;
1151 }
1152
1153 /**
1154 * @return whether a data set is SYNCH or ASYNC
1155 */
1156 private static boolean isSyncDataSet() {
1157 ELEvaluator eval = ELEvaluator.getCurrent();
1158 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1159 if (ds == null) {
1160 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1161 }
1162 return ds.getType().equalsIgnoreCase("SYNC");
1163 }
1164
1165 /**
1166 * Check whether a function should be resolved.
1167 *
1168 * @param functionName
1169 * @param n
1170 * @return null if the functionName needs to be resolved otherwise return the calling function unresolved.
1171 */
1172 private static String checkIfResolved(String functionName, String n) {
1173 ELEvaluator eval = ELEvaluator.getCurrent();
1174 String replace = (String) eval.getVariable("resolve_" + functionName);
1175 if (replace == null || (replace != null && replace.equalsIgnoreCase("false"))) { // Don't
1176 // resolve
1177 // return "${coord:" + functionName + "(" + n +")}"; //Unresolved
1178 eval.setVariable(".wrap", "true");
1179 return "coord:" + functionName + "(" + n + ")"; // Unresolved
1180 }
1181 return null; // Resolved it
1182 }
1183
1184 private static String echoUnResolved(String functionName, String n) {
1185 return echoUnResolvedPre(functionName, n, "coord:");
1186 }
1187
1188 private static String echoUnResolvedPre(String functionName, String n, String prefix) {
1189 ELEvaluator eval = ELEvaluator.getCurrent();
1190 eval.setVariable(".wrap", "true");
1191 return prefix + functionName + "(" + n + ")"; // Unresolved
1192 }
1193
1194 /**
1195 * @return the initial instance of a DataSet in DATE
1196 */
1197 private static Date getInitialInstance() {
1198 ELEvaluator eval = ELEvaluator.getCurrent();
1199 return getInitialInstance(eval);
1200 }
1201
1202 /**
1203 * @return the initial instance of a DataSet in DATE
1204 */
1205 private static Date getInitialInstance(ELEvaluator eval) {
1206 return getInitialInstanceCal(eval).getTime();
1207 // return ds.getInitInstance();
1208 }
1209
1210 /**
1211 * @return the initial instance of a DataSet in Calendar
1212 */
1213 private static Calendar getInitialInstanceCal() {
1214 ELEvaluator eval = ELEvaluator.getCurrent();
1215 return getInitialInstanceCal(eval);
1216 }
1217
1218 /**
1219 * @return the initial instance of a DataSet in Calendar
1220 */
1221 private static Calendar getInitialInstanceCal(ELEvaluator eval) {
1222 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1223 if (ds == null) {
1224 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1225 }
1226 Calendar effInitTS = new GregorianCalendar(ds.getTimeZone());
1227 effInitTS.setTime(ds.getInitInstance());
1228 // To adjust EOD/EOM
1229 DateUtils.moveToEnd(effInitTS, getDSEndOfFlag(eval));
1230 return effInitTS;
1231 // return ds.getInitInstance();
1232 }
1233
1234 /**
1235 * @return Nominal or action creation Time when all the dependencies of an application instance are met.
1236 */
1237 private static Date getActionCreationtime() {
1238 ELEvaluator eval = ELEvaluator.getCurrent();
1239 return getActionCreationtime(eval);
1240 }
1241
1242 /**
1243 * @return Nominal or action creation Time when all the dependencies of an application instance are met.
1244 */
1245 private static Date getActionCreationtime(ELEvaluator eval) {
1246 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
1247 if (coordAction == null) {
1248 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
1249 }
1250 return coordAction.getNominalTime();
1251 }
1252
1253 /**
1254 * @return Actual Time when all the dependencies of an application instance are met.
1255 */
1256 private static Date getActualTime() {
1257 ELEvaluator eval = ELEvaluator.getCurrent();
1258 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
1259 if (coordAction == null) {
1260 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
1261 }
1262 return coordAction.getActualTime();
1263 }
1264
1265 /**
1266 * @return TimeZone for the application or job.
1267 */
1268 private static TimeZone getJobTZ() {
1269 ELEvaluator eval = ELEvaluator.getCurrent();
1270 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
1271 if (coordAction == null) {
1272 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
1273 }
1274 return coordAction.getTimeZone();
1275 }
1276
1277 /**
1278 * Find the current instance based on effectiveTime (i.e Action_Creation_Time or Action_Start_Time)
1279 *
1280 * @return current instance i.e. current(0) returns null if effectiveTime is earlier than Initial Instance time of
1281 * the dataset.
1282 */
1283 public static Calendar getCurrentInstance(Date effectiveTime, int instanceCount[]) {
1284 ELEvaluator eval = ELEvaluator.getCurrent();
1285 return getCurrentInstance(effectiveTime, instanceCount, eval);
1286 }
1287
1288 /**
1289 * Find the current instance based on effectiveTime (i.e Action_Creation_Time or Action_Start_Time)
1290 *
1291 * @return current instance i.e. current(0) returns null if effectiveTime is earlier than Initial Instance time of
1292 * the dataset.
1293 */
1294 private static Calendar getCurrentInstance(Date effectiveTime, int instanceCount[], ELEvaluator eval) {
1295 Date datasetInitialInstance = getInitialInstance(eval);
1296 TimeUnit dsTimeUnit = getDSTimeUnit(eval);
1297 TimeZone dsTZ = getDatasetTZ(eval);
1298 int dsFreq = getDSFrequency(eval);
1299 // Convert Date to Calendar for corresponding TZ
1300 Calendar current = Calendar.getInstance(dsTZ);
1301 current.setTime(datasetInitialInstance);
1302
1303 Calendar calEffectiveTime = new GregorianCalendar(dsTZ);
1304 calEffectiveTime.setTime(effectiveTime);
1305 if (instanceCount == null) { // caller doesn't care about this value
1306 instanceCount = new int[1];
1307 }
1308 instanceCount[0] = 0;
1309 if (current.compareTo(calEffectiveTime) > 0) {
1310 return null;
1311 }
1312
1313 switch(dsTimeUnit) {
1314 case MINUTE:
1315 instanceCount[0] = (int) ((effectiveTime.getTime() - datasetInitialInstance.getTime()) / MINUTE_MSEC);
1316 break;
1317 case HOUR:
1318 instanceCount[0] = (int) ((effectiveTime.getTime() - datasetInitialInstance.getTime()) / HOUR_MSEC);
1319 break;
1320 case DAY:
1321 case END_OF_DAY:
1322 instanceCount[0] = (int) ((effectiveTime.getTime() - datasetInitialInstance.getTime()) / DAY_MSEC);
1323 break;
1324 case MONTH:
1325 case END_OF_MONTH:
1326 instanceCount[0] = (int) ((effectiveTime.getTime() - datasetInitialInstance.getTime()) / MONTH_MSEC);
1327 break;
1328 case YEAR:
1329 instanceCount[0] = (int) ((effectiveTime.getTime() - datasetInitialInstance.getTime()) / YEAR_MSEC);
1330 break;
1331 default:
1332 throw new IllegalArgumentException("Unhandled dataset time unit " + dsTimeUnit);
1333 }
1334
1335 if (instanceCount[0] > 2) {
1336 instanceCount[0] = (instanceCount[0] / dsFreq);
1337 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * dsFreq);
1338 } else {
1339 instanceCount[0] = 0;
1340 }
1341 while (!current.getTime().after(effectiveTime)) {
1342 current.add(dsTimeUnit.getCalendarUnit(), dsFreq);
1343 instanceCount[0]++;
1344 }
1345 current.add(dsTimeUnit.getCalendarUnit(), -dsFreq);
1346 instanceCount[0]--;
1347 return current;
1348 }
1349
1350 /**
1351 * Find the current instance based on effectiveTime (i.e Action_Creation_Time or Action_Start_Time)
1352 *
1353 * @return current instance i.e. current(0) returns null if effectiveTime is earlier than Initial Instance time of
1354 * the dataset.
1355 */
1356 private static Calendar getCurrentInstance_old(Date effectiveTime, int instanceCount[], ELEvaluator eval) {
1357 Date datasetInitialInstance = getInitialInstance(eval);
1358 TimeUnit dsTimeUnit = getDSTimeUnit(eval);
1359 TimeZone dsTZ = getDatasetTZ(eval);
1360 int dsFreq = getDSFrequency(eval);
1361 // Convert Date to Calendar for corresponding TZ
1362 Calendar current = Calendar.getInstance();
1363 current.setTime(datasetInitialInstance);
1364 current.setTimeZone(dsTZ);
1365
1366 Calendar calEffectiveTime = Calendar.getInstance();
1367 calEffectiveTime.setTime(effectiveTime);
1368 calEffectiveTime.setTimeZone(dsTZ);
1369 if (instanceCount == null) { // caller doesn't care about this value
1370 instanceCount = new int[1];
1371 }
1372 instanceCount[0] = 0;
1373 if (current.compareTo(calEffectiveTime) > 0) {
1374 return null;
1375 }
1376 Calendar origCurrent = (Calendar) current.clone();
1377 while (current.compareTo(calEffectiveTime) <= 0) {
1378 current = (Calendar) origCurrent.clone();
1379 instanceCount[0]++;
1380 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * dsFreq);
1381 }
1382 instanceCount[0]--;
1383
1384 current = (Calendar) origCurrent.clone();
1385 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * dsFreq);
1386 return current;
1387 }
1388
1389 public static Calendar getEffectiveNominalTime() {
1390 Date datasetInitialInstance = getInitialInstance();
1391 TimeZone dsTZ = getDatasetTZ();
1392 // Convert Date to Calendar for corresponding TZ
1393 Calendar current = Calendar.getInstance();
1394 current.setTime(datasetInitialInstance);
1395 current.setTimeZone(dsTZ);
1396
1397 Calendar calEffectiveTime = Calendar.getInstance();
1398 calEffectiveTime.setTime(getActionCreationtime());
1399 calEffectiveTime.setTimeZone(dsTZ);
1400 if (current.compareTo(calEffectiveTime) > 0) {
1401 // Nominal Time < initial Instance
1402 // TODO: getClass() call doesn't work from static method.
1403 // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+
1404 // current.getTime());
1405 return null;
1406 }
1407 return calEffectiveTime;
1408 }
1409
1410 /**
1411 * @return dataset frequency in minutes
1412 */
1413 private static int getDSFrequency() {
1414 ELEvaluator eval = ELEvaluator.getCurrent();
1415 return getDSFrequency(eval);
1416 }
1417
1418 /**
1419 * @return dataset frequency in minutes
1420 */
1421 private static int getDSFrequency(ELEvaluator eval) {
1422 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1423 if (ds == null) {
1424 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1425 }
1426 return ds.getFrequency();
1427 }
1428
1429 /**
1430 * @return dataset TimeUnit
1431 */
1432 private static TimeUnit getDSTimeUnit() {
1433 ELEvaluator eval = ELEvaluator.getCurrent();
1434 return getDSTimeUnit(eval);
1435 }
1436
1437 /**
1438 * @return dataset TimeUnit
1439 */
1440 private static TimeUnit getDSTimeUnit(ELEvaluator eval) {
1441 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1442 if (ds == null) {
1443 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1444 }
1445 return ds.getTimeUnit();
1446 }
1447
1448 /**
1449 * @return dataset TimeZone
1450 */
1451 public static TimeZone getDatasetTZ() {
1452 ELEvaluator eval = ELEvaluator.getCurrent();
1453 return getDatasetTZ(eval);
1454 }
1455
1456 /**
1457 * @return dataset TimeZone
1458 */
1459 private static TimeZone getDatasetTZ(ELEvaluator eval) {
1460 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1461 if (ds == null) {
1462 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1463 }
1464 return ds.getTimeZone();
1465 }
1466
1467 /**
1468 * @return dataset TimeUnit
1469 */
1470 private static TimeUnit getDSEndOfFlag() {
1471 ELEvaluator eval = ELEvaluator.getCurrent();
1472 return getDSEndOfFlag(eval);
1473 }
1474
1475 /**
1476 * @return dataset TimeUnit
1477 */
1478 private static TimeUnit getDSEndOfFlag(ELEvaluator eval) {
1479 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1480 if (ds == null) {
1481 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1482 }
1483 return ds.getEndOfDuration();// == null ? "": ds.getEndOfDuration();
1484 }
1485
1486 /**
1487 * Return a job configuration property for the coordinator.
1488 *
1489 * @param property property name.
1490 * @return the value of the property, <code>null</code> if the property is undefined.
1491 */
1492 public static String coord_conf(String property) {
1493 ELEvaluator eval = ELEvaluator.getCurrent();
1494 return (String) eval.getVariable(property);
1495 }
1496
1497 /**
1498 * Return the user that submitted the coordinator job.
1499 *
1500 * @return the user that submitted the coordinator job.
1501 */
1502 public static String coord_user() {
1503 ELEvaluator eval = ELEvaluator.getCurrent();
1504 return (String) eval.getVariable(OozieClient.USER_NAME);
1505 }
1506
1507 /**
1508 * Takes two offset times and returns a list of multiples of the frequency offset from the effective nominal time that occur
1509 * between them. The caller should make sure that startCal is earlier than endCal.
1510 * <p>
1511 * As a simple example, assume its the same day: startCal is 1:00, endCal is 2:00, frequency is 20min, and effective nominal
1512 * time is 1:20 -- then this method would return a list containing: -20, 0, 20, 40, 60
1513 *
1514 * @param startCal The earlier offset time
1515 * @param endCal The later offset time
1516 * @param eval The ELEvaluator to use; cannot be null
1517 * @return A list of multiple of the frequency offset from the effective nominal time that occur between the startCal and endCal
1518 */
1519 public static List<Integer> expandOffsetTimes(Calendar startCal, Calendar endCal, ELEvaluator eval) {
1520 List<Integer> expandedFreqs = new ArrayList<Integer>();
1521 // Use eval because the "current" eval isn't set
1522 int freq = getDSFrequency(eval);
1523 TimeUnit freqUnit = getDSTimeUnit(eval);
1524 Calendar cal = getCurrentInstance(getActionCreationtime(eval), null, eval);
1525 int totalFreq = 0;
1526 if (startCal.before(cal)) {
1527 while (cal.after(startCal)) {
1528 cal.add(freqUnit.getCalendarUnit(), -freq);
1529 totalFreq += -freq;
1530 }
1531 if (cal.before(startCal)) {
1532 cal.add(freqUnit.getCalendarUnit(), freq);
1533 totalFreq += freq;
1534 }
1535 }
1536 else if (startCal.after(cal)) {
1537 while (cal.before(startCal)) {
1538 cal.add(freqUnit.getCalendarUnit(), freq);
1539 totalFreq += freq;
1540 }
1541 }
1542 // At this point, cal is the smallest multiple of the dataset frequency that is >= to the startCal and offset from the
1543 // effective nominal time. Now we can find all of the instances that occur between startCal and endCal, inclusive.
1544 while (cal.before(endCal) || cal.equals(endCal)) {
1545 expandedFreqs.add(totalFreq);
1546 cal.add(freqUnit.getCalendarUnit(), freq);
1547 totalFreq += freq;
1548 }
1549 return expandedFreqs;
1550 }
1551
1552 /**
1553 * Resolve the offset time from the effective nominal time
1554 *
1555 * @param n offset amount (integer)
1556 * @param timeUnit TimeUnit for offset n ("MINUTE", "HOUR", "DAY", "MONTH", "YEAR")
1557 * @param eval The ELEvaluator to use; or null to use the "current" eval
1558 * @return A Calendar of the offset time
1559 */
1560 public static Calendar resolveOffsetRawTime(int n, TimeUnit timeUnit, ELEvaluator eval) {
1561 // Use eval if given (for when the "current" eval isn't set)
1562 Calendar cal;
1563 if (eval == null) {
1564 cal = getCurrentInstance(getActionCreationtime(), null);
1565 }
1566 else {
1567 cal = getCurrentInstance(getActionCreationtime(eval), null, eval);
1568 }
1569 if (cal == null) {
1570 XLog.getLog(CoordELFunctions.class).warn("If the initial instance of the dataset is later than the nominal time, an"
1571 + " empty string is returned. This means that no data is available at the offset instance specified by the user"
1572 + " and the user could try modifying his or her initial-instance to an earlier time.");
1573 return null;
1574 }
1575 cal.add(timeUnit.getCalendarUnit(), n);
1576 return cal;
1577 }
1578 }