001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.command.coord;
019
020 import java.io.IOException;
021 import java.io.InputStreamReader;
022 import java.io.Reader;
023 import java.io.StringReader;
024 import java.io.StringWriter;
025 import java.net.URI;
026 import java.net.URISyntaxException;
027 import java.util.ArrayList;
028 import java.util.Calendar;
029 import java.util.Date;
030 import java.util.HashMap;
031 import java.util.HashSet;
032 import java.util.Iterator;
033 import java.util.List;
034 import java.util.Map;
035 import java.util.Set;
036 import java.util.TreeSet;
037
038 import javax.xml.transform.stream.StreamSource;
039 import javax.xml.validation.Validator;
040
041 import org.apache.hadoop.conf.Configuration;
042 import org.apache.hadoop.fs.FileSystem;
043 import org.apache.hadoop.fs.Path;
044 import org.apache.oozie.CoordinatorJobBean;
045 import org.apache.oozie.ErrorCode;
046 import org.apache.oozie.client.CoordinatorJob;
047 import org.apache.oozie.client.Job;
048 import org.apache.oozie.client.OozieClient;
049 import org.apache.oozie.client.CoordinatorJob.Execution;
050 import org.apache.oozie.command.CommandException;
051 import org.apache.oozie.command.SubmitTransitionXCommand;
052 import org.apache.oozie.command.bundle.BundleStatusUpdateXCommand;
053 import org.apache.oozie.coord.CoordELEvaluator;
054 import org.apache.oozie.coord.CoordELFunctions;
055 import org.apache.oozie.coord.CoordinatorJobException;
056 import org.apache.oozie.coord.TimeUnit;
057 import org.apache.oozie.executor.jpa.CoordJobQueryExecutor;
058 import org.apache.oozie.executor.jpa.JPAExecutorException;
059 import org.apache.oozie.service.DagXLogInfoService;
060 import org.apache.oozie.service.HadoopAccessorException;
061 import org.apache.oozie.service.HadoopAccessorService;
062 import org.apache.oozie.service.JPAService;
063 import org.apache.oozie.service.SchemaService;
064 import org.apache.oozie.service.Service;
065 import org.apache.oozie.service.Services;
066 import org.apache.oozie.service.UUIDService;
067 import org.apache.oozie.service.SchemaService.SchemaName;
068 import org.apache.oozie.service.UUIDService.ApplicationType;
069 import org.apache.oozie.util.ConfigUtils;
070 import org.apache.oozie.util.DateUtils;
071 import org.apache.oozie.util.ELEvaluator;
072 import org.apache.oozie.util.ELUtils;
073 import org.apache.oozie.util.IOUtils;
074 import org.apache.oozie.util.InstrumentUtils;
075 import org.apache.oozie.util.LogUtils;
076 import org.apache.oozie.util.ParamChecker;
077 import org.apache.oozie.util.ParameterVerifier;
078 import org.apache.oozie.util.ParameterVerifierException;
079 import org.apache.oozie.util.PropertiesUtils;
080 import org.apache.oozie.util.XConfiguration;
081 import org.apache.oozie.util.XLog;
082 import org.apache.oozie.util.XmlUtils;
083 import org.jdom.Attribute;
084 import org.jdom.Element;
085 import org.jdom.JDOMException;
086 import org.jdom.Namespace;
087 import org.xml.sax.SAXException;
088
089 /**
090 * This class provides the functionalities to resolve a coordinator job XML and write the job information into a DB
091 * table.
092 * <p/>
093 * Specifically it performs the following functions: 1. Resolve all the variables or properties using job
094 * configurations. 2. Insert all datasets definition as part of the <data-in> and <data-out> tags. 3. Validate the XML
095 * at runtime.
096 */
097 public class CoordSubmitXCommand extends SubmitTransitionXCommand {
098
099 protected Configuration conf;
100 private final String bundleId;
101 private final String coordName;
102 protected boolean dryrun;
103 protected JPAService jpaService = null;
104 private CoordinatorJob.Status prevStatus = CoordinatorJob.Status.PREP;
105
106 public static final String CONFIG_DEFAULT = "coord-config-default.xml";
107 public static final String COORDINATOR_XML_FILE = "coordinator.xml";
108 public final String COORD_INPUT_EVENTS ="input-events";
109 public final String COORD_OUTPUT_EVENTS = "output-events";
110 public final String COORD_INPUT_EVENTS_DATA_IN ="data-in";
111 public final String COORD_OUTPUT_EVENTS_DATA_OUT = "data-out";
112
113 private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>();
114 private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
115
116 protected CoordinatorJobBean coordJob = null;
117 /**
118 * Default timeout for normal jobs, in minutes, after which coordinator input check will timeout
119 */
120 public static final String CONF_DEFAULT_TIMEOUT_NORMAL = Service.CONF_PREFIX + "coord.normal.default.timeout";
121
122 public static final String CONF_DEFAULT_CONCURRENCY = Service.CONF_PREFIX + "coord.default.concurrency";
123
124 public static final String CONF_DEFAULT_THROTTLE = Service.CONF_PREFIX + "coord.default.throttle";
125
126 public static final String CONF_MAT_THROTTLING_FACTOR = Service.CONF_PREFIX
127 + "coord.materialization.throttling.factor";
128
129 /**
130 * Default MAX timeout in minutes, after which coordinator input check will timeout
131 */
132 public static final String CONF_DEFAULT_MAX_TIMEOUT = Service.CONF_PREFIX + "coord.default.max.timeout";
133
134 public static final String CONF_QUEUE_SIZE = Service.CONF_PREFIX + "CallableQueueService.queue.size";
135
136 public static final String CONF_CHECK_MAX_FREQUENCY = Service.CONF_PREFIX + "coord.check.maximum.frequency";
137
138 private ELEvaluator evalFreq = null;
139 private ELEvaluator evalNofuncs = null;
140 private ELEvaluator evalData = null;
141 private ELEvaluator evalInst = null;
142 private ELEvaluator evalAction = null;
143 private ELEvaluator evalSla = null;
144 private ELEvaluator evalTimeout = null;
145
146 static {
147 String[] badUserProps = { PropertiesUtils.YEAR, PropertiesUtils.MONTH, PropertiesUtils.DAY,
148 PropertiesUtils.HOUR, PropertiesUtils.MINUTE, PropertiesUtils.DAYS, PropertiesUtils.HOURS,
149 PropertiesUtils.MINUTES, PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB,
150 PropertiesUtils.TB, PropertiesUtils.PB, PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN,
151 PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN, PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS };
152 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES);
153
154 String[] badDefaultProps = { PropertiesUtils.HADOOP_USER};
155 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
156 PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
157 }
158
159 /**
160 * Constructor to create the Coordinator Submit Command.
161 *
162 * @param conf : Configuration for Coordinator job
163 */
164 public CoordSubmitXCommand(Configuration conf) {
165 super("coord_submit", "coord_submit", 1);
166 this.conf = ParamChecker.notNull(conf, "conf");
167 this.bundleId = null;
168 this.coordName = null;
169 }
170
171 /**
172 * Constructor to create the Coordinator Submit Command by bundle job.
173 *
174 * @param conf : Configuration for Coordinator job
175 * @param bundleId : bundle id
176 * @param coordName : coord name
177 */
178 public CoordSubmitXCommand(Configuration conf, String bundleId, String coordName) {
179 super("coord_submit", "coord_submit", 1);
180 this.conf = ParamChecker.notNull(conf, "conf");
181 this.bundleId = ParamChecker.notEmpty(bundleId, "bundleId");
182 this.coordName = ParamChecker.notEmpty(coordName, "coordName");
183 }
184
185 /**
186 * Constructor to create the Coordinator Submit Command.
187 *
188 * @param dryrun : if dryrun
189 * @param conf : Configuration for Coordinator job
190 */
191 public CoordSubmitXCommand(boolean dryrun, Configuration conf) {
192 this(conf);
193 this.dryrun = dryrun;
194 }
195
196 /* (non-Javadoc)
197 * @see org.apache.oozie.command.XCommand#execute()
198 */
199 @Override
200 protected String submit() throws CommandException {
201 LOG.info("STARTED Coordinator Submit");
202 String jobId = submitJob();
203 LOG.info("ENDED Coordinator Submit jobId=" + jobId);
204 return jobId;
205 }
206
207 protected String submitJob() throws CommandException {
208 String jobId = null;
209 InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
210
211 boolean exceptionOccured = false;
212 try {
213 mergeDefaultConfig();
214
215 String appXml = readAndValidateXml();
216 coordJob.setOrigJobXml(appXml);
217 LOG.debug("jobXml after initial validation " + XmlUtils.prettyPrint(appXml).toString());
218
219 Element eXml = XmlUtils.parseXml(appXml);
220
221 String appNamespace = readAppNamespace(eXml);
222 coordJob.setAppNamespace(appNamespace);
223
224 ParameterVerifier.verifyParameters(conf, eXml);
225
226 appXml = XmlUtils.removeComments(appXml);
227 initEvaluators();
228 Element eJob = basicResolveAndIncludeDS(appXml, conf, coordJob);
229
230 validateCoordinatorJob();
231
232 // checking if the coordinator application data input/output events
233 // specify multiple data instance values in erroneous manner
234 checkMultipleTimeInstances(eJob, COORD_INPUT_EVENTS, COORD_INPUT_EVENTS_DATA_IN);
235 checkMultipleTimeInstances(eJob, COORD_OUTPUT_EVENTS, COORD_OUTPUT_EVENTS_DATA_OUT);
236
237 LOG.debug("jobXml after all validation " + XmlUtils.prettyPrint(eJob).toString());
238
239 jobId = storeToDB(appXml, eJob, coordJob);
240 // log job info for coordinator job
241 LogUtils.setLogInfo(coordJob, logInfo);
242 LOG = XLog.resetPrefix(LOG);
243
244 if (!dryrun) {
245 queueMaterializeTransitionXCommand(jobId);
246 }
247 else {
248 return getDryRun(coordJob);
249 }
250 }
251 catch (JDOMException jex) {
252 exceptionOccured = true;
253 LOG.warn("ERROR: ", jex);
254 throw new CommandException(ErrorCode.E0700, jex.getMessage(), jex);
255 }
256 catch (CoordinatorJobException cex) {
257 exceptionOccured = true;
258 LOG.warn("ERROR: ", cex);
259 throw new CommandException(cex);
260 }
261 catch (ParameterVerifierException pex) {
262 exceptionOccured = true;
263 LOG.warn("ERROR: ", pex);
264 throw new CommandException(pex);
265 }
266 catch (IllegalArgumentException iex) {
267 exceptionOccured = true;
268 LOG.warn("ERROR: ", iex);
269 throw new CommandException(ErrorCode.E1003, iex.getMessage(), iex);
270 }
271 catch (Exception ex) {
272 exceptionOccured = true;
273 LOG.warn("ERROR: ", ex);
274 throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
275 }
276 finally {
277 if (exceptionOccured) {
278 if (coordJob.getId() == null || coordJob.getId().equalsIgnoreCase("")) {
279 coordJob.setStatus(CoordinatorJob.Status.FAILED);
280 coordJob.resetPending();
281 }
282 }
283 }
284 return jobId;
285 }
286
287 /**
288 * Gets the dryrun output.
289 *
290 * @param jobId the job id
291 * @return the dry run
292 * @throws Exception the exception
293 */
294 protected String getDryRun(CoordinatorJobBean coordJob) throws Exception{
295 Date startTime = coordJob.getStartTime();
296 long startTimeMilli = startTime.getTime();
297 long endTimeMilli = startTimeMilli + (3600 * 1000);
298 Date jobEndTime = coordJob.getEndTime();
299 Date endTime = new Date(endTimeMilli);
300 if (endTime.compareTo(jobEndTime) > 0) {
301 endTime = jobEndTime;
302 }
303 String jobId = coordJob.getId();
304 LOG.info("[" + jobId + "]: Update status to RUNNING");
305 coordJob.setStatus(Job.Status.RUNNING);
306 coordJob.setPending();
307 CoordActionMaterializeCommand coordActionMatCom = new CoordActionMaterializeCommand(jobId, startTime,
308 endTime);
309 Configuration jobConf = null;
310 try {
311 jobConf = new XConfiguration(new StringReader(coordJob.getConf()));
312 }
313 catch (IOException e1) {
314 LOG.warn("Configuration parse error. read from DB :" + coordJob.getConf(), e1);
315 }
316 String action = coordActionMatCom.materializeJobs(true, coordJob, jobConf, null);
317 String output = coordJob.getJobXml() + System.getProperty("line.separator")
318 + "***actions for instance***" + action;
319 return output;
320 }
321
322 /**
323 * Queue MaterializeTransitionXCommand
324 */
325 protected void queueMaterializeTransitionXCommand(String jobId) {
326 // submit a command to materialize jobs for the next 1 hour (3600 secs)
327 // so we don't wait 10 mins for the Service to run.
328 queue(new CoordMaterializeTransitionXCommand(jobId, 3600), 100);
329 }
330
331 /**
332 * Method that validates values in the definition for correctness. Placeholder to add more.
333 */
334 private void validateCoordinatorJob() throws Exception {
335 // check if startTime < endTime
336 if (!coordJob.getStartTime().before(coordJob.getEndTime())) {
337 throw new IllegalArgumentException("Coordinator Start Time must be earlier than End Time.");
338 }
339
340 try {
341 // Check if a coord job with cron frequency will materialize actions
342 int freq = Integer.parseInt(coordJob.getFrequency());
343
344 // Check if the frequency is faster than 5 min if enabled
345 if (Services.get().getConf().getBoolean(CONF_CHECK_MAX_FREQUENCY, true)) {
346 CoordinatorJob.Timeunit unit = coordJob.getTimeUnit();
347 if (freq == 0 || (freq < 5 && unit == CoordinatorJob.Timeunit.MINUTE)) {
348 throw new IllegalArgumentException("Coordinator job with frequency [" + freq +
349 "] minutes is faster than allowed maximum of 5 minutes ("
350 + CONF_CHECK_MAX_FREQUENCY + " is set to true)");
351 }
352 }
353 } catch (NumberFormatException e) {
354 Date start = coordJob.getStartTime();
355 Calendar cal = Calendar.getInstance();
356 cal.setTime(start);
357 cal.add(Calendar.MINUTE, -1);
358 start = cal.getTime();
359
360 Date nextTime = CoordCommandUtils.getNextValidActionTimeForCronFrequency(start, coordJob);
361 if (nextTime == null) {
362 throw new IllegalArgumentException("Invalid coordinator cron frequency: " + coordJob.getFrequency());
363 }
364 if (!nextTime.before(coordJob.getEndTime())) {
365 throw new IllegalArgumentException("Coordinator job with frequency '" +
366 coordJob.getFrequency() + "' materializes no actions between start and end time.");
367 }
368 }
369 }
370
371 /*
372 * Check against multiple data instance values inside a single <instance> <start-instance> or <end-instance> tag
373 * If found, the job is not submitted and user is informed to correct the error, instead of defaulting to the first instance value in the list
374 */
375 private void checkMultipleTimeInstances(Element eCoordJob, String eventType, String dataType) throws CoordinatorJobException {
376 Element eventsSpec, dataSpec, instance;
377 List<Element> instanceSpecList;
378 Namespace ns = eCoordJob.getNamespace();
379 String instanceValue;
380 eventsSpec = eCoordJob.getChild(eventType, ns);
381 if (eventsSpec != null) {
382 dataSpec = eventsSpec.getChild(dataType, ns);
383 if (dataSpec != null) {
384 // In case of input-events, there can be multiple child <instance> datasets. Iterating to ensure none of them have errors
385 instanceSpecList = dataSpec.getChildren("instance", ns);
386 Iterator instanceIter = instanceSpecList.iterator();
387 while(instanceIter.hasNext()) {
388 instance = ((Element) instanceIter.next());
389 if(instance.getContentSize() == 0) { //empty string or whitespace
390 throw new CoordinatorJobException(ErrorCode.E1021, "<instance> tag within " + eventType + " is empty!");
391 }
392 instanceValue = instance.getContent(0).toString();
393 boolean isInvalid = false;
394 try {
395 isInvalid = evalAction.checkForExistence(instanceValue, ",");
396 } catch (Exception e) {
397 handleELParseException(eventType, dataType, instanceValue);
398 }
399 if (isInvalid) { // reaching this block implies instance is not empty i.e. length > 0
400 handleExpresionWithMultipleInstances(eventType, dataType, instanceValue);
401 }
402 }
403
404 // In case of input-events, there can be multiple child <start-instance> datasets. Iterating to ensure none of them have errors
405 instanceSpecList = dataSpec.getChildren("start-instance", ns);
406 instanceIter = instanceSpecList.iterator();
407 while(instanceIter.hasNext()) {
408 instance = ((Element) instanceIter.next());
409 if(instance.getContentSize() == 0) { //empty string or whitespace
410 throw new CoordinatorJobException(ErrorCode.E1021, "<start-instance> tag within " + eventType + " is empty!");
411 }
412 instanceValue = instance.getContent(0).toString();
413 boolean isInvalid = false;
414 try {
415 isInvalid = evalAction.checkForExistence(instanceValue, ",");
416 } catch (Exception e) {
417 handleELParseException(eventType, dataType, instanceValue);
418 }
419 if (isInvalid) { // reaching this block implies start instance is not empty i.e. length > 0
420 handleExpresionWithStartMultipleInstances(eventType, dataType, instanceValue);
421 }
422 }
423
424 // In case of input-events, there can be multiple child <end-instance> datasets. Iterating to ensure none of them have errors
425 instanceSpecList = dataSpec.getChildren("end-instance", ns);
426 instanceIter = instanceSpecList.iterator();
427 while(instanceIter.hasNext()) {
428 instance = ((Element) instanceIter.next());
429 if(instance.getContentSize() == 0) { //empty string or whitespace
430 throw new CoordinatorJobException(ErrorCode.E1021, "<end-instance> tag within " + eventType + " is empty!");
431 }
432 instanceValue = instance.getContent(0).toString();
433 boolean isInvalid = false;
434 try {
435 isInvalid = evalAction.checkForExistence(instanceValue, ",");
436 } catch (Exception e) {
437 handleELParseException(eventType, dataType, instanceValue);
438 }
439 if (isInvalid) { // reaching this block implies instance is not empty i.e. length > 0
440 handleExpresionWithMultipleEndInstances(eventType, dataType, instanceValue);
441 }
442 }
443
444 }
445 }
446 }
447
448 private void handleELParseException(String eventType, String dataType, String instanceValue)
449 throws CoordinatorJobException {
450 String correctAction = null;
451 if(dataType.equals(COORD_INPUT_EVENTS_DATA_IN)) {
452 correctAction = "Coordinator app definition should have valid <instance> tag for data-in";
453 } else if(dataType.equals(COORD_OUTPUT_EVENTS_DATA_OUT)) {
454 correctAction = "Coordinator app definition should have valid <instance> tag for data-out";
455 }
456 throw new CoordinatorJobException(ErrorCode.E1021, eventType + " instance '" + instanceValue
457 + "' is not valid. Coordinator job NOT SUBMITTED. " + correctAction);
458 }
459
460 private void handleExpresionWithMultipleInstances(String eventType, String dataType, String instanceValue)
461 throws CoordinatorJobException {
462 String correctAction = null;
463 if(dataType.equals(COORD_INPUT_EVENTS_DATA_IN)) {
464 correctAction = "Coordinator app definition should have separate <instance> tag per data-in instance";
465 } else if(dataType.equals(COORD_OUTPUT_EVENTS_DATA_OUT)) {
466 correctAction = "Coordinator app definition can have only one <instance> tag per data-out instance";
467 }
468 throw new CoordinatorJobException(ErrorCode.E1021, eventType + " instance '" + instanceValue
469 + "' contains more than one date instance. Coordinator job NOT SUBMITTED. " + correctAction);
470 }
471
472 private void handleExpresionWithStartMultipleInstances(String eventType, String dataType, String instanceValue)
473 throws CoordinatorJobException {
474 String correctAction = "Coordinator app definition should not have multiple start-instances";
475 throw new CoordinatorJobException(ErrorCode.E1021, eventType + " start-instance '" + instanceValue
476 + "' contains more than one date start-instance. Coordinator job NOT SUBMITTED. " + correctAction);
477 }
478
479 private void handleExpresionWithMultipleEndInstances(String eventType, String dataType, String instanceValue)
480 throws CoordinatorJobException {
481 String correctAction = "Coordinator app definition should not have multiple end-instances";
482 throw new CoordinatorJobException(ErrorCode.E1021, eventType + " end-instance '" + instanceValue
483 + "' contains more than one date end-instance. Coordinator job NOT SUBMITTED. " + correctAction);
484 }
485 /**
486 * Read the application XML and validate against coordinator Schema
487 *
488 * @return validated coordinator XML
489 * @throws CoordinatorJobException thrown if unable to read or validate coordinator xml
490 */
491 protected String readAndValidateXml() throws CoordinatorJobException {
492 String appPath = ParamChecker.notEmpty(conf.get(OozieClient.COORDINATOR_APP_PATH),
493 OozieClient.COORDINATOR_APP_PATH);
494 String coordXml = readDefinition(appPath);
495 validateXml(coordXml);
496 return coordXml;
497 }
498
499 /**
500 * Validate against Coordinator XSD file
501 *
502 * @param xmlContent : Input coordinator xml
503 * @throws CoordinatorJobException thrown if unable to validate coordinator xml
504 */
505 private void validateXml(String xmlContent) throws CoordinatorJobException {
506 javax.xml.validation.Schema schema = Services.get().get(SchemaService.class).getSchema(SchemaName.COORDINATOR);
507 Validator validator = schema.newValidator();
508 try {
509 validator.validate(new StreamSource(new StringReader(xmlContent)));
510 }
511 catch (SAXException ex) {
512 LOG.warn("SAXException :", ex);
513 throw new CoordinatorJobException(ErrorCode.E0701, ex.getMessage(), ex);
514 }
515 catch (IOException ex) {
516 LOG.warn("IOException :", ex);
517 throw new CoordinatorJobException(ErrorCode.E0702, ex.getMessage(), ex);
518 }
519 }
520
521 /**
522 * Read the application XML schema namespace
523 *
524 * @param coordXmlElement input coordinator xml Element
525 * @return app xml namespace
526 * @throws CoordinatorJobException
527 */
528 private String readAppNamespace(Element coordXmlElement) throws CoordinatorJobException {
529 Namespace ns = coordXmlElement.getNamespace();
530 if (ns != null && bundleId != null && ns.getURI().equals(SchemaService.COORDINATOR_NAMESPACE_URI_1)) {
531 throw new CoordinatorJobException(ErrorCode.E1319, "bundle app can not submit coordinator namespace "
532 + SchemaService.COORDINATOR_NAMESPACE_URI_1 + ", please use 0.2 or later");
533 }
534 if (ns != null) {
535 return ns.getURI();
536 }
537 else {
538 throw new CoordinatorJobException(ErrorCode.E0700, "the application xml namespace is not given");
539 }
540 }
541
542 /**
543 * Merge default configuration with user-defined configuration.
544 *
545 * @throws CommandException thrown if failed to read or merge configurations
546 */
547 protected void mergeDefaultConfig() throws CommandException {
548 Path configDefault = null;
549 try {
550 String coordAppPathStr = conf.get(OozieClient.COORDINATOR_APP_PATH);
551 Path coordAppPath = new Path(coordAppPathStr);
552 String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
553 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
554 Configuration fsConf = has.createJobConf(coordAppPath.toUri().getAuthority());
555 FileSystem fs = has.createFileSystem(user, coordAppPath.toUri(), fsConf);
556
557 // app path could be a directory
558 if (!fs.isFile(coordAppPath)) {
559 configDefault = new Path(coordAppPath, CONFIG_DEFAULT);
560 } else {
561 configDefault = new Path(coordAppPath.getParent(), CONFIG_DEFAULT);
562 }
563
564 if (fs.exists(configDefault)) {
565 Configuration defaultConf = new XConfiguration(fs.open(configDefault));
566 PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
567 XConfiguration.injectDefaults(defaultConf, conf);
568 }
569 else {
570 LOG.info("configDefault Doesn't exist " + configDefault);
571 }
572 PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
573
574 // Resolving all variables in the job properties.
575 // This ensures the Hadoop Configuration semantics is preserved.
576 XConfiguration resolvedVarsConf = new XConfiguration();
577 for (Map.Entry<String, String> entry : conf) {
578 resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
579 }
580 conf = resolvedVarsConf;
581 }
582 catch (IOException e) {
583 throw new CommandException(ErrorCode.E0702, e.getMessage() + " : Problem reading default config "
584 + configDefault, e);
585 }
586 catch (HadoopAccessorException e) {
587 throw new CommandException(e);
588 }
589 LOG.debug("Merged CONF :" + XmlUtils.prettyPrint(conf).toString());
590 }
591
592 /**
593 * The method resolve all the variables that are defined in configuration. It also include the data set definition
594 * from dataset file into XML.
595 *
596 * @param appXml : Original job XML
597 * @param conf : Configuration of the job
598 * @param coordJob : Coordinator job bean to be populated.
599 * @return Resolved and modified job XML element.
600 * @throws CoordinatorJobException thrown if failed to resolve basic entities or include referred datasets
601 * @throws Exception thrown if failed to resolve basic entities or include referred datasets
602 */
603 public Element basicResolveAndIncludeDS(String appXml, Configuration conf, CoordinatorJobBean coordJob)
604 throws CoordinatorJobException, Exception {
605 Element basicResolvedApp = resolveInitial(conf, appXml, coordJob);
606 includeDataSets(basicResolvedApp, conf);
607 return basicResolvedApp;
608 }
609
610 /**
611 * Insert data set into data-in and data-out tags.
612 *
613 * @param eAppXml : coordinator application XML
614 * @param eDatasets : DataSet XML
615 */
616 @SuppressWarnings("unchecked")
617 private void insertDataSet(Element eAppXml, Element eDatasets) {
618 // Adding DS definition in the coordinator XML
619 Element inputList = eAppXml.getChild("input-events", eAppXml.getNamespace());
620 if (inputList != null) {
621 for (Element dataIn : (List<Element>) inputList.getChildren("data-in", eAppXml.getNamespace())) {
622 Element eDataset = findDataSet(eDatasets, dataIn.getAttributeValue("dataset"));
623 dataIn.getContent().add(0, eDataset);
624 }
625 }
626 Element outputList = eAppXml.getChild("output-events", eAppXml.getNamespace());
627 if (outputList != null) {
628 for (Element dataOut : (List<Element>) outputList.getChildren("data-out", eAppXml.getNamespace())) {
629 Element eDataset = findDataSet(eDatasets, dataOut.getAttributeValue("dataset"));
630 dataOut.getContent().add(0, eDataset);
631 }
632 }
633 }
634
635 /**
636 * Find a specific dataset from a list of Datasets.
637 *
638 * @param eDatasets : List of data sets
639 * @param name : queried data set name
640 * @return one Dataset element. otherwise throw Exception
641 */
642 @SuppressWarnings("unchecked")
643 private static Element findDataSet(Element eDatasets, String name) {
644 for (Element eDataset : (List<Element>) eDatasets.getChildren("dataset", eDatasets.getNamespace())) {
645 if (eDataset.getAttributeValue("name").equals(name)) {
646 eDataset = (Element) eDataset.clone();
647 eDataset.detach();
648 return eDataset;
649 }
650 }
651 throw new RuntimeException("undefined dataset: " + name);
652 }
653
654 /**
655 * Initialize all the required EL Evaluators.
656 */
657 protected void initEvaluators() {
658 evalFreq = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-freq");
659 evalNofuncs = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-nofuncs");
660 evalInst = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-instances");
661 evalAction = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-action-start");
662 evalTimeout = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-wait-timeout");
663 }
664
665 /**
666 * Resolve basic entities using job Configuration.
667 *
668 * @param conf :Job configuration
669 * @param appXml : Original job XML
670 * @param coordJob : Coordinator job bean to be populated.
671 * @return Resolved job XML element.
672 * @throws CoordinatorJobException thrown if failed to resolve basic entities
673 * @throws Exception thrown if failed to resolve basic entities
674 */
675 @SuppressWarnings("unchecked")
676 protected Element resolveInitial(Configuration conf, String appXml, CoordinatorJobBean coordJob)
677 throws CoordinatorJobException, Exception {
678 Element eAppXml = XmlUtils.parseXml(appXml);
679 // job's main attributes
680 // frequency
681 String val = resolveAttribute("frequency", eAppXml, evalFreq);
682 int ival = 0;
683
684 val = ParamChecker.checkFrequency(val);
685 coordJob.setFrequency(val);
686 TimeUnit tmp = (evalFreq.getVariable("timeunit") == null) ? TimeUnit.MINUTE : ((TimeUnit) evalFreq
687 .getVariable("timeunit"));
688 try {
689 Integer.parseInt(val);
690 }
691 catch (NumberFormatException ex) {
692 tmp=TimeUnit.CRON;
693 }
694
695 addAnAttribute("freq_timeunit", eAppXml, tmp.toString());
696 // TimeUnit
697 coordJob.setTimeUnit(CoordinatorJob.Timeunit.valueOf(tmp.toString()));
698 // End Of Duration
699 tmp = evalFreq.getVariable("endOfDuration") == null ? TimeUnit.NONE : ((TimeUnit) evalFreq
700 .getVariable("endOfDuration"));
701 addAnAttribute("end_of_duration", eAppXml, tmp.toString());
702 // coordJob.setEndOfDuration(tmp) // TODO: Add new attribute in Job bean
703
704 // Application name
705 if (this.coordName == null) {
706 String name = ELUtils.resolveAppName(eAppXml.getAttribute("name").getValue(), conf);
707 coordJob.setAppName(name);
708 }
709 else {
710 // this coord job is created from bundle
711 coordJob.setAppName(this.coordName);
712 }
713
714 // start time
715 val = resolveAttribute("start", eAppXml, evalNofuncs);
716 ParamChecker.checkDateOozieTZ(val, "start");
717 coordJob.setStartTime(DateUtils.parseDateOozieTZ(val));
718 // end time
719 val = resolveAttribute("end", eAppXml, evalNofuncs);
720 ParamChecker.checkDateOozieTZ(val, "end");
721 coordJob.setEndTime(DateUtils.parseDateOozieTZ(val));
722 // Time zone
723 val = resolveAttribute("timezone", eAppXml, evalNofuncs);
724 ParamChecker.checkTimeZone(val, "timezone");
725 coordJob.setTimeZone(val);
726
727 // controls
728 val = resolveTagContents("timeout", eAppXml.getChild("controls", eAppXml.getNamespace()), evalTimeout);
729 if (val != null && val != "") {
730 int t = Integer.parseInt(val);
731 tmp = (evalTimeout.getVariable("timeunit") == null) ? TimeUnit.MINUTE : ((TimeUnit) evalTimeout
732 .getVariable("timeunit"));
733 switch (tmp) {
734 case HOUR:
735 val = String.valueOf(t * 60);
736 break;
737 case DAY:
738 val = String.valueOf(t * 60 * 24);
739 break;
740 case MONTH:
741 val = String.valueOf(t * 60 * 24 * 30);
742 break;
743 default:
744 break;
745 }
746 }
747 else {
748 val = Services.get().getConf().get(CONF_DEFAULT_TIMEOUT_NORMAL);
749 }
750
751 ival = ParamChecker.checkInteger(val, "timeout");
752 if (ival < 0 || ival > Services.get().getConf().getInt(CONF_DEFAULT_MAX_TIMEOUT, 129600)) {
753 ival = Services.get().getConf().getInt(CONF_DEFAULT_MAX_TIMEOUT, 129600);
754 }
755 coordJob.setTimeout(ival);
756
757 val = resolveTagContents("concurrency", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
758 if (val == null || val.isEmpty()) {
759 val = Services.get().getConf().get(CONF_DEFAULT_CONCURRENCY, "1");
760 }
761 ival = ParamChecker.checkInteger(val, "concurrency");
762 coordJob.setConcurrency(ival);
763
764 val = resolveTagContents("throttle", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
765 if (val == null || val.isEmpty()) {
766 int defaultThrottle = Services.get().getConf().getInt(CONF_DEFAULT_THROTTLE, 12);
767 ival = defaultThrottle;
768 }
769 else {
770 ival = ParamChecker.checkInteger(val, "throttle");
771 }
772 int maxQueue = Services.get().getConf().getInt(CONF_QUEUE_SIZE, 10000);
773 float factor = Services.get().getConf().getFloat(CONF_MAT_THROTTLING_FACTOR, 0.10f);
774 int maxThrottle = (int) (maxQueue * factor);
775 if (ival > maxThrottle || ival < 1) {
776 ival = maxThrottle;
777 }
778 LOG.debug("max throttle " + ival);
779 coordJob.setMatThrottling(ival);
780
781 val = resolveTagContents("execution", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
782 if (val == "") {
783 val = Execution.FIFO.toString();
784 }
785 coordJob.setExecutionOrder(Execution.valueOf(val));
786 String[] acceptedVals = { Execution.LIFO.toString(), Execution.FIFO.toString(), Execution.LAST_ONLY.toString(),
787 Execution.NONE.toString()};
788 ParamChecker.isMember(val, acceptedVals, "execution");
789
790 // datasets
791 resolveTagContents("include", eAppXml.getChild("datasets", eAppXml.getNamespace()), evalNofuncs);
792 // for each data set
793 resolveDataSets(eAppXml);
794 HashMap<String, String> dataNameList = new HashMap<String, String>();
795 resolveIODataset(eAppXml);
796 resolveIOEvents(eAppXml, dataNameList);
797
798 resolveTagContents("app-path", eAppXml.getChild("action", eAppXml.getNamespace()).getChild("workflow",
799 eAppXml.getNamespace()), evalNofuncs);
800 // TODO: If action or workflow tag is missing, NullPointerException will
801 // occur
802 Element configElem = eAppXml.getChild("action", eAppXml.getNamespace()).getChild("workflow",
803 eAppXml.getNamespace()).getChild("configuration", eAppXml.getNamespace());
804 evalData = CoordELEvaluator.createELEvaluatorForDataEcho(conf, "coord-job-submit-data", dataNameList);
805 if (configElem != null) {
806 for (Element propElem : (List<Element>) configElem.getChildren("property", configElem.getNamespace())) {
807 resolveTagContents("name", propElem, evalData);
808 // Want to check the data-integrity but don't want to modify the
809 // XML
810 // for properties only
811 Element tmpProp = (Element) propElem.clone();
812 resolveTagContents("value", tmpProp, evalData);
813 }
814 }
815 evalSla = CoordELEvaluator.createELEvaluatorForDataAndConf(conf, "coord-sla-submit", dataNameList);
816 resolveSLA(eAppXml, coordJob);
817 return eAppXml;
818 }
819
820 /**
821 * Resolve SLA events
822 *
823 * @param eAppXml job XML
824 * @param coordJob coordinator job bean
825 * @throws CommandException thrown if failed to resolve sla events
826 */
827 private void resolveSLA(Element eAppXml, CoordinatorJobBean coordJob) throws CommandException {
828 Element eSla = XmlUtils.getSLAElement(eAppXml.getChild("action", eAppXml.getNamespace()));
829
830 if (eSla != null) {
831 String slaXml = XmlUtils.prettyPrint(eSla).toString();
832 try {
833 // EL evaluation
834 slaXml = evalSla.evaluate(slaXml, String.class);
835 // Validate against semantic SXD
836 XmlUtils.validateData(slaXml, SchemaName.SLA_ORIGINAL);
837 }
838 catch (Exception e) {
839 throw new CommandException(ErrorCode.E1004, "Validation ERROR :" + e.getMessage(), e);
840 }
841 }
842 }
843
844 /**
845 * Resolve input-events/data-in and output-events/data-out tags.
846 *
847 * @param eJobOrg : Job element
848 * @throws CoordinatorJobException thrown if failed to resolve input and output events
849 */
850 @SuppressWarnings("unchecked")
851 private void resolveIOEvents(Element eJobOrg, HashMap<String, String> dataNameList) throws CoordinatorJobException {
852 // Resolving input-events/data-in
853 // Clone the job and don't update anything in the original
854 Element eJob = (Element) eJobOrg.clone();
855 Element inputList = eJob.getChild("input-events", eJob.getNamespace());
856 if (inputList != null) {
857 TreeSet<String> eventNameSet = new TreeSet<String>();
858 for (Element dataIn : (List<Element>) inputList.getChildren("data-in", eJob.getNamespace())) {
859 String dataInName = dataIn.getAttributeValue("name");
860 dataNameList.put(dataInName, "data-in");
861 // check whether there is any duplicate data-in name
862 if (eventNameSet.contains(dataInName)) {
863 throw new RuntimeException("Duplicate dataIn name " + dataInName);
864 }
865 else {
866 eventNameSet.add(dataInName);
867 }
868 resolveTagContents("instance", dataIn, evalInst);
869 resolveTagContents("start-instance", dataIn, evalInst);
870 resolveTagContents("end-instance", dataIn, evalInst);
871
872 }
873 }
874 // Resolving output-events/data-out
875 Element outputList = eJob.getChild("output-events", eJob.getNamespace());
876 if (outputList != null) {
877 TreeSet<String> eventNameSet = new TreeSet<String>();
878 for (Element dataOut : (List<Element>) outputList.getChildren("data-out", eJob.getNamespace())) {
879 String dataOutName = dataOut.getAttributeValue("name");
880 dataNameList.put(dataOutName, "data-out");
881 // check whether there is any duplicate data-out name
882 if (eventNameSet.contains(dataOutName)) {
883 throw new RuntimeException("Duplicate dataIn name " + dataOutName);
884 }
885 else {
886 eventNameSet.add(dataOutName);
887 }
888 resolveTagContents("instance", dataOut, evalInst);
889
890 }
891 }
892
893 }
894
895 /**
896 * Resolve input-events/dataset and output-events/dataset tags.
897 *
898 * @param eJob : Job element
899 * @throws CoordinatorJobException thrown if failed to resolve input and output events
900 */
901 @SuppressWarnings("unchecked")
902 private void resolveIODataset(Element eAppXml) throws CoordinatorJobException {
903 // Resolving input-events/data-in
904 Element inputList = eAppXml.getChild("input-events", eAppXml.getNamespace());
905 if (inputList != null) {
906 for (Element dataIn : (List<Element>) inputList.getChildren("data-in", eAppXml.getNamespace())) {
907 resolveAttribute("dataset", dataIn, evalInst);
908
909 }
910 }
911 // Resolving output-events/data-out
912 Element outputList = eAppXml.getChild("output-events", eAppXml.getNamespace());
913 if (outputList != null) {
914 for (Element dataOut : (List<Element>) outputList.getChildren("data-out", eAppXml.getNamespace())) {
915 resolveAttribute("dataset", dataOut, evalInst);
916
917 }
918 }
919
920 }
921
922
923 /**
924 * Add an attribute into XML element.
925 *
926 * @param attrName :attribute name
927 * @param elem : Element to add attribute
928 * @param value :Value of attribute
929 */
930 private void addAnAttribute(String attrName, Element elem, String value) {
931 elem.setAttribute(attrName, value);
932 }
933
934 /**
935 * Resolve datasets using job configuration.
936 *
937 * @param eAppXml : Job Element XML
938 * @throws Exception thrown if failed to resolve datasets
939 */
940 @SuppressWarnings("unchecked")
941 private void resolveDataSets(Element eAppXml) throws Exception {
942 Element datasetList = eAppXml.getChild("datasets", eAppXml.getNamespace());
943 if (datasetList != null) {
944
945 List<Element> dsElems = datasetList.getChildren("dataset", eAppXml.getNamespace());
946 resolveDataSets(dsElems);
947 resolveTagContents("app-path", eAppXml.getChild("action", eAppXml.getNamespace()).getChild("workflow",
948 eAppXml.getNamespace()), evalNofuncs);
949 }
950 }
951
952 /**
953 * Resolve datasets using job configuration.
954 *
955 * @param dsElems : Data set XML element.
956 * @throws CoordinatorJobException thrown if failed to resolve datasets
957 */
958 private void resolveDataSets(List<Element> dsElems) throws CoordinatorJobException {
959 for (Element dsElem : dsElems) {
960 // Setting up default TimeUnit and EndOFDuraion
961 evalFreq.setVariable("timeunit", TimeUnit.MINUTE);
962 evalFreq.setVariable("endOfDuration", TimeUnit.NONE);
963
964 String val = resolveAttribute("frequency", dsElem, evalFreq);
965 int ival = ParamChecker.checkInteger(val, "frequency");
966 ParamChecker.checkGTZero(ival, "frequency");
967 addAnAttribute("freq_timeunit", dsElem, evalFreq.getVariable("timeunit") == null ? TimeUnit.MINUTE
968 .toString() : ((TimeUnit) evalFreq.getVariable("timeunit")).toString());
969 addAnAttribute("end_of_duration", dsElem, evalFreq.getVariable("endOfDuration") == null ? TimeUnit.NONE
970 .toString() : ((TimeUnit) evalFreq.getVariable("endOfDuration")).toString());
971 val = resolveAttribute("initial-instance", dsElem, evalNofuncs);
972 ParamChecker.checkDateOozieTZ(val, "initial-instance");
973 checkInitialInstance(val);
974 val = resolveAttribute("timezone", dsElem, evalNofuncs);
975 ParamChecker.checkTimeZone(val, "timezone");
976 resolveTagContents("uri-template", dsElem, evalNofuncs);
977 resolveTagContents("done-flag", dsElem, evalNofuncs);
978 }
979 }
980
981 /**
982 * Resolve the content of a tag.
983 *
984 * @param tagName : Tag name of job XML i.e. <timeout> 10 </timeout>
985 * @param elem : Element where the tag exists.
986 * @param eval : EL evealuator
987 * @return Resolved tag content.
988 * @throws CoordinatorJobException thrown if failed to resolve tag content
989 */
990 @SuppressWarnings("unchecked")
991 private String resolveTagContents(String tagName, Element elem, ELEvaluator eval) throws CoordinatorJobException {
992 String ret = "";
993 if (elem != null) {
994 for (Element tagElem : (List<Element>) elem.getChildren(tagName, elem.getNamespace())) {
995 if (tagElem != null) {
996 String updated;
997 try {
998 updated = CoordELFunctions.evalAndWrap(eval, tagElem.getText().trim());
999
1000 }
1001 catch (Exception e) {
1002 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e);
1003 }
1004 tagElem.removeContent();
1005 tagElem.addContent(updated);
1006 ret += updated;
1007 }
1008 }
1009 }
1010 return ret;
1011 }
1012
1013 /**
1014 * Resolve an attribute value.
1015 *
1016 * @param attrName : Attribute name.
1017 * @param elem : XML Element where attribute is defiend
1018 * @param eval : ELEvaluator used to resolve
1019 * @return Resolved attribute value
1020 * @throws CoordinatorJobException thrown if failed to resolve an attribute value
1021 */
1022 private String resolveAttribute(String attrName, Element elem, ELEvaluator eval) throws CoordinatorJobException {
1023 Attribute attr = elem.getAttribute(attrName);
1024 String val = null;
1025 if (attr != null) {
1026 try {
1027 val = CoordELFunctions.evalAndWrap(eval, attr.getValue().trim());
1028 }
1029 catch (Exception e) {
1030 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e);
1031 }
1032 attr.setValue(val);
1033 }
1034 return val;
1035 }
1036
1037 /**
1038 * Include referred datasets into XML.
1039 *
1040 * @param resolvedXml : Job XML element.
1041 * @param conf : Job configuration
1042 * @throws CoordinatorJobException thrown if failed to include referred datasets into XML
1043 */
1044 @SuppressWarnings("unchecked")
1045 protected void includeDataSets(Element resolvedXml, Configuration conf) throws CoordinatorJobException {
1046 Element datasets = resolvedXml.getChild("datasets", resolvedXml.getNamespace());
1047 Element allDataSets = new Element("all_datasets", resolvedXml.getNamespace());
1048 List<String> dsList = new ArrayList<String>();
1049 if (datasets != null) {
1050 for (Element includeElem : (List<Element>) datasets.getChildren("include", datasets.getNamespace())) {
1051 String incDSFile = includeElem.getTextTrim();
1052 includeOneDSFile(incDSFile, dsList, allDataSets, datasets.getNamespace());
1053 }
1054 for (Element e : (List<Element>) datasets.getChildren("dataset", datasets.getNamespace())) {
1055 String dsName = e.getAttributeValue("name");
1056 if (dsList.contains(dsName)) {// Override with this DS
1057 // Remove duplicate
1058 removeDataSet(allDataSets, dsName);
1059 }
1060 else {
1061 dsList.add(dsName);
1062 }
1063 allDataSets.addContent((Element) e.clone());
1064 }
1065 }
1066 insertDataSet(resolvedXml, allDataSets);
1067 resolvedXml.removeChild("datasets", resolvedXml.getNamespace());
1068 }
1069
1070 /**
1071 * Include one dataset file.
1072 *
1073 * @param incDSFile : Include data set filename.
1074 * @param dsList :List of dataset names to verify the duplicate.
1075 * @param allDataSets : Element that includes all dataset definitions.
1076 * @param dsNameSpace : Data set name space
1077 * @throws CoordinatorJobException thrown if failed to include one dataset file
1078 */
1079 @SuppressWarnings("unchecked")
1080 private void includeOneDSFile(String incDSFile, List<String> dsList, Element allDataSets, Namespace dsNameSpace)
1081 throws CoordinatorJobException {
1082 Element tmpDataSets = null;
1083 try {
1084 String dsXml = readDefinition(incDSFile);
1085 LOG.debug("DSFILE :" + incDSFile + "\n" + dsXml);
1086 tmpDataSets = XmlUtils.parseXml(dsXml);
1087 }
1088 catch (JDOMException e) {
1089 LOG.warn("Error parsing included dataset [{0}]. Message [{1}]", incDSFile, e.getMessage());
1090 throw new CoordinatorJobException(ErrorCode.E0700, e.getMessage());
1091 }
1092 resolveDataSets(tmpDataSets.getChildren("dataset"));
1093 for (Element e : (List<Element>) tmpDataSets.getChildren("dataset")) {
1094 String dsName = e.getAttributeValue("name");
1095 if (dsList.contains(dsName)) {
1096 throw new RuntimeException("Duplicate Dataset " + dsName);
1097 }
1098 dsList.add(dsName);
1099 Element tmp = (Element) e.clone();
1100 // TODO: Don't like to over-write the external/include DS's namespace
1101 tmp.setNamespace(dsNameSpace);
1102 tmp.getChild("uri-template").setNamespace(dsNameSpace);
1103 if (e.getChild("done-flag") != null) {
1104 tmp.getChild("done-flag").setNamespace(dsNameSpace);
1105 }
1106 allDataSets.addContent(tmp);
1107 }
1108 // nested include
1109 for (Element includeElem : (List<Element>) tmpDataSets.getChildren("include", tmpDataSets.getNamespace())) {
1110 String incFile = includeElem.getTextTrim();
1111 includeOneDSFile(incFile, dsList, allDataSets, dsNameSpace);
1112 }
1113 }
1114
1115 /**
1116 * Remove a dataset from a list of dataset.
1117 *
1118 * @param eDatasets : List of dataset
1119 * @param name : Dataset name to be removed.
1120 */
1121 @SuppressWarnings("unchecked")
1122 private static void removeDataSet(Element eDatasets, String name) {
1123 for (Element eDataset : (List<Element>) eDatasets.getChildren("dataset", eDatasets.getNamespace())) {
1124 if (eDataset.getAttributeValue("name").equals(name)) {
1125 eDataset.detach();
1126 return;
1127 }
1128 }
1129 throw new RuntimeException("undefined dataset: " + name);
1130 }
1131
1132 /**
1133 * Read coordinator definition.
1134 *
1135 * @param appPath application path.
1136 * @return coordinator definition.
1137 * @throws CoordinatorJobException thrown if the definition could not be read.
1138 */
1139 protected String readDefinition(String appPath) throws CoordinatorJobException {
1140 String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
1141 // Configuration confHadoop = CoordUtils.getHadoopConf(conf);
1142 try {
1143 URI uri = new URI(appPath);
1144 LOG.debug("user =" + user);
1145 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
1146 Configuration fsConf = has.createJobConf(uri.getAuthority());
1147 FileSystem fs = has.createFileSystem(user, uri, fsConf);
1148 Path appDefPath = null;
1149
1150 // app path could be a directory
1151 Path path = new Path(uri.getPath());
1152 // check file exists for dataset include file, app xml already checked
1153 if (!fs.exists(path)) {
1154 throw new URISyntaxException(path.toString(), "path not existed : " + path.toString());
1155 }
1156 if (!fs.isFile(path)) {
1157 appDefPath = new Path(path, COORDINATOR_XML_FILE);
1158 } else {
1159 appDefPath = path;
1160 }
1161
1162 Reader reader = new InputStreamReader(fs.open(appDefPath));
1163 StringWriter writer = new StringWriter();
1164 IOUtils.copyCharStream(reader, writer);
1165 return writer.toString();
1166 }
1167 catch (IOException ex) {
1168 LOG.warn("IOException :" + XmlUtils.prettyPrint(conf), ex);
1169 throw new CoordinatorJobException(ErrorCode.E1001, ex.getMessage(), ex);
1170 }
1171 catch (URISyntaxException ex) {
1172 LOG.warn("URISyException :" + ex.getMessage());
1173 throw new CoordinatorJobException(ErrorCode.E1002, appPath, ex.getMessage(), ex);
1174 }
1175 catch (HadoopAccessorException ex) {
1176 throw new CoordinatorJobException(ex);
1177 }
1178 catch (Exception ex) {
1179 LOG.warn("Exception :", ex);
1180 throw new CoordinatorJobException(ErrorCode.E1001, ex.getMessage(), ex);
1181 }
1182 }
1183
1184 /**
1185 * Write a coordinator job into database
1186 *
1187 *@param appXML : Coordinator definition xml
1188 * @param eJob : XML element of job
1189 * @param coordJob : Coordinator job bean
1190 * @return Job id
1191 * @throws CommandException thrown if unable to save coordinator job to db
1192 */
1193 protected String storeToDB(String appXML, Element eJob, CoordinatorJobBean coordJob) throws CommandException {
1194 String jobId = Services.get().get(UUIDService.class).generateId(ApplicationType.COORDINATOR);
1195 coordJob.setId(jobId);
1196
1197 coordJob.setAppPath(conf.get(OozieClient.COORDINATOR_APP_PATH));
1198 coordJob.setCreatedTime(new Date());
1199 coordJob.setUser(conf.get(OozieClient.USER_NAME));
1200 String group = ConfigUtils.getWithDeprecatedCheck(conf, OozieClient.JOB_ACL, OozieClient.GROUP_NAME, null);
1201 coordJob.setGroup(group);
1202 coordJob.setConf(XmlUtils.prettyPrint(conf).toString());
1203 coordJob.setJobXml(XmlUtils.prettyPrint(eJob).toString());
1204 coordJob.setLastActionNumber(0);
1205 coordJob.setLastModifiedTime(new Date());
1206
1207 if (!dryrun) {
1208 coordJob.setLastModifiedTime(new Date());
1209 try {
1210 CoordJobQueryExecutor.getInstance().insert(coordJob);
1211 }
1212 catch (JPAExecutorException jpaee) {
1213 coordJob.setId(null);
1214 coordJob.setStatus(CoordinatorJob.Status.FAILED);
1215 throw new CommandException(jpaee);
1216 }
1217 }
1218 return jobId;
1219 }
1220
1221 /*
1222 * this method checks if the initial-instance specified for a particular
1223 is not a date earlier than the oozie server default Jan 01, 1970 00:00Z UTC
1224 */
1225 private void checkInitialInstance(String val) throws CoordinatorJobException, IllegalArgumentException {
1226 Date initialInstance, givenInstance;
1227 try {
1228 initialInstance = DateUtils.parseDateUTC("1970-01-01T00:00Z");
1229 givenInstance = DateUtils.parseDateOozieTZ(val);
1230 }
1231 catch (Exception e) {
1232 throw new IllegalArgumentException("Unable to parse dataset initial-instance string '" + val +
1233 "' to Date object. ",e);
1234 }
1235 if(givenInstance.compareTo(initialInstance) < 0) {
1236 throw new CoordinatorJobException(ErrorCode.E1021, "Dataset initial-instance " + val +
1237 " is earlier than the default initial instance " + DateUtils.formatDateOozieTZ(initialInstance));
1238 }
1239 }
1240
1241 /* (non-Javadoc)
1242 * @see org.apache.oozie.command.XCommand#getEntityKey()
1243 */
1244 @Override
1245 public String getEntityKey() {
1246 return null;
1247 }
1248
1249 /* (non-Javadoc)
1250 * @see org.apache.oozie.command.XCommand#isLockRequired()
1251 */
1252 @Override
1253 protected boolean isLockRequired() {
1254 return false;
1255 }
1256
1257 /* (non-Javadoc)
1258 * @see org.apache.oozie.command.XCommand#loadState()
1259 */
1260 @Override
1261 protected void loadState() throws CommandException {
1262 jpaService = Services.get().get(JPAService.class);
1263 if (jpaService == null) {
1264 throw new CommandException(ErrorCode.E0610);
1265 }
1266 coordJob = new CoordinatorJobBean();
1267 if (this.bundleId != null) {
1268 // this coord job is created from bundle
1269 coordJob.setBundleId(this.bundleId);
1270 // first use bundle id if submit thru bundle
1271 logInfo.setParameter(DagXLogInfoService.JOB, this.bundleId);
1272 LogUtils.setLogInfo(logInfo);
1273 }
1274 if (this.coordName != null) {
1275 // this coord job is created from bundle
1276 coordJob.setAppName(this.coordName);
1277 }
1278 setJob(coordJob);
1279
1280 }
1281
1282 /* (non-Javadoc)
1283 * @see org.apache.oozie.command.XCommand#verifyPrecondition()
1284 */
1285 @Override
1286 protected void verifyPrecondition() throws CommandException {
1287
1288 }
1289
1290 /* (non-Javadoc)
1291 * @see org.apache.oozie.command.TransitionXCommand#notifyParent()
1292 */
1293 @Override
1294 public void notifyParent() throws CommandException {
1295 // update bundle action
1296 if (coordJob.getBundleId() != null) {
1297 LOG.debug("Updating bundle record: " + coordJob.getBundleId() + " for coord id: " + coordJob.getId());
1298 BundleStatusUpdateXCommand bundleStatusUpdate = new BundleStatusUpdateXCommand(coordJob, prevStatus);
1299 bundleStatusUpdate.call();
1300 }
1301 }
1302
1303 /* (non-Javadoc)
1304 * @see org.apache.oozie.command.TransitionXCommand#updateJob()
1305 */
1306 @Override
1307 public void updateJob() throws CommandException {
1308 }
1309
1310 /* (non-Javadoc)
1311 * @see org.apache.oozie.command.TransitionXCommand#getJob()
1312 */
1313 @Override
1314 public Job getJob() {
1315 return coordJob;
1316 }
1317
1318 @Override
1319 public void performWrites() throws CommandException {
1320 }
1321 }