001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.command.bundle;
019
020 import java.io.IOException;
021 import java.io.InputStreamReader;
022 import java.io.Reader;
023 import java.io.StringReader;
024 import java.io.StringWriter;
025 import java.net.URI;
026 import java.net.URISyntaxException;
027 import java.util.Date;
028 import java.util.HashSet;
029 import java.util.List;
030 import java.util.Map;
031 import java.util.Set;
032
033 import javax.xml.transform.stream.StreamSource;
034 import javax.xml.validation.Validator;
035
036 import org.apache.hadoop.conf.Configuration;
037 import org.apache.hadoop.fs.FileSystem;
038 import org.apache.hadoop.fs.Path;
039 import org.apache.oozie.BundleJobBean;
040 import org.apache.oozie.ErrorCode;
041 import org.apache.oozie.client.Job;
042 import org.apache.oozie.client.OozieClient;
043 import org.apache.oozie.command.CommandException;
044 import org.apache.oozie.command.PreconditionException;
045 import org.apache.oozie.command.SubmitTransitionXCommand;
046 import org.apache.oozie.executor.jpa.BundleJobQueryExecutor;
047 import org.apache.oozie.service.HadoopAccessorException;
048 import org.apache.oozie.service.HadoopAccessorService;
049 import org.apache.oozie.service.SchemaService;
050 import org.apache.oozie.service.Services;
051 import org.apache.oozie.service.UUIDService;
052 import org.apache.oozie.service.SchemaService.SchemaName;
053 import org.apache.oozie.service.UUIDService.ApplicationType;
054 import org.apache.oozie.util.ConfigUtils;
055 import org.apache.oozie.util.DateUtils;
056 import org.apache.oozie.util.ELEvaluator;
057 import org.apache.oozie.util.ELUtils;
058 import org.apache.oozie.util.IOUtils;
059 import org.apache.oozie.util.InstrumentUtils;
060 import org.apache.oozie.util.LogUtils;
061 import org.apache.oozie.util.ParamChecker;
062 import org.apache.oozie.util.PropertiesUtils;
063 import org.apache.oozie.util.XConfiguration;
064 import org.apache.oozie.util.XmlUtils;
065 import org.apache.oozie.util.ParameterVerifier;
066 import org.jdom.Attribute;
067 import org.jdom.Element;
068 import org.jdom.JDOMException;
069 import org.xml.sax.SAXException;
070
071 /**
072 * This Command will submit the bundle.
073 */
074 public class BundleSubmitXCommand extends SubmitTransitionXCommand {
075
076 private Configuration conf;
077 public static final String CONFIG_DEFAULT = "bundle-config-default.xml";
078 public static final String BUNDLE_XML_FILE = "bundle.xml";
079 private final BundleJobBean bundleBean = new BundleJobBean();
080 private String jobId;
081 private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>();
082 private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
083
084 static {
085 String[] badUserProps = { PropertiesUtils.YEAR, PropertiesUtils.MONTH, PropertiesUtils.DAY,
086 PropertiesUtils.HOUR, PropertiesUtils.MINUTE, PropertiesUtils.DAYS, PropertiesUtils.HOURS,
087 PropertiesUtils.MINUTES, PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB,
088 PropertiesUtils.TB, PropertiesUtils.PB, PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN,
089 PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN, PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS };
090 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES);
091
092 String[] badDefaultProps = { PropertiesUtils.HADOOP_USER};
093 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
094 PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
095 }
096
097 /**
098 * Constructor to create the bundle submit command.
099 *
100 * @param conf configuration for bundle job
101 */
102 public BundleSubmitXCommand(Configuration conf) {
103 super("bundle_submit", "bundle_submit", 1);
104 this.conf = ParamChecker.notNull(conf, "conf");
105 }
106
107 /**
108 * Constructor to create the bundle submit command.
109 *
110 * @param dryrun true if dryrun is enable
111 * @param conf configuration for bundle job
112 */
113 public BundleSubmitXCommand(boolean dryrun, Configuration conf) {
114 this(conf);
115 this.dryrun = dryrun;
116 }
117
118 /* (non-Javadoc)
119 * @see org.apache.oozie.command.SubmitTransitionXCommand#submit()
120 */
121 @Override
122 protected String submit() throws CommandException {
123 LOG.info("STARTED Bundle Submit");
124 try {
125 InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
126
127 ParameterVerifier.verifyParameters(conf, XmlUtils.parseXml(bundleBean.getOrigJobXml()));
128
129 String jobXmlWithNoComment = XmlUtils.removeComments(this.bundleBean.getOrigJobXml().toString());
130 // Resolving all variables in the job properties.
131 // This ensures the Hadoop Configuration semantics is preserved.
132 XConfiguration resolvedVarsConf = new XConfiguration();
133 for (Map.Entry<String, String> entry : conf) {
134 resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
135 }
136 conf = resolvedVarsConf;
137
138 String resolvedJobXml = resolvedVars(jobXmlWithNoComment, conf);
139
140 //verify the uniqueness of coord names
141 verifyCoordNameUnique(resolvedJobXml);
142 this.jobId = storeToDB(bundleBean, resolvedJobXml);
143 LogUtils.setLogInfo(bundleBean, logInfo);
144
145 if (dryrun) {
146 Date startTime = bundleBean.getStartTime();
147 long startTimeMilli = startTime.getTime();
148 long endTimeMilli = startTimeMilli + (3600 * 1000);
149 Date jobEndTime = bundleBean.getEndTime();
150 Date endTime = new Date(endTimeMilli);
151 if (endTime.compareTo(jobEndTime) > 0) {
152 endTime = jobEndTime;
153 }
154 jobId = bundleBean.getId();
155 LOG.info("[" + jobId + "]: Update status to PREP");
156 bundleBean.setStatus(Job.Status.PREP);
157 try {
158 new XConfiguration(new StringReader(bundleBean.getConf()));
159 }
160 catch (IOException e1) {
161 LOG.warn("Configuration parse error. read from DB :" + bundleBean.getConf(), e1);
162 }
163 String output = bundleBean.getJobXml() + System.getProperty("line.separator");
164 return output;
165 }
166 else {
167 if (bundleBean.getKickoffTime() == null) {
168 // If there is no KickOffTime, default kickoff is NOW.
169 LOG.debug("Since kickoff time is not defined for job id " + jobId
170 + ". Queuing and BundleStartXCommand immediately after submission");
171 queue(new BundleStartXCommand(jobId));
172 }
173 }
174 }
175 catch (Exception ex) {
176 throw new CommandException(ErrorCode.E1310, ex.getMessage(), ex);
177 }
178 LOG.info("ENDED Bundle Submit");
179 return this.jobId;
180 }
181
182 /* (non-Javadoc)
183 * @see org.apache.oozie.command.TransitionXCommand#notifyParent()
184 */
185 @Override
186 public void notifyParent() throws CommandException {
187 }
188
189 /* (non-Javadoc)
190 * @see org.apache.oozie.command.XCommand#getEntityKey()
191 */
192 @Override
193 public String getEntityKey() {
194 return null;
195 }
196
197 /* (non-Javadoc)
198 * @see org.apache.oozie.command.XCommand#isLockRequired()
199 */
200 @Override
201 protected boolean isLockRequired() {
202 return false;
203 }
204
205 @Override
206 protected void loadState() throws CommandException {
207 }
208
209 @Override
210 protected void verifyPrecondition() throws CommandException, PreconditionException {
211 }
212
213 @Override
214 protected void eagerLoadState() throws CommandException {
215 }
216
217 @Override
218 protected void eagerVerifyPrecondition() throws CommandException, PreconditionException {
219 try {
220 mergeDefaultConfig();
221 String appXml = readAndValidateXml();
222 bundleBean.setOrigJobXml(appXml);
223 LOG.debug("jobXml after initial validation " + XmlUtils.prettyPrint(appXml).toString());
224 }
225 catch (BundleJobException ex) {
226 LOG.warn("BundleJobException: ", ex);
227 throw new CommandException(ex);
228 }
229 catch (IllegalArgumentException iex) {
230 LOG.warn("IllegalArgumentException: ", iex);
231 throw new CommandException(ErrorCode.E1310, iex.getMessage(), iex);
232 }
233 catch (Exception ex) {
234 LOG.warn("Exception: ", ex);
235 throw new CommandException(ErrorCode.E1310, ex.getMessage(), ex);
236 }
237 }
238
239 /**
240 * Merge default configuration with user-defined configuration.
241 *
242 * @throws CommandException thrown if failed to merge configuration
243 */
244 protected void mergeDefaultConfig() throws CommandException {
245 Path configDefault = null;
246 try {
247 String bundleAppPathStr = conf.get(OozieClient.BUNDLE_APP_PATH);
248 Path bundleAppPath = new Path(bundleAppPathStr);
249 String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
250 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
251 Configuration fsConf = has.createJobConf(bundleAppPath.toUri().getAuthority());
252 FileSystem fs = has.createFileSystem(user, bundleAppPath.toUri(), fsConf);
253
254 // app path could be a directory
255 if (!fs.isFile(bundleAppPath)) {
256 configDefault = new Path(bundleAppPath, CONFIG_DEFAULT);
257 } else {
258 configDefault = new Path(bundleAppPath.getParent(), CONFIG_DEFAULT);
259 }
260
261 if (fs.exists(configDefault)) {
262 Configuration defaultConf = new XConfiguration(fs.open(configDefault));
263 PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
264 XConfiguration.injectDefaults(defaultConf, conf);
265 }
266 else {
267 LOG.info("configDefault Doesn't exist " + configDefault);
268 }
269 PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
270 }
271 catch (IOException e) {
272 throw new CommandException(ErrorCode.E0702, e.getMessage() + " : Problem reading default config "
273 + configDefault, e);
274 }
275 catch (HadoopAccessorException e) {
276 throw new CommandException(e);
277 }
278 LOG.debug("Merged CONF :" + XmlUtils.prettyPrint(conf).toString());
279 }
280
281 /**
282 * Read the application XML and validate against bundle Schema
283 *
284 * @return validated bundle XML
285 * @throws BundleJobException thrown if failed to read or validate xml
286 */
287 private String readAndValidateXml() throws BundleJobException {
288 String appPath = ParamChecker.notEmpty(conf.get(OozieClient.BUNDLE_APP_PATH), OozieClient.BUNDLE_APP_PATH);
289 String bundleXml = readDefinition(appPath);
290 validateXml(bundleXml);
291 return bundleXml;
292 }
293
294 /**
295 * Read bundle definition.
296 *
297 * @param appPath application path.
298 * @param user user name.
299 * @param group group name.
300 * @return bundle definition.
301 * @throws BundleJobException thrown if the definition could not be read.
302 */
303 protected String readDefinition(String appPath) throws BundleJobException {
304 String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
305 //Configuration confHadoop = CoordUtils.getHadoopConf(conf);
306 try {
307 URI uri = new URI(appPath);
308 LOG.debug("user =" + user);
309 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
310 Configuration fsConf = has.createJobConf(uri.getAuthority());
311 FileSystem fs = has.createFileSystem(user, uri, fsConf);
312 Path appDefPath = null;
313
314 // app path could be a directory
315 Path path = new Path(uri.getPath());
316 if (!fs.isFile(path)) {
317 appDefPath = new Path(path, BUNDLE_XML_FILE);
318 } else {
319 appDefPath = path;
320 }
321
322 Reader reader = new InputStreamReader(fs.open(appDefPath));
323 StringWriter writer = new StringWriter();
324 IOUtils.copyCharStream(reader, writer);
325 return writer.toString();
326 }
327 catch (IOException ex) {
328 LOG.warn("IOException :" + XmlUtils.prettyPrint(conf), ex);
329 throw new BundleJobException(ErrorCode.E1301, ex.getMessage(), ex);
330 }
331 catch (URISyntaxException ex) {
332 LOG.warn("URISyException :" + ex.getMessage());
333 throw new BundleJobException(ErrorCode.E1302, appPath, ex.getMessage(), ex);
334 }
335 catch (HadoopAccessorException ex) {
336 throw new BundleJobException(ex);
337 }
338 catch (Exception ex) {
339 LOG.warn("Exception :", ex);
340 throw new BundleJobException(ErrorCode.E1301, ex.getMessage(), ex);
341 }
342 }
343
344 /**
345 * Validate against Bundle XSD file
346 *
347 * @param xmlContent input bundle xml
348 * @throws BundleJobException thrown if failed to validate xml
349 */
350 private void validateXml(String xmlContent) throws BundleJobException {
351 javax.xml.validation.Schema schema = Services.get().get(SchemaService.class).getSchema(SchemaName.BUNDLE);
352 Validator validator = schema.newValidator();
353 try {
354 validator.validate(new StreamSource(new StringReader(xmlContent)));
355 }
356 catch (SAXException ex) {
357 LOG.warn("SAXException :", ex);
358 throw new BundleJobException(ErrorCode.E0701, ex.getMessage(), ex);
359 }
360 catch (IOException ex) {
361 LOG.warn("IOException :", ex);
362 throw new BundleJobException(ErrorCode.E0702, ex.getMessage(), ex);
363 }
364 }
365
366 /**
367 * Write a Bundle Job into database
368 *
369 * @param Bundle job bean
370 * @return job id
371 * @throws CommandException thrown if failed to store bundle job bean to db
372 */
373 private String storeToDB(BundleJobBean bundleJob, String resolvedJobXml) throws CommandException {
374 try {
375 jobId = Services.get().get(UUIDService.class).generateId(ApplicationType.BUNDLE);
376
377 bundleJob.setId(jobId);
378 String name = XmlUtils.parseXml(bundleBean.getOrigJobXml()).getAttributeValue("name");
379 name = ELUtils.resolveAppName(name, conf);
380 bundleJob.setAppName(name);
381 bundleJob.setAppPath(conf.get(OozieClient.BUNDLE_APP_PATH));
382 // bundleJob.setStatus(BundleJob.Status.PREP); //This should be set in parent class.
383 bundleJob.setCreatedTime(new Date());
384 bundleJob.setUser(conf.get(OozieClient.USER_NAME));
385 String group = ConfigUtils.getWithDeprecatedCheck(conf, OozieClient.JOB_ACL, OozieClient.GROUP_NAME, null);
386 bundleJob.setGroup(group);
387 bundleJob.setConf(XmlUtils.prettyPrint(conf).toString());
388 bundleJob.setJobXml(resolvedJobXml);
389 Element jobElement = XmlUtils.parseXml(resolvedJobXml);
390 Element controlsElement = jobElement.getChild("controls", jobElement.getNamespace());
391 if (controlsElement != null) {
392 Element kickoffTimeElement = controlsElement.getChild("kick-off-time", jobElement.getNamespace());
393 if (kickoffTimeElement != null && !kickoffTimeElement.getValue().isEmpty()) {
394 Date kickoffTime = DateUtils.parseDateOozieTZ(kickoffTimeElement.getValue());
395 bundleJob.setKickoffTime(kickoffTime);
396 }
397 }
398 bundleJob.setLastModifiedTime(new Date());
399
400 if (!dryrun) {
401 BundleJobQueryExecutor.getInstance().insert(bundleJob);
402 }
403 }
404 catch (Exception ex) {
405 throw new CommandException(ErrorCode.E1301, ex.getMessage(), ex);
406 }
407 return jobId;
408 }
409
410 /* (non-Javadoc)
411 * @see org.apache.oozie.command.TransitionXCommand#getJob()
412 */
413 @Override
414 public Job getJob() {
415 return bundleBean;
416 }
417
418 /**
419 * Resolve job xml with conf
420 *
421 * @param bundleXml bundle job xml
422 * @param conf job configuration
423 * @return resolved job xml
424 * @throws BundleJobException thrown if failed to resolve variables
425 */
426 private String resolvedVars(String bundleXml, Configuration conf) throws BundleJobException {
427 try {
428 ELEvaluator eval = createEvaluator(conf);
429 return eval.evaluate(bundleXml, String.class);
430 }
431 catch (Exception e) {
432 throw new BundleJobException(ErrorCode.E1004, e.getMessage(), e);
433 }
434 }
435
436 /**
437 * Create ELEvaluator
438 *
439 * @param conf job configuration
440 * @return ELEvaluator the evaluator for el function
441 * @throws BundleJobException thrown if failed to create evaluator
442 */
443 public ELEvaluator createEvaluator(Configuration conf) throws BundleJobException {
444 ELEvaluator eval;
445 ELEvaluator.Context context;
446 try {
447 context = new ELEvaluator.Context();
448 eval = new ELEvaluator(context);
449 for (Map.Entry<String, String> entry : conf) {
450 eval.setVariable(entry.getKey(), entry.getValue());
451 }
452 }
453 catch (Exception e) {
454 throw new BundleJobException(ErrorCode.E1004, e.getMessage(), e);
455 }
456 return eval;
457 }
458
459 /**
460 * Verify the uniqueness of coordinator names
461 *
462 * @param resolved job xml
463 * @throws CommandException thrown if failed to verify the uniqueness of coordinator names
464 */
465 @SuppressWarnings("unchecked")
466 private Void verifyCoordNameUnique(String resolvedJobXml) throws CommandException {
467 Set<String> set = new HashSet<String>();
468 try {
469 Element bAppXml = XmlUtils.parseXml(resolvedJobXml);
470 List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace());
471 for (Element elem : coordElems) {
472 Attribute name = elem.getAttribute("name");
473 if (name != null) {
474 if (set.contains(name.getValue())) {
475 throw new CommandException(ErrorCode.E1304, name);
476 }
477 set.add(name.getValue());
478 }
479 else {
480 throw new CommandException(ErrorCode.E1305);
481 }
482 }
483 }
484 catch (JDOMException jex) {
485 throw new CommandException(ErrorCode.E1301, jex.getMessage(), jex);
486 }
487
488 return null;
489 }
490
491 /* (non-Javadoc)
492 * @see org.apache.oozie.command.TransitionXCommand#updateJob()
493 */
494 @Override
495 public void updateJob() throws CommandException {
496 }
497
498 @Override
499 public void performWrites() throws CommandException {
500 }
501 }