001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.cli;
019
020 import com.google.common.annotations.VisibleForTesting;
021 import org.apache.commons.cli.CommandLine;
022 import org.apache.commons.cli.Option;
023 import org.apache.commons.cli.OptionBuilder;
024 import org.apache.commons.cli.OptionGroup;
025 import org.apache.commons.cli.Options;
026 import org.apache.commons.cli.ParseException;
027 import org.apache.oozie.BuildInfo;
028 import org.apache.oozie.client.AuthOozieClient;
029 import org.apache.oozie.client.BulkResponse;
030 import org.apache.oozie.client.BundleJob;
031 import org.apache.oozie.client.CoordinatorAction;
032 import org.apache.oozie.client.CoordinatorJob;
033 import org.apache.oozie.client.OozieClient;
034 import org.apache.oozie.client.OozieClient.SYSTEM_MODE;
035 import org.apache.oozie.client.OozieClientException;
036 import org.apache.oozie.client.WorkflowAction;
037 import org.apache.oozie.client.WorkflowJob;
038 import org.apache.oozie.client.XOozieClient;
039 import org.apache.oozie.client.rest.RestConstants;
040 import org.w3c.dom.DOMException;
041 import org.w3c.dom.Document;
042 import org.w3c.dom.Element;
043 import org.w3c.dom.Node;
044 import org.w3c.dom.NodeList;
045 import org.w3c.dom.Text;
046 import org.xml.sax.SAXException;
047
048 import javax.xml.XMLConstants;
049 import javax.xml.parsers.DocumentBuilder;
050 import javax.xml.parsers.DocumentBuilderFactory;
051 import javax.xml.parsers.ParserConfigurationException;
052 import javax.xml.transform.stream.StreamSource;
053 import javax.xml.validation.Schema;
054 import javax.xml.validation.SchemaFactory;
055 import javax.xml.validation.Validator;
056 import java.io.File;
057 import java.io.FileInputStream;
058 import java.io.FileReader;
059 import java.io.IOException;
060 import java.io.InputStream;
061 import java.io.PrintStream;
062 import java.text.SimpleDateFormat;
063 import java.util.ArrayList;
064 import java.util.Date;
065 import java.util.List;
066 import java.util.Locale;
067 import java.util.Map;
068 import java.util.Properties;
069 import java.util.TimeZone;
070 import java.util.concurrent.Callable;
071 import java.util.regex.Matcher;
072 import java.util.regex.Pattern;
073
074 /**
075 * Oozie command line utility.
076 */
077 public class OozieCLI {
078 public static final String ENV_OOZIE_URL = "OOZIE_URL";
079 public static final String ENV_OOZIE_DEBUG = "OOZIE_DEBUG";
080 public static final String ENV_OOZIE_TIME_ZONE = "OOZIE_TIMEZONE";
081 public static final String ENV_OOZIE_AUTH = "OOZIE_AUTH";
082 public static final String OOZIE_RETRY_COUNT = "oozie.connection.retry.count";
083 public static final String WS_HEADER_PREFIX = "header:";
084
085 public static final String HELP_CMD = "help";
086 public static final String VERSION_CMD = "version";
087 public static final String JOB_CMD = "job";
088 public static final String JOBS_CMD = "jobs";
089 public static final String ADMIN_CMD = "admin";
090 public static final String VALIDATE_CMD = "validate";
091 public static final String SLA_CMD = "sla";
092 public static final String PIG_CMD = "pig";
093 public static final String HIVE_CMD = "hive";
094 public static final String SQOOP_CMD = "sqoop";
095 public static final String MR_CMD = "mapreduce";
096 public static final String INFO_CMD = "info";
097
098 public static final String OOZIE_OPTION = "oozie";
099 public static final String CONFIG_OPTION = "config";
100 public static final String SUBMIT_OPTION = "submit";
101 public static final String OFFSET_OPTION = "offset";
102 public static final String START_OPTION = "start";
103 public static final String RUN_OPTION = "run";
104 public static final String DRYRUN_OPTION = "dryrun";
105 public static final String SUSPEND_OPTION = "suspend";
106 public static final String RESUME_OPTION = "resume";
107 public static final String KILL_OPTION = "kill";
108 public static final String CHANGE_OPTION = "change";
109 public static final String CHANGE_VALUE_OPTION = "value";
110 public static final String RERUN_OPTION = "rerun";
111 public static final String INFO_OPTION = "info";
112 public static final String LOG_OPTION = "log";
113 public static final String ACTION_OPTION = "action";
114 public static final String DEFINITION_OPTION = "definition";
115 public static final String CONFIG_CONTENT_OPTION = "configcontent";
116 public static final String SQOOP_COMMAND_OPTION = "command";
117 public static final String SHOWDIFF_OPTION = "diff";
118 public static final String UPDATE_OPTION = "update";
119 public static final String IGNORE_OPTION = "ignore";
120
121 public static final String DO_AS_OPTION = "doas";
122
123 public static final String LEN_OPTION = "len";
124 public static final String FILTER_OPTION = "filter";
125 public static final String JOBTYPE_OPTION = "jobtype";
126 public static final String SYSTEM_MODE_OPTION = "systemmode";
127 public static final String VERSION_OPTION = "version";
128 public static final String STATUS_OPTION = "status";
129 public static final String LOCAL_TIME_OPTION = "localtime";
130 public static final String TIME_ZONE_OPTION = "timezone";
131 public static final String QUEUE_DUMP_OPTION = "queuedump";
132 public static final String RERUN_COORD_OPTION = "coordinator";
133 public static final String DATE_OPTION = "date";
134 public static final String RERUN_REFRESH_OPTION = "refresh";
135 public static final String RERUN_NOCLEANUP_OPTION = "nocleanup";
136 public static final String ORDER_OPTION = "order";
137
138 public static final String UPDATE_SHARELIB_OPTION = "sharelibupdate";
139
140 public static final String LIST_SHARELIB_LIB_OPTION = "shareliblist";
141
142
143
144 public static final String AUTH_OPTION = "auth";
145
146 public static final String VERBOSE_OPTION = "verbose";
147 public static final String VERBOSE_DELIMITER = "\t";
148 public static final String DEBUG_OPTION = "debug";
149
150 public static final String SCRIPTFILE_OPTION = "file";
151
152 public static final String INFO_TIME_ZONES_OPTION = "timezones";
153
154 public static final String BULK_OPTION = "bulk";
155
156 public static final String AVAILABLE_SERVERS_OPTION = "servers";
157
158 public static final String ALL_WORKFLOWS_FOR_COORD_ACTION = "allruns";
159
160 private static final String[] OOZIE_HELP = {
161 "the env variable '" + ENV_OOZIE_URL + "' is used as default value for the '-" + OOZIE_OPTION + "' option",
162 "the env variable '" + ENV_OOZIE_TIME_ZONE + "' is used as default value for the '-" + TIME_ZONE_OPTION + "' option",
163 "the env variable '" + ENV_OOZIE_AUTH + "' is used as default value for the '-" + AUTH_OPTION + "' option",
164 "custom headers for Oozie web services can be specified using '-D" + WS_HEADER_PREFIX + "NAME=VALUE'" };
165
166 private static final String RULER;
167 private static final int LINE_WIDTH = 132;
168
169 private static final int RETRY_COUNT = 4;
170
171 private boolean used;
172
173 private static final String INSTANCE_SEPARATOR = "#";
174
175 private static final String MAPRED_MAPPER = "mapred.mapper.class";
176 private static final String MAPRED_MAPPER_2 = "mapreduce.map.class";
177 private static final String MAPRED_REDUCER = "mapred.reducer.class";
178 private static final String MAPRED_REDUCER_2 = "mapreduce.reduce.class";
179 private static final String MAPRED_INPUT = "mapred.input.dir";
180 private static final String MAPRED_OUTPUT = "mapred.output.dir";
181
182 private static final Pattern GMT_OFFSET_SHORTEN_PATTERN = Pattern.compile("(.* )GMT((?:-|\\+)\\d{2}:\\d{2})");
183
184 static {
185 StringBuilder sb = new StringBuilder();
186 for (int i = 0; i < LINE_WIDTH; i++) {
187 sb.append("-");
188 }
189 RULER = sb.toString();
190 }
191
192 /**
193 * Entry point for the Oozie CLI when invoked from the command line.
194 * <p/>
195 * Upon completion this method exits the JVM with '0' (success) or '-1' (failure).
196 *
197 * @param args options and arguments for the Oozie CLI.
198 */
199 public static void main(String[] args) {
200 if (!System.getProperties().containsKey(AuthOozieClient.USE_AUTH_TOKEN_CACHE_SYS_PROP)) {
201 System.setProperty(AuthOozieClient.USE_AUTH_TOKEN_CACHE_SYS_PROP, "true");
202 }
203 System.exit(new OozieCLI().run(args));
204 }
205
206 /**
207 * Create an Oozie CLI instance.
208 */
209 public OozieCLI() {
210 used = false;
211 }
212
213 /**
214 * Return Oozie CLI top help lines.
215 *
216 * @return help lines.
217 */
218 protected String[] getCLIHelp() {
219 return OOZIE_HELP;
220 }
221
222 /**
223 * Add authentication specific options to oozie cli
224 *
225 * @param options the collection of options to add auth options
226 */
227 protected void addAuthOptions(Options options) {
228 Option auth = new Option(AUTH_OPTION, true, "select authentication type [SIMPLE|KERBEROS]");
229 options.addOption(auth);
230 }
231
232 /**
233 * Create option for command line option 'admin'
234 * @return admin options
235 */
236 protected Options createAdminOptions() {
237 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
238 Option system_mode = new Option(SYSTEM_MODE_OPTION, true,
239 "Supported in Oozie-2.0 or later versions ONLY. Change oozie system mode [NORMAL|NOWEBSERVICE|SAFEMODE]");
240 Option status = new Option(STATUS_OPTION, false, "show the current system status");
241 Option version = new Option(VERSION_OPTION, false, "show Oozie server build version");
242 Option queuedump = new Option(QUEUE_DUMP_OPTION, false, "show Oozie server queue elements");
243 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
244 Option availServers = new Option(AVAILABLE_SERVERS_OPTION, false, "list available Oozie servers"
245 + " (more than one only if HA is enabled)");
246 Option sharelibUpdate = new Option(UPDATE_SHARELIB_OPTION, false, "Update server to use a newer version of sharelib");
247
248 Option sharelib = new Option(LIST_SHARELIB_LIB_OPTION, false,
249 "List available sharelib that can be specified in a workflow action");
250 sharelib.setOptionalArg(true);
251
252 Options adminOptions = new Options();
253 adminOptions.addOption(oozie);
254 adminOptions.addOption(doAs);
255 OptionGroup group = new OptionGroup();
256 group.addOption(system_mode);
257 group.addOption(status);
258 group.addOption(version);
259 group.addOption(queuedump);
260 group.addOption(availServers);
261 group.addOption(sharelibUpdate);
262 group.addOption(sharelib);
263 adminOptions.addOptionGroup(group);
264 addAuthOptions(adminOptions);
265 return adminOptions;
266 }
267
268 /**
269 * Create option for command line option 'job'
270 * @return job options
271 */
272 protected Options createJobOptions() {
273 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
274 Option config = new Option(CONFIG_OPTION, true, "job configuration file '.xml' or '.properties'");
275 Option submit = new Option(SUBMIT_OPTION, false, "submit a job");
276 Option run = new Option(RUN_OPTION, false, "run a job");
277 Option debug = new Option(DEBUG_OPTION, false, "Use debug mode to see debugging statements on stdout");
278 Option rerun = new Option(RERUN_OPTION, true,
279 "rerun a job (coordinator requires -action or -date, bundle requires -coordinator or -date)");
280 Option dryrun = new Option(DRYRUN_OPTION, false, "Dryrun a workflow (since 3.3.2) or coordinator (since 2.0) job without"
281 + " actually executing it");
282 Option update = new Option(UPDATE_OPTION, true, "Update coord definition and properties");
283 Option showdiff = new Option(SHOWDIFF_OPTION, true,
284 "Show diff of the new coord definition and properties with the existing one (default true)");
285 Option start = new Option(START_OPTION, true, "start a job");
286 Option suspend = new Option(SUSPEND_OPTION, true, "suspend a job");
287 Option resume = new Option(RESUME_OPTION, true, "resume a job");
288 Option kill = new Option(KILL_OPTION, true, "kill a job (coordinator can mention -action or -date)");
289 Option change = new Option(CHANGE_OPTION, true, "change a coordinator job");
290 Option changeValue = new Option(CHANGE_VALUE_OPTION, true,
291 "new endtime/concurrency/pausetime value for changing a coordinator job");
292 Option info = new Option(INFO_OPTION, true, "info of a job");
293 Option offset = new Option(OFFSET_OPTION, true, "job info offset of actions (default '1', requires -info)");
294 Option len = new Option(LEN_OPTION, true, "number of actions (default TOTAL ACTIONS, requires -info)");
295 Option filter = new Option(FILTER_OPTION, true,
296 "<key><comparator><value>[;<key><comparator><value>]*\n"
297 + "(All Coordinator actions satisfying the filters will be retreived).\n"
298 + "key: status or nominaltime\n"
299 + "comparator: =, !=, <, <=, >, >=. = is used as OR and others as AND\n"
300 + "status: values are valid status like SUCCEEDED, KILLED etc. Only = and != apply for status\n"
301 + "nominaltime: time of format yyyy-MM-dd'T'HH:mm'Z'");
302 Option order = new Option(ORDER_OPTION, true,
303 "order to show coord actions (default ascending order, 'desc' for descending order, requires -info)");
304 Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (same as passing your time zone to -" +
305 TIME_ZONE_OPTION + "). Overrides -" + TIME_ZONE_OPTION + " option");
306 Option timezone = new Option(TIME_ZONE_OPTION, true,
307 "use time zone with the specified ID (default GMT).\nSee 'oozie info -timezones' for a list");
308 Option log = new Option(LOG_OPTION, true, "job log");
309 Option logFilter = new Option(
310 RestConstants.LOG_FILTER_OPTION, true,
311 "job log search parameter. Can be specified as -logfilter opt1=val1;opt2=val1;opt3=val1. "
312 + "Supported options are recent, start, end, loglevel, text, limit and debug");
313 Option definition = new Option(DEFINITION_OPTION, true, "job definition");
314 Option config_content = new Option(CONFIG_CONTENT_OPTION, true, "job configuration");
315 Option verbose = new Option(VERBOSE_OPTION, false, "verbose mode");
316 Option action = new Option(ACTION_OPTION, true,
317 "coordinator rerun on action ids (requires -rerun); coordinator log retrieval on action ids (requires -log)");
318 Option date = new Option(DATE_OPTION, true,
319 "coordinator/bundle rerun on action dates (requires -rerun); coordinator log retrieval on action dates (requires -log)");
320 Option rerun_coord = new Option(RERUN_COORD_OPTION, true, "bundle rerun on coordinator names (requires -rerun)");
321 Option rerun_refresh = new Option(RERUN_REFRESH_OPTION, false,
322 "re-materialize the coordinator rerun actions (requires -rerun)");
323 Option rerun_nocleanup = new Option(RERUN_NOCLEANUP_OPTION, false,
324 "do not clean up output-events of the coordiantor rerun actions (requires -rerun)");
325 Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
326 "set/override value for given property").create("D");
327 Option getAllWorkflows = new Option(ALL_WORKFLOWS_FOR_COORD_ACTION, false,
328 "Get workflow jobs corresponding to a coordinator action including all the reruns");
329 Option ignore = new Option(IGNORE_OPTION, true,
330 "change status of a coordinator job or action to IGNORED"
331 + " (-action required to ignore coord actions)");
332
333 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
334
335 OptionGroup actions = new OptionGroup();
336 actions.addOption(submit);
337 actions.addOption(start);
338 actions.addOption(run);
339 actions.addOption(dryrun);
340 actions.addOption(suspend);
341 actions.addOption(resume);
342 actions.addOption(kill);
343 actions.addOption(change);
344 actions.addOption(update);
345 actions.addOption(info);
346 actions.addOption(rerun);
347 actions.addOption(log);
348 actions.addOption(definition);
349 actions.addOption(config_content);
350 actions.addOption(ignore);
351 actions.setRequired(true);
352 Options jobOptions = new Options();
353 jobOptions.addOption(oozie);
354 jobOptions.addOption(doAs);
355 jobOptions.addOption(config);
356 jobOptions.addOption(property);
357 jobOptions.addOption(changeValue);
358 jobOptions.addOption(localtime);
359 jobOptions.addOption(timezone);
360 jobOptions.addOption(verbose);
361 jobOptions.addOption(debug);
362 jobOptions.addOption(offset);
363 jobOptions.addOption(len);
364 jobOptions.addOption(filter);
365 jobOptions.addOption(order);
366 jobOptions.addOption(action);
367 jobOptions.addOption(date);
368 jobOptions.addOption(rerun_coord);
369 jobOptions.addOption(rerun_refresh);
370 jobOptions.addOption(rerun_nocleanup);
371 jobOptions.addOption(getAllWorkflows);
372 jobOptions.addOptionGroup(actions);
373 jobOptions.addOption(logFilter);
374 addAuthOptions(jobOptions);
375 jobOptions.addOption(showdiff);
376
377 //Needed to make dryrun and update mutually exclusive options
378 OptionGroup updateOption = new OptionGroup();
379 updateOption.addOption(dryrun);
380 jobOptions.addOptionGroup(updateOption);
381 return jobOptions;
382 }
383
384 /**
385 * Create option for command line option 'jobs'
386 * @return jobs options
387 */
388 protected Options createJobsOptions() {
389 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
390 Option start = new Option(OFFSET_OPTION, true, "jobs offset (default '1')");
391 Option jobtype = new Option(JOBTYPE_OPTION, true,
392 "job type ('Supported in Oozie-2.0 or later versions ONLY - 'coordinator' or 'bundle' or 'wf'(default))");
393 Option len = new Option(LEN_OPTION, true, "number of jobs (default '100')");
394 Option filter = new Option(FILTER_OPTION, true, "user=<U>\\;name=<N>\\;group=<G>\\;status=<S>\\;frequency=<F>\\;unit=<M> " +
395 "(Valid unit values are 'months', 'days', 'hours' or 'minutes'.)");
396 Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (same as passing your time zone to -" +
397 TIME_ZONE_OPTION + "). Overrides -" + TIME_ZONE_OPTION + " option");
398 Option timezone = new Option(TIME_ZONE_OPTION, true,
399 "use time zone with the specified ID (default GMT).\nSee 'oozie info -timezones' for a list");
400 Option verbose = new Option(VERBOSE_OPTION, false, "verbose mode");
401 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
402 Option bulkMonitor = new Option(BULK_OPTION, true, "key-value pairs to filter bulk jobs response. e.g. bundle=<B>\\;" +
403 "coordinators=<C>\\;actionstatus=<S>\\;startcreatedtime=<SC>\\;endcreatedtime=<EC>\\;" +
404 "startscheduledtime=<SS>\\;endscheduledtime=<ES>\\; bundle, coordinators and actionstatus can be multiple comma separated values" +
405 "bundle and coordinators can be id(s) or appName(s) of those jobs. Specifying bundle is mandatory, other params are optional");
406 start.setType(Integer.class);
407 len.setType(Integer.class);
408 Options jobsOptions = new Options();
409 jobsOptions.addOption(oozie);
410 jobsOptions.addOption(doAs);
411 jobsOptions.addOption(localtime);
412 jobsOptions.addOption(timezone);
413 jobsOptions.addOption(start);
414 jobsOptions.addOption(len);
415 jobsOptions.addOption(oozie);
416 jobsOptions.addOption(filter);
417 jobsOptions.addOption(jobtype);
418 jobsOptions.addOption(verbose);
419 jobsOptions.addOption(bulkMonitor);
420 addAuthOptions(jobsOptions);
421 return jobsOptions;
422 }
423
424 /**
425 * Create option for command line option 'sla'
426 *
427 * @return sla options
428 */
429 protected Options createSlaOptions() {
430 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
431 Option start = new Option(OFFSET_OPTION, true, "start offset (default '0')");
432 Option len = new Option(LEN_OPTION, true, "number of results (default '100', max '1000')");
433 Option filter = new Option(FILTER_OPTION, true, "filter of SLA events. e.g., jobid=<J>\\;appname=<A>");
434 start.setType(Integer.class);
435 len.setType(Integer.class);
436 Options slaOptions = new Options();
437 slaOptions.addOption(start);
438 slaOptions.addOption(len);
439 slaOptions.addOption(filter);
440 slaOptions.addOption(oozie);
441 addAuthOptions(slaOptions);
442 return slaOptions;
443 }
444
445 /**
446 * Create option for command line option 'pig' or 'hive'
447 * @return pig or hive options
448 */
449 @SuppressWarnings("static-access")
450 protected Options createScriptLanguageOptions(String jobType) {
451 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
452 Option config = new Option(CONFIG_OPTION, true, "job configuration file '.properties'");
453 Option file = new Option(SCRIPTFILE_OPTION, true, jobType + " script");
454 Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
455 "set/override value for given property").create("D");
456 Option params = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
457 "set parameters for script").create("P");
458 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
459 Options Options = new Options();
460 Options.addOption(oozie);
461 Options.addOption(doAs);
462 Options.addOption(config);
463 Options.addOption(property);
464 Options.addOption(params);
465 Options.addOption(file);
466 addAuthOptions(Options);
467 return Options;
468 }
469
470 /**
471 * Create option for command line option 'sqoop'
472 * @return sqoop options
473 */
474 @SuppressWarnings("static-access")
475 protected Options createSqoopCLIOptions() {
476 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
477 Option config = new Option(CONFIG_OPTION, true, "job configuration file '.properties'");
478 Option command = OptionBuilder.withArgName(SQOOP_COMMAND_OPTION).hasArgs().withValueSeparator().withDescription(
479 "sqoop command").create(SQOOP_COMMAND_OPTION);
480 Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
481 "set/override value for given property").create("D");
482 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
483 Options Options = new Options();
484 Options.addOption(oozie);
485 Options.addOption(doAs);
486 Options.addOption(config);
487 Options.addOption(property);
488 Options.addOption(command);
489 addAuthOptions(Options);
490 return Options;
491 }
492
493 /**
494 * Create option for command line option 'info'
495 * @return info options
496 */
497 protected Options createInfoOptions() {
498 Option timezones = new Option(INFO_TIME_ZONES_OPTION, false, "display a list of available time zones");
499 Options infoOptions = new Options();
500 infoOptions.addOption(timezones);
501 return infoOptions;
502 }
503
504 /**
505 * Create option for command line option 'mapreduce'
506 * @return mapreduce options
507 */
508 @SuppressWarnings("static-access")
509 protected Options createMROptions() {
510 Option oozie = new Option(OOZIE_OPTION, true, "Oozie URL");
511 Option config = new Option(CONFIG_OPTION, true, "job configuration file '.properties'");
512 Option property = OptionBuilder.withArgName("property=value").hasArgs(2).withValueSeparator().withDescription(
513 "set/override value for given property").create("D");
514 Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user");
515 Options mrOptions = new Options();
516 mrOptions.addOption(oozie);
517 mrOptions.addOption(doAs);
518 mrOptions.addOption(config);
519 mrOptions.addOption(property);
520 addAuthOptions(mrOptions);
521 return mrOptions;
522 }
523
524 /**
525 * Run a CLI programmatically.
526 * <p/>
527 * It does not exit the JVM.
528 * <p/>
529 * A CLI instance can be used only once.
530 *
531 * @param args options and arguments for the Oozie CLI.
532 * @return '0' (success), '-1' (failure).
533 */
534 public synchronized int run(String[] args) {
535 if (used) {
536 throw new IllegalStateException("CLI instance already used");
537 }
538 used = true;
539 final CLIParser parser = getCLIParser();
540 try {
541 final CLIParser.Command command = parser.parse(args);
542
543 String doAsUser = command.getCommandLine().getOptionValue(DO_AS_OPTION);
544
545 if (doAsUser != null) {
546 OozieClient.doAs(doAsUser, new Callable<Void>() {
547 @Override
548 public Void call() throws Exception {
549 processCommand(parser, command);
550 return null;
551 }
552 });
553 }
554 else {
555 processCommand(parser, command);
556 }
557 return 0;
558 }
559 catch (OozieCLIException ex) {
560 System.err.println("Error: " + ex.getMessage());
561 return -1;
562 }
563 catch (ParseException ex) {
564 System.err.println("Invalid sub-command: " + ex.getMessage());
565 System.err.println();
566 System.err.println(parser.shortHelp());
567 return -1;
568 }
569 catch (Exception ex) {
570 ex.printStackTrace();
571 System.err.println(ex.getMessage());
572 return -1;
573 }
574 }
575
576 @VisibleForTesting
577 public CLIParser getCLIParser(){
578 CLIParser parser = new CLIParser(OOZIE_OPTION, getCLIHelp());
579 parser.addCommand(HELP_CMD, "", "display usage for all commands or specified command", new Options(), false);
580 parser.addCommand(VERSION_CMD, "", "show client version", new Options(), false);
581 parser.addCommand(JOB_CMD, "", "job operations", createJobOptions(), false);
582 parser.addCommand(JOBS_CMD, "", "jobs status", createJobsOptions(), false);
583 parser.addCommand(ADMIN_CMD, "", "admin operations", createAdminOptions(), false);
584 parser.addCommand(VALIDATE_CMD, "", "validate a workflow XML file", new Options(), true);
585 parser.addCommand(SLA_CMD, "", "sla operations (Deprecated with Oozie 4.0)", createSlaOptions(), false);
586 parser.addCommand(PIG_CMD, "-X ", "submit a pig job, everything after '-X' are pass-through parameters to pig, any '-D' "
587 + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(PIG_CMD), true);
588 parser.addCommand(HIVE_CMD, "-X ", "submit a hive job, everything after '-X' are pass-through parameters to hive, any '-D' "
589 + "arguments after '-X' are put in <configuration>", createScriptLanguageOptions(HIVE_CMD), true);
590 parser.addCommand(SQOOP_CMD, "-X ", "submit a sqoop job, everything after '-X' are pass-through parameters " +
591 "to sqoop, any '-D' arguments after '-X' are put in <configuration>", createSqoopCLIOptions(), true);
592 parser.addCommand(INFO_CMD, "", "get more detailed info about specific topics", createInfoOptions(), false);
593 parser.addCommand(MR_CMD, "", "submit a mapreduce job", createMROptions(), false);
594 return parser;
595 }
596
597 public void processCommand(CLIParser parser, CLIParser.Command command) throws Exception {
598 if (command.getName().equals(HELP_CMD)) {
599 parser.showHelp(command.getCommandLine());
600 }
601 else if (command.getName().equals(JOB_CMD)) {
602 jobCommand(command.getCommandLine());
603 }
604 else if (command.getName().equals(JOBS_CMD)) {
605 jobsCommand(command.getCommandLine());
606 }
607 else if (command.getName().equals(ADMIN_CMD)) {
608 adminCommand(command.getCommandLine());
609 }
610 else if (command.getName().equals(VERSION_CMD)) {
611 versionCommand();
612 }
613 else if (command.getName().equals(VALIDATE_CMD)) {
614 validateCommand(command.getCommandLine());
615 }
616 else if (command.getName().equals(SLA_CMD)) {
617 slaCommand(command.getCommandLine());
618 }
619 else if (command.getName().equals(PIG_CMD)) {
620 scriptLanguageCommand(command.getCommandLine(), PIG_CMD);
621 }
622 else if (command.getName().equals(HIVE_CMD)) {
623 scriptLanguageCommand(command.getCommandLine(), HIVE_CMD);
624 }
625 else if (command.getName().equals(SQOOP_CMD)) {
626 sqoopCommand(command.getCommandLine());
627 }
628 else if (command.getName().equals(INFO_CMD)) {
629 infoCommand(command.getCommandLine());
630 }
631 else if (command.getName().equals(MR_CMD)){
632 mrCommand(command.getCommandLine());
633 }
634 }
635 protected String getOozieUrl(CommandLine commandLine) {
636 String url = commandLine.getOptionValue(OOZIE_OPTION);
637 if (url == null) {
638 url = System.getenv(ENV_OOZIE_URL);
639 if (url == null) {
640 throw new IllegalArgumentException(
641 "Oozie URL is not available neither in command option or in the environment");
642 }
643 }
644 return url;
645 }
646
647 private String getTimeZoneId(CommandLine commandLine)
648 {
649 if (commandLine.hasOption(LOCAL_TIME_OPTION)) {
650 return null;
651 }
652 if (commandLine.hasOption(TIME_ZONE_OPTION)) {
653 return commandLine.getOptionValue(TIME_ZONE_OPTION);
654 }
655 String timeZoneId = System.getenv(ENV_OOZIE_TIME_ZONE);
656 if (timeZoneId != null) {
657 return timeZoneId;
658 }
659 return "GMT";
660 }
661
662 // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
663 private Properties parse(InputStream is, Properties conf) throws IOException {
664 try {
665 DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
666 // ignore all comments inside the xml file
667 docBuilderFactory.setIgnoringComments(true);
668 DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
669 Document doc = builder.parse(is);
670 return parseDocument(doc, conf);
671 }
672 catch (SAXException e) {
673 throw new IOException(e);
674 }
675 catch (ParserConfigurationException e) {
676 throw new IOException(e);
677 }
678 }
679
680 // Canibalized from Hadoop <code>Configuration.loadResource()</code>.
681 private Properties parseDocument(Document doc, Properties conf) throws IOException {
682 try {
683 Element root = doc.getDocumentElement();
684 if (!"configuration".equals(root.getTagName())) {
685 throw new RuntimeException("bad conf file: top-level element not <configuration>");
686 }
687 NodeList props = root.getChildNodes();
688 for (int i = 0; i < props.getLength(); i++) {
689 Node propNode = props.item(i);
690 if (!(propNode instanceof Element)) {
691 continue;
692 }
693 Element prop = (Element) propNode;
694 if (!"property".equals(prop.getTagName())) {
695 throw new RuntimeException("bad conf file: element not <property>");
696 }
697 NodeList fields = prop.getChildNodes();
698 String attr = null;
699 String value = null;
700 for (int j = 0; j < fields.getLength(); j++) {
701 Node fieldNode = fields.item(j);
702 if (!(fieldNode instanceof Element)) {
703 continue;
704 }
705 Element field = (Element) fieldNode;
706 if ("name".equals(field.getTagName()) && field.hasChildNodes()) {
707 attr = ((Text) field.getFirstChild()).getData();
708 }
709 if ("value".equals(field.getTagName()) && field.hasChildNodes()) {
710 value = ((Text) field.getFirstChild()).getData();
711 }
712 }
713
714 if (attr != null && value != null) {
715 conf.setProperty(attr, value);
716 }
717 }
718 return conf;
719 }
720 catch (DOMException e) {
721 throw new IOException(e);
722 }
723 }
724
725 private Properties getConfiguration(OozieClient wc, CommandLine commandLine) throws IOException {
726 if (!isConfigurationSpecified(wc, commandLine)) {
727 throw new IOException("configuration is not specified");
728 }
729 Properties conf = wc.createConfiguration();
730 String configFile = commandLine.getOptionValue(CONFIG_OPTION);
731 if (configFile != null) {
732 File file = new File(configFile);
733 if (!file.exists()) {
734 throw new IOException("configuration file [" + configFile + "] not found");
735 }
736 if (configFile.endsWith(".properties")) {
737 conf.load(new FileReader(file));
738 }
739 else if (configFile.endsWith(".xml")) {
740 parse(new FileInputStream(configFile), conf);
741 }
742 else {
743 throw new IllegalArgumentException("configuration must be a '.properties' or a '.xml' file");
744 }
745 }
746 if (commandLine.hasOption("D")) {
747 Properties commandLineProperties = commandLine.getOptionProperties("D");
748 conf.putAll(commandLineProperties);
749 }
750 return conf;
751 }
752
753 /**
754 * Check if configuration has specified
755 * @param wc
756 * @param commandLine
757 * @return
758 * @throws IOException
759 */
760 private boolean isConfigurationSpecified(OozieClient wc, CommandLine commandLine) throws IOException {
761 boolean isConf = false;
762 String configFile = commandLine.getOptionValue(CONFIG_OPTION);
763 if (configFile == null) {
764 isConf = false;
765 }
766 else {
767 isConf = new File(configFile).exists();
768 }
769 if (commandLine.hasOption("D")) {
770 isConf = true;
771 }
772 return isConf;
773 }
774
775 /**
776 * @param commandLine command line string.
777 * @return change value specified by -value.
778 * @throws OozieCLIException
779 */
780 private String getChangeValue(CommandLine commandLine) throws OozieCLIException {
781 String changeValue = commandLine.getOptionValue(CHANGE_VALUE_OPTION);
782
783 if (changeValue == null) {
784 throw new OozieCLIException("-value option needs to be specified for -change option");
785 }
786
787 return changeValue;
788 }
789
790 protected void addHeader(OozieClient wc) {
791 for (Map.Entry entry : System.getProperties().entrySet()) {
792 String key = (String) entry.getKey();
793 if (key.startsWith(WS_HEADER_PREFIX)) {
794 String header = key.substring(WS_HEADER_PREFIX.length());
795 wc.setHeader(header, (String) entry.getValue());
796 }
797 }
798 }
799
800 /**
801 * Get auth option from command line
802 *
803 * @param commandLine the command line object
804 * @return auth option
805 */
806 protected String getAuthOption(CommandLine commandLine) {
807 String authOpt = commandLine.getOptionValue(AUTH_OPTION);
808 if (authOpt == null) {
809 authOpt = System.getenv(ENV_OOZIE_AUTH);
810 }
811 if (commandLine.hasOption(DEBUG_OPTION)) {
812 System.out.println(" Auth type : " + authOpt);
813 }
814 return authOpt;
815 }
816
817 /**
818 * Create a OozieClient.
819 * <p/>
820 * It injects any '-Dheader:' as header to the the {@link org.apache.oozie.client.OozieClient}.
821 *
822 * @param commandLine the parsed command line options.
823 * @return a pre configured eXtended workflow client.
824 * @throws OozieCLIException thrown if the OozieClient could not be configured.
825 */
826 protected OozieClient createOozieClient(CommandLine commandLine) throws OozieCLIException {
827 return createXOozieClient(commandLine);
828 }
829
830 /**
831 * Create a XOozieClient.
832 * <p/>
833 * It injects any '-Dheader:' as header to the the {@link org.apache.oozie.client.OozieClient}.
834 *
835 * @param commandLine the parsed command line options.
836 * @return a pre configured eXtended workflow client.
837 * @throws OozieCLIException thrown if the XOozieClient could not be configured.
838 */
839 protected XOozieClient createXOozieClient(CommandLine commandLine) throws OozieCLIException {
840 XOozieClient wc = new AuthOozieClient(getOozieUrl(commandLine), getAuthOption(commandLine));
841 addHeader(wc);
842 setDebugMode(wc,commandLine.hasOption(DEBUG_OPTION));
843 setRetryCount(wc);
844 return wc;
845 }
846
847 protected void setDebugMode(OozieClient wc, boolean debugOpt) {
848
849 String debug = System.getenv(ENV_OOZIE_DEBUG);
850 if (debug != null && !debug.isEmpty()) {
851 int debugVal = 0;
852 try {
853 debugVal = Integer.parseInt(debug.trim());
854 }
855 catch (Exception ex) {
856 System.out.println("Unable to parse the debug settings. May be not an integer [" + debug + "]");
857 ex.printStackTrace();
858 }
859 wc.setDebugMode(debugVal);
860 }
861 else if(debugOpt){ // CLI argument "-debug" used
862 wc.setDebugMode(1);
863 }
864 }
865
866 protected void setRetryCount(OozieClient wc) {
867 String retryCount = System.getProperty(OOZIE_RETRY_COUNT);
868 if (retryCount != null && !retryCount.isEmpty()) {
869 try {
870 int retry = Integer.parseInt(retryCount.trim());
871 wc.setRetryCount(retry);
872 }
873 catch (Exception ex) {
874 System.err.println("Unable to parse the retry settings. May be not an integer [" + retryCount + "]");
875 ex.printStackTrace();
876 }
877 }
878 }
879
880 private static String JOB_ID_PREFIX = "job: ";
881
882 private void jobCommand(CommandLine commandLine) throws IOException, OozieCLIException {
883 XOozieClient wc = createXOozieClient(commandLine);
884
885 List<String> options = new ArrayList<String>();
886 for (Option option : commandLine.getOptions()) {
887 options.add(option.getOpt());
888 }
889
890 try {
891 if (options.contains(SUBMIT_OPTION)) {
892 System.out.println(JOB_ID_PREFIX + wc.submit(getConfiguration(wc, commandLine)));
893 }
894 else if (options.contains(START_OPTION)) {
895 wc.start(commandLine.getOptionValue(START_OPTION));
896 }
897 else if (options.contains(DRYRUN_OPTION) && !options.contains(UPDATE_OPTION)) {
898 String dryrunStr = wc.dryrun(getConfiguration(wc, commandLine));
899 if (dryrunStr.equals("OK")) { // workflow
900 System.out.println("OK");
901 } else { // coordinator
902 String[] dryrunStrs = dryrunStr.split("action for new instance");
903 int arraysize = dryrunStrs.length;
904 System.out.println("***coordJob after parsing: ***");
905 System.out.println(dryrunStrs[0]);
906 int aLen = dryrunStrs.length - 1;
907 if (aLen < 0) {
908 aLen = 0;
909 }
910 System.out.println("***total coord actions is " + aLen + " ***");
911 for (int i = 1; i <= arraysize - 1; i++) {
912 System.out.println(RULER);
913 System.out.println("coordAction instance: " + i + ":");
914 System.out.println(dryrunStrs[i]);
915 }
916 }
917 }
918 else if (options.contains(SUSPEND_OPTION)) {
919 wc.suspend(commandLine.getOptionValue(SUSPEND_OPTION));
920 }
921 else if (options.contains(RESUME_OPTION)) {
922 wc.resume(commandLine.getOptionValue(RESUME_OPTION));
923 }
924 else if (options.contains(IGNORE_OPTION)) {
925 String ignoreScope = null;
926 if (options.contains(ACTION_OPTION)) {
927 ignoreScope = commandLine.getOptionValue(ACTION_OPTION);
928 if (ignoreScope == null || ignoreScope.isEmpty()) {
929 throw new OozieCLIException("-" + ACTION_OPTION + " is empty");
930 }
931 }
932 printCoordActionsStatus(wc.ignore(commandLine.getOptionValue(IGNORE_OPTION), ignoreScope));
933 }
934 else if (options.contains(KILL_OPTION)) {
935 if (commandLine.getOptionValue(KILL_OPTION).contains("-C")
936 && (options.contains(DATE_OPTION) || options.contains(ACTION_OPTION))) {
937 String coordJobId = commandLine.getOptionValue(KILL_OPTION);
938 String scope = null;
939 String rangeType = null;
940 if (options.contains(DATE_OPTION) && options.contains(ACTION_OPTION)) {
941 throw new OozieCLIException("Invalid options provided for rerun: either" + DATE_OPTION + " or "
942 + ACTION_OPTION + " expected. Don't use both at the same time.");
943 }
944 if (options.contains(DATE_OPTION)) {
945 rangeType = RestConstants.JOB_COORD_SCOPE_DATE;
946 scope = commandLine.getOptionValue(DATE_OPTION);
947 }
948 else if (options.contains(ACTION_OPTION)) {
949 rangeType = RestConstants.JOB_COORD_SCOPE_ACTION;
950 scope = commandLine.getOptionValue(ACTION_OPTION);
951 }
952 else {
953 throw new OozieCLIException("Invalid options provided for rerun: " + DATE_OPTION + " or "
954 + ACTION_OPTION + " expected.");
955 }
956 printCoordActions(wc.kill(coordJobId, rangeType, scope));
957 }
958 else {
959 wc.kill(commandLine.getOptionValue(KILL_OPTION));
960 }
961 }
962 else if (options.contains(CHANGE_OPTION)) {
963 wc.change(commandLine.getOptionValue(CHANGE_OPTION), getChangeValue(commandLine));
964 }
965 else if (options.contains(RUN_OPTION)) {
966 System.out.println(JOB_ID_PREFIX + wc.run(getConfiguration(wc, commandLine)));
967 }
968 else if (options.contains(RERUN_OPTION)) {
969 if (commandLine.getOptionValue(RERUN_OPTION).contains("-W")) {
970 if (isConfigurationSpecified(wc, commandLine)) {
971 wc.reRun(commandLine.getOptionValue(RERUN_OPTION), getConfiguration(wc, commandLine));
972 }
973 else {
974 wc.reRun(commandLine.getOptionValue(RERUN_OPTION), new Properties());
975 }
976 }
977 else if (commandLine.getOptionValue(RERUN_OPTION).contains("-B")) {
978 String bundleJobId = commandLine.getOptionValue(RERUN_OPTION);
979 String coordScope = null;
980 String dateScope = null;
981 boolean refresh = false;
982 boolean noCleanup = false;
983 if (options.contains(ACTION_OPTION)) {
984 throw new OozieCLIException("Invalid options provided for bundle rerun. " + ACTION_OPTION
985 + " is not valid for bundle rerun");
986 }
987 if (options.contains(DATE_OPTION)) {
988 dateScope = commandLine.getOptionValue(DATE_OPTION);
989 }
990
991 if (options.contains(RERUN_COORD_OPTION)) {
992 coordScope = commandLine.getOptionValue(RERUN_COORD_OPTION);
993 }
994
995 if (options.contains(RERUN_REFRESH_OPTION)) {
996 refresh = true;
997 }
998 if (options.contains(RERUN_NOCLEANUP_OPTION)) {
999 noCleanup = true;
1000 }
1001 wc.reRunBundle(bundleJobId, coordScope, dateScope, refresh, noCleanup);
1002 if (coordScope != null && !coordScope.isEmpty()) {
1003 System.out.println("Coordinators [" + coordScope + "] of bundle " + bundleJobId
1004 + " are scheduled to rerun on date ranges [" + dateScope + "].");
1005 }
1006 else {
1007 System.out.println("All coordinators of bundle " + bundleJobId
1008 + " are scheduled to rerun on the date ranges [" + dateScope + "].");
1009 }
1010 }
1011 else {
1012 String coordJobId = commandLine.getOptionValue(RERUN_OPTION);
1013 String scope = null;
1014 String rerunType = null;
1015 boolean refresh = false;
1016 boolean noCleanup = false;
1017 if (options.contains(DATE_OPTION) && options.contains(ACTION_OPTION)) {
1018 throw new OozieCLIException("Invalid options provided for rerun: either" + DATE_OPTION + " or "
1019 + ACTION_OPTION + " expected. Don't use both at the same time.");
1020 }
1021 if (options.contains(DATE_OPTION)) {
1022 rerunType = RestConstants.JOB_COORD_SCOPE_DATE;
1023 scope = commandLine.getOptionValue(DATE_OPTION);
1024 }
1025 else if (options.contains(ACTION_OPTION)) {
1026 rerunType = RestConstants.JOB_COORD_SCOPE_ACTION;
1027 scope = commandLine.getOptionValue(ACTION_OPTION);
1028 }
1029 else {
1030 throw new OozieCLIException("Invalid options provided for rerun: " + DATE_OPTION + " or "
1031 + ACTION_OPTION + " expected.");
1032 }
1033 if (options.contains(RERUN_REFRESH_OPTION)) {
1034 refresh = true;
1035 }
1036 if (options.contains(RERUN_NOCLEANUP_OPTION)) {
1037 noCleanup = true;
1038 }
1039 printCoordActions(wc.reRunCoord(coordJobId, rerunType, scope, refresh, noCleanup));
1040 }
1041 }
1042 else if (options.contains(INFO_OPTION)) {
1043 String timeZoneId = getTimeZoneId(commandLine);
1044 final String optionValue = commandLine.getOptionValue(INFO_OPTION);
1045 if (optionValue.endsWith("-B")) {
1046 String filter = commandLine.getOptionValue(FILTER_OPTION);
1047 if (filter != null) {
1048 throw new OozieCLIException("Filter option is currently not supported for a Bundle job");
1049 }
1050 printBundleJob(wc.getBundleJobInfo(optionValue), timeZoneId,
1051 options.contains(VERBOSE_OPTION));
1052 }
1053 else if (optionValue.endsWith("-C")) {
1054 String s = commandLine.getOptionValue(OFFSET_OPTION);
1055 int start = Integer.parseInt((s != null) ? s : "-1");
1056 s = commandLine.getOptionValue(LEN_OPTION);
1057 int len = Integer.parseInt((s != null) ? s : "-1");
1058 String filter = commandLine.getOptionValue(FILTER_OPTION);
1059 String order = commandLine.getOptionValue(ORDER_OPTION);
1060 printCoordJob(wc.getCoordJobInfo(optionValue, filter, start, len, order), timeZoneId,
1061 options.contains(VERBOSE_OPTION));
1062 }
1063 else if (optionValue.contains("-C@")) {
1064 if (options.contains(ALL_WORKFLOWS_FOR_COORD_ACTION)) {
1065 printWfsForCoordAction(wc.getWfsForCoordAction(optionValue), timeZoneId);
1066 }
1067 else {
1068 String filter = commandLine.getOptionValue(FILTER_OPTION);
1069 if (filter != null) {
1070 throw new OozieCLIException("Filter option is not supported for a Coordinator action");
1071 }
1072 printCoordAction(wc.getCoordActionInfo(optionValue), timeZoneId);
1073 }
1074 }
1075 else if (optionValue.contains("-W@")) {
1076 String filter = commandLine.getOptionValue(FILTER_OPTION);
1077 if (filter != null) {
1078 throw new OozieCLIException("Filter option is not supported for a Workflow action");
1079 }
1080 printWorkflowAction(wc.getWorkflowActionInfo(optionValue), timeZoneId,
1081 options.contains(VERBOSE_OPTION));
1082 }
1083 else {
1084 String filter = commandLine.getOptionValue(FILTER_OPTION);
1085 if (filter != null) {
1086 throw new OozieCLIException("Filter option is currently not supported for a Workflow job");
1087 }
1088 String s = commandLine.getOptionValue(OFFSET_OPTION);
1089 int start = Integer.parseInt((s != null) ? s : "0");
1090 s = commandLine.getOptionValue(LEN_OPTION);
1091 String jobtype = commandLine.getOptionValue(JOBTYPE_OPTION);
1092 jobtype = (jobtype != null) ? jobtype : "wf";
1093 int len = Integer.parseInt((s != null) ? s : "0");
1094 printJob(wc.getJobInfo(optionValue, start, len), timeZoneId,
1095 options.contains(VERBOSE_OPTION));
1096 }
1097 }
1098 else if (options.contains(LOG_OPTION)) {
1099 PrintStream ps = System.out;
1100 String logFilter = null;
1101 if (options.contains(RestConstants.LOG_FILTER_OPTION)) {
1102 logFilter = commandLine.getOptionValue(RestConstants.LOG_FILTER_OPTION);
1103 }
1104 if (commandLine.getOptionValue(LOG_OPTION).contains("-C")) {
1105 String logRetrievalScope = null;
1106 String logRetrievalType = null;
1107 if (options.contains(ACTION_OPTION)) {
1108 logRetrievalType = RestConstants.JOB_LOG_ACTION;
1109 logRetrievalScope = commandLine.getOptionValue(ACTION_OPTION);
1110 }
1111 if (options.contains(DATE_OPTION)) {
1112 logRetrievalType = RestConstants.JOB_LOG_DATE;
1113 logRetrievalScope = commandLine.getOptionValue(DATE_OPTION);
1114 }
1115 try {
1116 wc.getJobLog(commandLine.getOptionValue(LOG_OPTION), logRetrievalType, logRetrievalScope,
1117 logFilter, ps);
1118 }
1119 finally {
1120 ps.close();
1121 }
1122 }
1123 else {
1124 if (!options.contains(ACTION_OPTION) && !options.contains(DATE_OPTION)) {
1125 wc.getJobLog(commandLine.getOptionValue(LOG_OPTION), null, null, logFilter, ps);
1126 }
1127 else {
1128 throw new OozieCLIException("Invalid options provided for log retrieval. " + ACTION_OPTION
1129 + " and " + DATE_OPTION + " are valid only for coordinator job log retrieval");
1130 }
1131 }
1132 }
1133 else if (options.contains(DEFINITION_OPTION)) {
1134 System.out.println(wc.getJobDefinition(commandLine.getOptionValue(DEFINITION_OPTION)));
1135 }
1136 else if (options.contains(CONFIG_CONTENT_OPTION)) {
1137 if (commandLine.getOptionValue(CONFIG_CONTENT_OPTION).endsWith("-C")) {
1138 System.out.println(wc.getCoordJobInfo(commandLine.getOptionValue(CONFIG_CONTENT_OPTION)).getConf());
1139 }
1140 else if (commandLine.getOptionValue(CONFIG_CONTENT_OPTION).endsWith("-W")) {
1141 System.out.println(wc.getJobInfo(commandLine.getOptionValue(CONFIG_CONTENT_OPTION)).getConf());
1142 }
1143 else if (commandLine.getOptionValue(CONFIG_CONTENT_OPTION).endsWith("-B")) {
1144 System.out
1145 .println(wc.getBundleJobInfo(commandLine.getOptionValue(CONFIG_CONTENT_OPTION)).getConf());
1146 }
1147 else {
1148 System.out.println("ERROR: job id [" + commandLine.getOptionValue(CONFIG_CONTENT_OPTION)
1149 + "] doesn't end with either C or W or B");
1150 }
1151 }
1152 else if (options.contains(UPDATE_OPTION)) {
1153 String coordJobId = commandLine.getOptionValue(UPDATE_OPTION);
1154 Properties conf = null;
1155
1156 String dryrun = "";
1157 String showdiff = "";
1158
1159 if (commandLine.getOptionValue(CONFIG_OPTION) != null) {
1160 conf = getConfiguration(wc, commandLine);
1161 }
1162 if (options.contains(DRYRUN_OPTION)) {
1163 dryrun = "true";
1164 }
1165 if (commandLine.getOptionValue(SHOWDIFF_OPTION) != null) {
1166 showdiff = commandLine.getOptionValue(SHOWDIFF_OPTION);
1167 }
1168 if (conf == null) {
1169 System.out.println(wc.updateCoord(coordJobId, dryrun, showdiff));
1170 }
1171 else {
1172 System.out.println(wc.updateCoord(coordJobId, conf, dryrun, showdiff));
1173 }
1174 }
1175 }
1176 catch (OozieClientException ex) {
1177 throw new OozieCLIException(ex.toString(), ex);
1178 }
1179 }
1180
1181 @VisibleForTesting
1182 void printCoordJob(CoordinatorJob coordJob, String timeZoneId, boolean verbose) {
1183 System.out.println("Job ID : " + coordJob.getId());
1184
1185 System.out.println(RULER);
1186
1187 List<CoordinatorAction> actions = coordJob.getActions();
1188 System.out.println("Job Name : " + maskIfNull(coordJob.getAppName()));
1189 System.out.println("App Path : " + maskIfNull(coordJob.getAppPath()));
1190 System.out.println("Status : " + coordJob.getStatus());
1191 System.out.println("Start Time : " + maskDate(coordJob.getStartTime(), timeZoneId, false));
1192 System.out.println("End Time : " + maskDate(coordJob.getEndTime(), timeZoneId, false));
1193 System.out.println("Pause Time : " + maskDate(coordJob.getPauseTime(), timeZoneId, false));
1194 System.out.println("Concurrency : " + coordJob.getConcurrency());
1195 System.out.println(RULER);
1196
1197 if (verbose) {
1198 System.out.println("ID" + VERBOSE_DELIMITER + "Action Number" + VERBOSE_DELIMITER + "Console URL"
1199 + VERBOSE_DELIMITER + "Error Code" + VERBOSE_DELIMITER + "Error Message" + VERBOSE_DELIMITER
1200 + "External ID" + VERBOSE_DELIMITER + "External Status" + VERBOSE_DELIMITER + "Job ID"
1201 + VERBOSE_DELIMITER + "Tracker URI" + VERBOSE_DELIMITER + "Created" + VERBOSE_DELIMITER
1202 + "Nominal Time" + VERBOSE_DELIMITER + "Status" + VERBOSE_DELIMITER + "Last Modified"
1203 + VERBOSE_DELIMITER + "Missing Dependencies");
1204 System.out.println(RULER);
1205
1206 for (CoordinatorAction action : actions) {
1207 System.out.println(maskIfNull(action.getId()) + VERBOSE_DELIMITER + action.getActionNumber()
1208 + VERBOSE_DELIMITER + maskIfNull(action.getConsoleUrl()) + VERBOSE_DELIMITER
1209 + maskIfNull(action.getErrorCode()) + VERBOSE_DELIMITER + maskIfNull(action.getErrorMessage())
1210 + VERBOSE_DELIMITER + maskIfNull(action.getExternalId()) + VERBOSE_DELIMITER
1211 + maskIfNull(action.getExternalStatus()) + VERBOSE_DELIMITER + maskIfNull(action.getJobId())
1212 + VERBOSE_DELIMITER + maskIfNull(action.getTrackerUri()) + VERBOSE_DELIMITER
1213 + maskDate(action.getCreatedTime(), timeZoneId, verbose) + VERBOSE_DELIMITER
1214 + maskDate(action.getNominalTime(), timeZoneId, verbose) + action.getStatus() + VERBOSE_DELIMITER
1215 + maskDate(action.getLastModifiedTime(), timeZoneId, verbose) + VERBOSE_DELIMITER
1216 + maskIfNull(getFirstMissingDependencies(action)));
1217
1218 System.out.println(RULER);
1219 }
1220 }
1221 else {
1222 System.out.println(String.format(COORD_ACTION_FORMATTER, "ID", "Status", "Ext ID", "Err Code", "Created",
1223 "Nominal Time", "Last Mod"));
1224
1225 for (CoordinatorAction action : actions) {
1226 System.out.println(String.format(COORD_ACTION_FORMATTER, maskIfNull(action.getId()),
1227 action.getStatus(), maskIfNull(action.getExternalId()), maskIfNull(action.getErrorCode()),
1228 maskDate(action.getCreatedTime(), timeZoneId, verbose), maskDate(action.getNominalTime(), timeZoneId, verbose),
1229 maskDate(action.getLastModifiedTime(), timeZoneId, verbose)));
1230
1231 System.out.println(RULER);
1232 }
1233 }
1234 }
1235
1236 @VisibleForTesting
1237 void printBundleJob(BundleJob bundleJob, String timeZoneId, boolean verbose) {
1238 System.out.println("Job ID : " + bundleJob.getId());
1239
1240 System.out.println(RULER);
1241
1242 List<CoordinatorJob> coordinators = bundleJob.getCoordinators();
1243 System.out.println("Job Name : " + maskIfNull(bundleJob.getAppName()));
1244 System.out.println("App Path : " + maskIfNull(bundleJob.getAppPath()));
1245 System.out.println("Status : " + bundleJob.getStatus());
1246 System.out.println("Kickoff time : " + bundleJob.getKickoffTime());
1247 System.out.println(RULER);
1248
1249 System.out.println(String.format(BUNDLE_COORD_JOBS_FORMATTER, "Job ID", "Status", "Freq", "Unit", "Started",
1250 "Next Materialized"));
1251 System.out.println(RULER);
1252
1253 for (CoordinatorJob job : coordinators) {
1254 System.out.println(String.format(BUNDLE_COORD_JOBS_FORMATTER, maskIfNull(job.getId()), job.getStatus(),
1255 job.getFrequency(), job.getTimeUnit(), maskDate(job.getStartTime(), timeZoneId, verbose),
1256 maskDate(job.getNextMaterializedTime(), timeZoneId, verbose)));
1257
1258 System.out.println(RULER);
1259 }
1260 }
1261
1262 @VisibleForTesting
1263 void printCoordAction(CoordinatorAction coordAction, String timeZoneId) {
1264 System.out.println("ID : " + maskIfNull(coordAction.getId()));
1265
1266 System.out.println(RULER);
1267
1268 System.out.println("Action Number : " + coordAction.getActionNumber());
1269 System.out.println("Console URL : " + maskIfNull(coordAction.getConsoleUrl()));
1270 System.out.println("Error Code : " + maskIfNull(coordAction.getErrorCode()));
1271 System.out.println("Error Message : " + maskIfNull(coordAction.getErrorMessage()));
1272 System.out.println("External ID : " + maskIfNull(coordAction.getExternalId()));
1273 System.out.println("External Status : " + maskIfNull(coordAction.getExternalStatus()));
1274 System.out.println("Job ID : " + maskIfNull(coordAction.getJobId()));
1275 System.out.println("Tracker URI : " + maskIfNull(coordAction.getTrackerUri()));
1276 System.out.println("Created : " + maskDate(coordAction.getCreatedTime(), timeZoneId, false));
1277 System.out.println("Nominal Time : " + maskDate(coordAction.getNominalTime(), timeZoneId, false));
1278 System.out.println("Status : " + coordAction.getStatus());
1279 System.out.println("Last Modified : " + maskDate(coordAction.getLastModifiedTime(), timeZoneId, false));
1280 System.out.println("First Missing Dependency : " + maskIfNull(getFirstMissingDependencies(coordAction)));
1281
1282 System.out.println(RULER);
1283 }
1284
1285 private void printCoordActions(List<CoordinatorAction> actions) {
1286 if (actions != null && actions.size() > 0) {
1287 System.out.println("Action ID" + VERBOSE_DELIMITER + "Nominal Time");
1288 System.out.println(RULER);
1289 for (CoordinatorAction action : actions) {
1290 System.out.println(maskIfNull(action.getId()) + VERBOSE_DELIMITER
1291 + maskDate(action.getNominalTime(), null,false));
1292 }
1293 }
1294 else {
1295 System.out.println("No Actions match your criteria!");
1296 }
1297 }
1298
1299 private void printCoordActionsStatus(List<CoordinatorAction> actions) {
1300 if (actions != null && actions.size() > 0) {
1301 System.out.println("Action ID" + VERBOSE_DELIMITER + "Nominal Time" + VERBOSE_DELIMITER + "Status");
1302 System.out.println(RULER);
1303 for (CoordinatorAction action : actions) {
1304 System.out.println(maskIfNull(action.getId()) + VERBOSE_DELIMITER
1305 + maskDate(action.getNominalTime(), null, false) + VERBOSE_DELIMITER
1306 + maskIfNull(action.getStatus().name()));
1307 }
1308 }
1309 }
1310
1311 @VisibleForTesting
1312 void printWorkflowAction(WorkflowAction action, String timeZoneId, boolean verbose) {
1313
1314 System.out.println("ID : " + maskIfNull(action.getId()));
1315
1316 System.out.println(RULER);
1317
1318 System.out.println("Console URL : " + maskIfNull(action.getConsoleUrl()));
1319 System.out.println("Error Code : " + maskIfNull(action.getErrorCode()));
1320 System.out.println("Error Message : " + maskIfNull(action.getErrorMessage()));
1321 System.out.println("External ID : " + maskIfNull(action.getExternalId()));
1322 System.out.println("External Status : " + maskIfNull(action.getExternalStatus()));
1323 System.out.println("Name : " + maskIfNull(action.getName()));
1324 System.out.println("Retries : " + action.getRetries());
1325 System.out.println("Tracker URI : " + maskIfNull(action.getTrackerUri()));
1326 System.out.println("Type : " + maskIfNull(action.getType()));
1327 System.out.println("Started : " + maskDate(action.getStartTime(), timeZoneId, verbose));
1328 System.out.println("Status : " + action.getStatus());
1329 System.out.println("Ended : " + maskDate(action.getEndTime(), timeZoneId, verbose));
1330
1331 if (verbose) {
1332 System.out.println("External Stats : " + action.getStats());
1333 System.out.println("External ChildIDs : " + action.getExternalChildIDs());
1334 }
1335
1336 System.out.println(RULER);
1337 }
1338
1339 private static final String WORKFLOW_JOBS_FORMATTER = "%-41s%-13s%-10s%-10s%-10s%-24s%-24s";
1340 private static final String COORD_JOBS_FORMATTER = "%-41s%-15s%-10s%-5s%-13s%-24s%-24s";
1341 private static final String BUNDLE_JOBS_FORMATTER = "%-41s%-15s%-10s%-20s%-20s%-13s%-13s";
1342 private static final String BUNDLE_COORD_JOBS_FORMATTER = "%-41s%-15s%-5s%-13s%-24s%-24s";
1343
1344 private static final String WORKFLOW_ACTION_FORMATTER = "%-78s%-10s%-23s%-11s%-10s";
1345 private static final String COORD_ACTION_FORMATTER = "%-43s%-10s%-37s%-10s%-21s%-21s";
1346 private static final String BULK_RESPONSE_FORMATTER = "%-13s%-38s%-13s%-41s%-10s%-38s%-21s%-38s";
1347
1348 @VisibleForTesting
1349 void printJob(WorkflowJob job, String timeZoneId, boolean verbose) throws IOException {
1350 System.out.println("Job ID : " + maskIfNull(job.getId()));
1351
1352 System.out.println(RULER);
1353
1354 System.out.println("Workflow Name : " + maskIfNull(job.getAppName()));
1355 System.out.println("App Path : " + maskIfNull(job.getAppPath()));
1356 System.out.println("Status : " + job.getStatus());
1357 System.out.println("Run : " + job.getRun());
1358 System.out.println("User : " + maskIfNull(job.getUser()));
1359 System.out.println("Group : " + maskIfNull(job.getGroup()));
1360 System.out.println("Created : " + maskDate(job.getCreatedTime(), timeZoneId, verbose));
1361 System.out.println("Started : " + maskDate(job.getStartTime(), timeZoneId, verbose));
1362 System.out.println("Last Modified : " + maskDate(job.getLastModifiedTime(), timeZoneId, verbose));
1363 System.out.println("Ended : " + maskDate(job.getEndTime(), timeZoneId, verbose));
1364 System.out.println("CoordAction ID: " + maskIfNull(job.getParentId()));
1365
1366 List<WorkflowAction> actions = job.getActions();
1367
1368 if (actions != null && actions.size() > 0) {
1369 System.out.println();
1370 System.out.println("Actions");
1371 System.out.println(RULER);
1372
1373 if (verbose) {
1374 System.out.println("ID" + VERBOSE_DELIMITER + "Console URL" + VERBOSE_DELIMITER + "Error Code"
1375 + VERBOSE_DELIMITER + "Error Message" + VERBOSE_DELIMITER + "External ID" + VERBOSE_DELIMITER
1376 + "External Status" + VERBOSE_DELIMITER + "Name" + VERBOSE_DELIMITER + "Retries"
1377 + VERBOSE_DELIMITER + "Tracker URI" + VERBOSE_DELIMITER + "Type" + VERBOSE_DELIMITER
1378 + "Started" + VERBOSE_DELIMITER + "Status" + VERBOSE_DELIMITER + "Ended");
1379 System.out.println(RULER);
1380
1381 for (WorkflowAction action : job.getActions()) {
1382 System.out.println(maskIfNull(action.getId()) + VERBOSE_DELIMITER
1383 + maskIfNull(action.getConsoleUrl()) + VERBOSE_DELIMITER
1384 + maskIfNull(action.getErrorCode()) + VERBOSE_DELIMITER
1385 + maskIfNull(action.getErrorMessage()) + VERBOSE_DELIMITER
1386 + maskIfNull(action.getExternalId()) + VERBOSE_DELIMITER
1387 + maskIfNull(action.getExternalStatus()) + VERBOSE_DELIMITER + maskIfNull(action.getName())
1388 + VERBOSE_DELIMITER + action.getRetries() + VERBOSE_DELIMITER
1389 + maskIfNull(action.getTrackerUri()) + VERBOSE_DELIMITER + maskIfNull(action.getType())
1390 + VERBOSE_DELIMITER + maskDate(action.getStartTime(), timeZoneId, verbose)
1391 + VERBOSE_DELIMITER + action.getStatus() + VERBOSE_DELIMITER
1392 + maskDate(action.getEndTime(), timeZoneId, verbose));
1393
1394 System.out.println(RULER);
1395 }
1396 }
1397 else {
1398 System.out.println(String.format(WORKFLOW_ACTION_FORMATTER, "ID", "Status", "Ext ID", "Ext Status",
1399 "Err Code"));
1400
1401 System.out.println(RULER);
1402
1403 for (WorkflowAction action : job.getActions()) {
1404 System.out.println(String.format(WORKFLOW_ACTION_FORMATTER, maskIfNull(action.getId()), action
1405 .getStatus(), maskIfNull(action.getExternalId()), maskIfNull(action.getExternalStatus()),
1406 maskIfNull(action.getErrorCode())));
1407
1408 System.out.println(RULER);
1409 }
1410 }
1411 }
1412 else {
1413 System.out.println(RULER);
1414 }
1415
1416 System.out.println();
1417 }
1418
1419 private void jobsCommand(CommandLine commandLine) throws IOException, OozieCLIException {
1420 XOozieClient wc = createXOozieClient(commandLine);
1421
1422 String filter = commandLine.getOptionValue(FILTER_OPTION);
1423 String s = commandLine.getOptionValue(OFFSET_OPTION);
1424 int start = Integer.parseInt((s != null) ? s : "0");
1425 s = commandLine.getOptionValue(LEN_OPTION);
1426 String jobtype = commandLine.getOptionValue(JOBTYPE_OPTION);
1427 String timeZoneId = getTimeZoneId(commandLine);
1428 jobtype = (jobtype != null) ? jobtype : "wf";
1429 int len = Integer.parseInt((s != null) ? s : "0");
1430 String bulkFilterString = commandLine.getOptionValue(BULK_OPTION);
1431
1432 try {
1433 if (bulkFilterString != null) {
1434 printBulkJobs(wc.getBulkInfo(bulkFilterString, start, len), timeZoneId, commandLine.hasOption(VERBOSE_OPTION));
1435 }
1436 else if (jobtype.toLowerCase().contains("wf")) {
1437 printJobs(wc.getJobsInfo(filter, start, len), timeZoneId, commandLine.hasOption(VERBOSE_OPTION));
1438 }
1439 else if (jobtype.toLowerCase().startsWith("coord")) {
1440 printCoordJobs(wc.getCoordJobsInfo(filter, start, len), timeZoneId, commandLine.hasOption(VERBOSE_OPTION));
1441 }
1442 else if (jobtype.toLowerCase().startsWith("bundle")) {
1443 printBundleJobs(wc.getBundleJobsInfo(filter, start, len), timeZoneId, commandLine.hasOption(VERBOSE_OPTION));
1444 }
1445
1446 }
1447 catch (OozieClientException ex) {
1448 throw new OozieCLIException(ex.toString(), ex);
1449 }
1450 }
1451
1452 @VisibleForTesting
1453 void printCoordJobs(List<CoordinatorJob> jobs, String timeZoneId, boolean verbose) throws IOException {
1454 if (jobs != null && jobs.size() > 0) {
1455 if (verbose) {
1456 System.out.println("Job ID" + VERBOSE_DELIMITER + "App Name" + VERBOSE_DELIMITER + "App Path"
1457 + VERBOSE_DELIMITER + "Console URL" + VERBOSE_DELIMITER + "User" + VERBOSE_DELIMITER + "Group"
1458 + VERBOSE_DELIMITER + "Concurrency" + VERBOSE_DELIMITER + "Frequency" + VERBOSE_DELIMITER
1459 + "Time Unit" + VERBOSE_DELIMITER + "Time Zone" + VERBOSE_DELIMITER + "Time Out"
1460 + VERBOSE_DELIMITER + "Started" + VERBOSE_DELIMITER + "Next Materialize" + VERBOSE_DELIMITER
1461 + "Status" + VERBOSE_DELIMITER + "Last Action" + VERBOSE_DELIMITER + "Ended");
1462 System.out.println(RULER);
1463
1464 for (CoordinatorJob job : jobs) {
1465 System.out.println(maskIfNull(job.getId()) + VERBOSE_DELIMITER + maskIfNull(job.getAppName())
1466 + VERBOSE_DELIMITER + maskIfNull(job.getAppPath()) + VERBOSE_DELIMITER
1467 + maskIfNull(job.getConsoleUrl()) + VERBOSE_DELIMITER + maskIfNull(job.getUser())
1468 + VERBOSE_DELIMITER + maskIfNull(job.getGroup()) + VERBOSE_DELIMITER + job.getConcurrency()
1469 + VERBOSE_DELIMITER + job.getFrequency() + VERBOSE_DELIMITER + job.getTimeUnit()
1470 + VERBOSE_DELIMITER + maskIfNull(job.getTimeZone()) + VERBOSE_DELIMITER + job.getTimeout()
1471 + VERBOSE_DELIMITER + maskDate(job.getStartTime(), timeZoneId, verbose) + VERBOSE_DELIMITER
1472 + maskDate(job.getNextMaterializedTime(), timeZoneId, verbose) + VERBOSE_DELIMITER
1473 + job.getStatus() + VERBOSE_DELIMITER
1474 + maskDate(job.getLastActionTime(), timeZoneId, verbose) + VERBOSE_DELIMITER
1475 + maskDate(job.getEndTime(), timeZoneId, verbose));
1476
1477 System.out.println(RULER);
1478 }
1479 }
1480 else {
1481 System.out.println(String.format(COORD_JOBS_FORMATTER, "Job ID", "App Name", "Status", "Freq", "Unit",
1482 "Started", "Next Materialized"));
1483 System.out.println(RULER);
1484
1485 for (CoordinatorJob job : jobs) {
1486 System.out.println(String.format(COORD_JOBS_FORMATTER, maskIfNull(job.getId()), maskIfNull(job
1487 .getAppName()), job.getStatus(), job.getFrequency(), job.getTimeUnit(), maskDate(job
1488 .getStartTime(), timeZoneId, verbose), maskDate(job.getNextMaterializedTime(), timeZoneId, verbose)));
1489
1490 System.out.println(RULER);
1491 }
1492 }
1493 }
1494 else {
1495 System.out.println("No Jobs match your criteria!");
1496 }
1497 }
1498
1499 @VisibleForTesting
1500 void printBulkJobs(List<BulkResponse> jobs, String timeZoneId, boolean verbose) throws IOException {
1501 if (jobs != null && jobs.size() > 0) {
1502 for (BulkResponse response : jobs) {
1503 BundleJob bundle = response.getBundle();
1504 CoordinatorJob coord = response.getCoordinator();
1505 CoordinatorAction action = response.getAction();
1506 if (verbose) {
1507 System.out.println();
1508 System.out.println("Bundle Name : " + maskIfNull(bundle.getAppName()));
1509
1510 System.out.println(RULER);
1511
1512 System.out.println("Bundle ID : " + maskIfNull(bundle.getId()));
1513 System.out.println("Coordinator Name : " + maskIfNull(coord.getAppName()));
1514 System.out.println("Coord Action ID : " + maskIfNull(action.getId()));
1515 System.out.println("Action Status : " + action.getStatus());
1516 System.out.println("External ID : " + maskIfNull(action.getExternalId()));
1517 System.out.println("Created Time : " + maskDate(action.getCreatedTime(), timeZoneId, false));
1518 System.out.println("User : " + maskIfNull(bundle.getUser()));
1519 System.out.println("Error Message : " + maskIfNull(action.getErrorMessage()));
1520 System.out.println(RULER);
1521 }
1522 else {
1523 System.out.println(String.format(BULK_RESPONSE_FORMATTER, "Bundle Name", "Bundle ID", "Coord Name",
1524 "Coord Action ID", "Status", "External ID", "Created Time", "Error Message"));
1525 System.out.println(RULER);
1526 System.out
1527 .println(String.format(BULK_RESPONSE_FORMATTER, maskIfNull(bundle.getAppName()),
1528 maskIfNull(bundle.getId()), maskIfNull(coord.getAppName()),
1529 maskIfNull(action.getId()), action.getStatus(), maskIfNull(action.getExternalId()),
1530 maskDate(action.getCreatedTime(), timeZoneId, false),
1531 maskIfNull(action.getErrorMessage())));
1532 System.out.println(RULER);
1533 }
1534 }
1535 }
1536 else {
1537 System.out.println("Bulk request criteria did not match any coordinator actions");
1538 }
1539 }
1540
1541 @VisibleForTesting
1542 void printBundleJobs(List<BundleJob> jobs, String timeZoneId, boolean verbose) throws IOException {
1543 if (jobs != null && jobs.size() > 0) {
1544 if (verbose) {
1545 System.out.println("Job ID" + VERBOSE_DELIMITER + "Bundle Name" + VERBOSE_DELIMITER + "Bundle Path"
1546 + VERBOSE_DELIMITER + "User" + VERBOSE_DELIMITER + "Group" + VERBOSE_DELIMITER + "Status"
1547 + VERBOSE_DELIMITER + "Kickoff" + VERBOSE_DELIMITER + "Pause" + VERBOSE_DELIMITER + "Created"
1548 + VERBOSE_DELIMITER + "Console URL");
1549 System.out.println(RULER);
1550
1551 for (BundleJob job : jobs) {
1552 System.out.println(maskIfNull(job.getId()) + VERBOSE_DELIMITER + maskIfNull(job.getAppName())
1553 + VERBOSE_DELIMITER + maskIfNull(job.getAppPath()) + VERBOSE_DELIMITER
1554 + maskIfNull(job.getUser()) + VERBOSE_DELIMITER + maskIfNull(job.getGroup())
1555 + VERBOSE_DELIMITER + job.getStatus() + VERBOSE_DELIMITER
1556 + maskDate(job.getKickoffTime(), timeZoneId, verbose) + VERBOSE_DELIMITER
1557 + maskDate(job.getPauseTime(), timeZoneId, verbose) + VERBOSE_DELIMITER
1558 + maskDate(job.getCreatedTime(), timeZoneId, verbose) + VERBOSE_DELIMITER
1559 + maskIfNull(job.getConsoleUrl()));
1560
1561 System.out.println(RULER);
1562 }
1563 }
1564 else {
1565 System.out.println(String.format(BUNDLE_JOBS_FORMATTER, "Job ID", "Bundle Name", "Status", "Kickoff",
1566 "Created", "User", "Group"));
1567 System.out.println(RULER);
1568
1569 for (BundleJob job : jobs) {
1570 System.out.println(String.format(BUNDLE_JOBS_FORMATTER, maskIfNull(job.getId()),
1571 maskIfNull(job.getAppName()), job.getStatus(),
1572 maskDate(job.getKickoffTime(), timeZoneId, verbose),
1573 maskDate(job.getCreatedTime(), timeZoneId, verbose), maskIfNull(job.getUser()),
1574 maskIfNull(job.getGroup())));
1575 System.out.println(RULER);
1576 }
1577 }
1578 }
1579 else {
1580 System.out.println("No Jobs match your criteria!");
1581 }
1582 }
1583
1584 private void slaCommand(CommandLine commandLine) throws IOException, OozieCLIException {
1585 XOozieClient wc = createXOozieClient(commandLine);
1586 List<String> options = new ArrayList<String>();
1587 for (Option option : commandLine.getOptions()) {
1588 options.add(option.getOpt());
1589 }
1590
1591 String s = commandLine.getOptionValue(OFFSET_OPTION);
1592 int start = Integer.parseInt((s != null) ? s : "0");
1593 s = commandLine.getOptionValue(LEN_OPTION);
1594 int len = Integer.parseInt((s != null) ? s : "100");
1595 String filter = commandLine.getOptionValue(FILTER_OPTION);
1596
1597 try {
1598 wc.getSlaInfo(start, len, filter);
1599 }
1600 catch (OozieClientException ex) {
1601 throw new OozieCLIException(ex.toString(), ex);
1602 }
1603 }
1604
1605 private void adminCommand(CommandLine commandLine) throws OozieCLIException {
1606 XOozieClient wc = createXOozieClient(commandLine);
1607
1608 List<String> options = new ArrayList<String>();
1609 for (Option option : commandLine.getOptions()) {
1610 options.add(option.getOpt());
1611 }
1612
1613 try {
1614 SYSTEM_MODE status = SYSTEM_MODE.NORMAL;
1615 if (options.contains(VERSION_OPTION)) {
1616 System.out.println("Oozie server build version: " + wc.getServerBuildVersion());
1617 }
1618 else if (options.contains(SYSTEM_MODE_OPTION)) {
1619 String systemModeOption = commandLine.getOptionValue(SYSTEM_MODE_OPTION).toUpperCase();
1620 try {
1621 status = SYSTEM_MODE.valueOf(systemModeOption);
1622 }
1623 catch (Exception e) {
1624 throw new OozieCLIException("Invalid input provided for option: " + SYSTEM_MODE_OPTION
1625 + " value given :" + systemModeOption
1626 + " Expected values are: NORMAL/NOWEBSERVICE/SAFEMODE ");
1627 }
1628 wc.setSystemMode(status);
1629 System.out.println("System mode: " + status);
1630 }
1631 else if (options.contains(STATUS_OPTION)) {
1632 status = wc.getSystemMode();
1633 System.out.println("System mode: " + status);
1634 }
1635
1636 else if (options.contains(UPDATE_SHARELIB_OPTION)) {
1637 System.out.println(wc.updateShareLib());
1638 }
1639
1640 else if (options.contains(LIST_SHARELIB_LIB_OPTION)) {
1641 String sharelibKey = null;
1642 if (commandLine.getArgList().size() > 0) {
1643 sharelibKey = (String) commandLine.getArgList().get(0);
1644 }
1645 System.out.println(wc.listShareLib(sharelibKey));
1646 }
1647
1648 else if (options.contains(QUEUE_DUMP_OPTION)) {
1649
1650 List<String> list = wc.getQueueDump();
1651 if (list != null && list.size() != 0) {
1652 for (String str : list) {
1653 System.out.println(str);
1654 }
1655 }
1656 else {
1657 System.out.println("QueueDump is null!");
1658 }
1659 }
1660 else if (options.contains(AVAILABLE_SERVERS_OPTION)) {
1661 Map<String, String> availableOozieServers = wc.getAvailableOozieServers();
1662 for (String key : availableOozieServers.keySet()) {
1663 System.out.println(key + " : " + availableOozieServers.get(key));
1664 }
1665 }
1666 }
1667 catch (OozieClientException ex) {
1668 throw new OozieCLIException(ex.toString(), ex);
1669 }
1670 }
1671
1672 private void versionCommand() throws OozieCLIException {
1673 System.out.println("Oozie client build version: "
1674 + BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION));
1675 }
1676
1677 @VisibleForTesting
1678 void printJobs(List<WorkflowJob> jobs, String timeZoneId, boolean verbose) throws IOException {
1679 if (jobs != null && jobs.size() > 0) {
1680 if (verbose) {
1681 System.out.println("Job ID" + VERBOSE_DELIMITER + "App Name" + VERBOSE_DELIMITER + "App Path"
1682 + VERBOSE_DELIMITER + "Console URL" + VERBOSE_DELIMITER + "User" + VERBOSE_DELIMITER + "Group"
1683 + VERBOSE_DELIMITER + "Run" + VERBOSE_DELIMITER + "Created" + VERBOSE_DELIMITER + "Started"
1684 + VERBOSE_DELIMITER + "Status" + VERBOSE_DELIMITER + "Last Modified" + VERBOSE_DELIMITER
1685 + "Ended");
1686 System.out.println(RULER);
1687
1688 for (WorkflowJob job : jobs) {
1689 System.out.println(maskIfNull(job.getId()) + VERBOSE_DELIMITER + maskIfNull(job.getAppName())
1690 + VERBOSE_DELIMITER + maskIfNull(job.getAppPath()) + VERBOSE_DELIMITER
1691 + maskIfNull(job.getConsoleUrl()) + VERBOSE_DELIMITER + maskIfNull(job.getUser())
1692 + VERBOSE_DELIMITER + maskIfNull(job.getGroup()) + VERBOSE_DELIMITER + job.getRun()
1693 + VERBOSE_DELIMITER + maskDate(job.getCreatedTime(), timeZoneId, verbose)
1694 + VERBOSE_DELIMITER + maskDate(job.getStartTime(), timeZoneId, verbose) + VERBOSE_DELIMITER
1695 + job.getStatus() + VERBOSE_DELIMITER
1696 + maskDate(job.getLastModifiedTime(), timeZoneId, verbose) + VERBOSE_DELIMITER
1697 + maskDate(job.getEndTime(), timeZoneId, verbose));
1698
1699 System.out.println(RULER);
1700 }
1701 }
1702 else {
1703 System.out.println(String.format(WORKFLOW_JOBS_FORMATTER, "Job ID", "App Name", "Status", "User",
1704 "Group", "Started", "Ended"));
1705 System.out.println(RULER);
1706
1707 for (WorkflowJob job : jobs) {
1708 System.out.println(String.format(WORKFLOW_JOBS_FORMATTER, maskIfNull(job.getId()),
1709 maskIfNull(job.getAppName()), job.getStatus(), maskIfNull(job.getUser()),
1710 maskIfNull(job.getGroup()), maskDate(job.getStartTime(), timeZoneId, verbose),
1711 maskDate(job.getEndTime(), timeZoneId, verbose)));
1712
1713 System.out.println(RULER);
1714 }
1715 }
1716 }
1717 else {
1718 System.out.println("No Jobs match your criteria!");
1719 }
1720 }
1721
1722 void printWfsForCoordAction(List<WorkflowJob> jobs, String timeZoneId) throws IOException {
1723 if (jobs != null && jobs.size() > 0) {
1724 System.out.println(String.format("%-41s%-10s%-24s%-24s", "Job ID", "Status", "Started", "Ended"));
1725 System.out.println(RULER);
1726
1727 for (WorkflowJob job : jobs) {
1728 System.out
1729 .println(String.format("%-41s%-10s%-24s%-24s", maskIfNull(job.getId()), job.getStatus(),
1730 maskDate(job.getStartTime(), timeZoneId, false),
1731 maskDate(job.getEndTime(), timeZoneId, false)));
1732 System.out.println(RULER);
1733 }
1734 }
1735 }
1736
1737 private String maskIfNull(String value) {
1738 if (value != null && value.length() > 0) {
1739 return value;
1740 }
1741 return "-";
1742 }
1743
1744 private String maskDate(Date date, String timeZoneId, boolean verbose) {
1745 if (date == null) {
1746 return "-";
1747 }
1748
1749 SimpleDateFormat dateFormater = null;
1750 if (verbose) {
1751 dateFormater = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss zzz", Locale.US);
1752 }
1753 else {
1754 dateFormater = new SimpleDateFormat("yyyy-MM-dd HH:mm zzz", Locale.US);
1755 }
1756
1757 if (timeZoneId != null) {
1758 dateFormater.setTimeZone(TimeZone.getTimeZone(timeZoneId));
1759 }
1760 String dateString = dateFormater.format(date);
1761 // Most TimeZones are 3 or 4 characters; GMT offsets (e.g. GMT-07:00) are 9, so lets remove the "GMT" part to make it 6
1762 // to fit better
1763 Matcher m = GMT_OFFSET_SHORTEN_PATTERN.matcher(dateString);
1764 if (m.matches() && m.groupCount() == 2) {
1765 dateString = m.group(1) + m.group(2);
1766 }
1767 return dateString;
1768 }
1769
1770 private void validateCommand(CommandLine commandLine) throws OozieCLIException {
1771 String[] args = commandLine.getArgs();
1772 if (args.length != 1) {
1773 throw new OozieCLIException("One file must be specified");
1774 }
1775 File file = new File(args[0]);
1776 if (file.exists()) {
1777 try {
1778 List<StreamSource> sources = new ArrayList<StreamSource>();
1779 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1780 "oozie-workflow-0.1.xsd")));
1781 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1782 "shell-action-0.1.xsd")));
1783 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1784 "shell-action-0.2.xsd")));
1785 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1786 "shell-action-0.3.xsd")));
1787 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1788 "email-action-0.1.xsd")));
1789 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1790 "email-action-0.2.xsd")));
1791 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1792 "distcp-action-0.1.xsd")));
1793 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1794 "distcp-action-0.2.xsd")));
1795 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1796 "oozie-workflow-0.2.xsd")));
1797 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1798 "oozie-workflow-0.2.5.xsd")));
1799 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1800 "oozie-workflow-0.3.xsd")));
1801 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1802 "oozie-workflow-0.4.xsd")));
1803 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1804 "oozie-workflow-0.4.5.xsd")));
1805 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1806 "oozie-workflow-0.5.xsd")));
1807 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1808 "oozie-coordinator-0.1.xsd")));
1809 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1810 "oozie-coordinator-0.2.xsd")));
1811 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1812 "oozie-coordinator-0.3.xsd")));
1813 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1814 "oozie-coordinator-0.4.xsd")));
1815 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1816 "oozie-bundle-0.1.xsd")));
1817 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1818 "oozie-bundle-0.2.xsd")));
1819 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1820 "oozie-sla-0.1.xsd")));
1821 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1822 "oozie-sla-0.2.xsd")));
1823 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1824 "hive-action-0.2.xsd")));
1825 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1826 "hive-action-0.3.xsd")));
1827 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1828 "hive-action-0.4.xsd")));
1829 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1830 "hive-action-0.5.xsd")));
1831 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1832 "sqoop-action-0.2.xsd")));
1833 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1834 "sqoop-action-0.3.xsd")));
1835 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1836 "sqoop-action-0.4.xsd")));
1837 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1838 "ssh-action-0.1.xsd")));
1839 sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
1840 "ssh-action-0.2.xsd")));
1841 SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
1842 Schema schema = factory.newSchema(sources.toArray(new StreamSource[sources.size()]));
1843 Validator validator = schema.newValidator();
1844 validator.validate(new StreamSource(new FileReader(file)));
1845 System.out.println("Valid workflow-app");
1846 }
1847 catch (Exception ex) {
1848 throw new OozieCLIException("Invalid app definition, " + ex.toString(), ex);
1849 }
1850 }
1851 else {
1852 throw new OozieCLIException("File does not exists");
1853 }
1854 }
1855
1856 private void scriptLanguageCommand(CommandLine commandLine, String jobType) throws IOException, OozieCLIException {
1857 List<String> args = commandLine.getArgList();
1858 if (args.size() > 0) {
1859 // checking if args starts with -X (because CLIParser cannot check this)
1860 if (!args.get(0).equals("-X")) {
1861 throw new OozieCLIException("Unrecognized option: " + args.get(0) + " Expecting -X");
1862 }
1863 args.remove(0);
1864 }
1865
1866 if (!commandLine.hasOption(SCRIPTFILE_OPTION)) {
1867 throw new OozieCLIException("Need to specify -file <scriptfile>");
1868 }
1869
1870 if (!commandLine.hasOption(CONFIG_OPTION)) {
1871 throw new OozieCLIException("Need to specify -config <configfile>");
1872 }
1873
1874 try {
1875 XOozieClient wc = createXOozieClient(commandLine);
1876 Properties conf = getConfiguration(wc, commandLine);
1877 String script = commandLine.getOptionValue(SCRIPTFILE_OPTION);
1878 List<String> paramsList = new ArrayList<String>();
1879 if (commandLine.hasOption("P")) {
1880 Properties params = commandLine.getOptionProperties("P");
1881 for (String key : params.stringPropertyNames()) {
1882 paramsList.add(key + "=" + params.getProperty(key));
1883 }
1884 }
1885 System.out.println(JOB_ID_PREFIX + wc.submitScriptLanguage(conf, script, args.toArray(new String[args.size()]),
1886 paramsList.toArray(new String[paramsList.size()]), jobType));
1887 }
1888 catch (OozieClientException ex) {
1889 throw new OozieCLIException(ex.toString(), ex);
1890 }
1891 }
1892
1893 private void sqoopCommand(CommandLine commandLine) throws IOException, OozieCLIException {
1894 List<String> args = commandLine.getArgList();
1895 if (args.size() > 0) {
1896 // checking if args starts with -X (because CLIParser cannot check this)
1897 if (!args.get(0).equals("-X")) {
1898 throw new OozieCLIException("Unrecognized option: " + args.get(0) + " Expecting -X");
1899 }
1900 args.remove(0);
1901 }
1902
1903 if (!commandLine.hasOption(SQOOP_COMMAND_OPTION)) {
1904 throw new OozieCLIException("Need to specify -command");
1905 }
1906
1907 if (!commandLine.hasOption(CONFIG_OPTION)) {
1908 throw new OozieCLIException("Need to specify -config <configfile>");
1909 }
1910
1911 try {
1912 XOozieClient wc = createXOozieClient(commandLine);
1913 Properties conf = getConfiguration(wc, commandLine);
1914 String[] command = commandLine.getOptionValues(SQOOP_COMMAND_OPTION);
1915 System.out.println(JOB_ID_PREFIX + wc.submitSqoop(conf, command, args.toArray(new String[args.size()])));
1916 }
1917 catch (OozieClientException ex) {
1918 throw new OozieCLIException(ex.toString(), ex);
1919 }
1920 }
1921
1922 private void infoCommand(CommandLine commandLine) throws OozieCLIException {
1923 for (Option option : commandLine.getOptions()) {
1924 String opt = option.getOpt();
1925 if (opt.equals(INFO_TIME_ZONES_OPTION)) {
1926 printAvailableTimeZones();
1927 }
1928 }
1929 }
1930
1931 private void printAvailableTimeZones() {
1932 System.out.println("The format is \"SHORT_NAME (ID)\"\nGive the ID to the -timezone argument");
1933 System.out.println("GMT offsets can also be used (e.g. GMT-07:00, GMT-0700, GMT+05:30, GMT+0530)");
1934 System.out.println("Available Time Zones:");
1935 for (String tzId : TimeZone.getAvailableIDs()) {
1936 // skip id's that are like "Etc/GMT+01:00" because their display names are like "GMT-01:00", which is confusing
1937 if (!tzId.startsWith("Etc/GMT")) {
1938 TimeZone tZone = TimeZone.getTimeZone(tzId);
1939 System.out.println(" " + tZone.getDisplayName(false, TimeZone.SHORT) + " (" + tzId + ")");
1940 }
1941 }
1942 }
1943
1944
1945 private void mrCommand(CommandLine commandLine) throws IOException, OozieCLIException {
1946 try {
1947 XOozieClient wc = createXOozieClient(commandLine);
1948 Properties conf = getConfiguration(wc, commandLine);
1949
1950 String mapper = conf.getProperty(MAPRED_MAPPER, conf.getProperty(MAPRED_MAPPER_2));
1951 if (mapper == null) {
1952 throw new OozieCLIException("mapper (" + MAPRED_MAPPER + " or " + MAPRED_MAPPER_2 + ") must be specified in conf");
1953 }
1954
1955 String reducer = conf.getProperty(MAPRED_REDUCER, conf.getProperty(MAPRED_REDUCER_2));
1956 if (reducer == null) {
1957 throw new OozieCLIException("reducer (" + MAPRED_REDUCER + " or " + MAPRED_REDUCER_2
1958 + ") must be specified in conf");
1959 }
1960
1961 String inputDir = conf.getProperty(MAPRED_INPUT);
1962 if (inputDir == null) {
1963 throw new OozieCLIException("input dir (" + MAPRED_INPUT +") must be specified in conf");
1964 }
1965
1966 String outputDir = conf.getProperty(MAPRED_OUTPUT);
1967 if (outputDir == null) {
1968 throw new OozieCLIException("output dir (" + MAPRED_OUTPUT +") must be specified in conf");
1969 }
1970
1971 System.out.println(JOB_ID_PREFIX + wc.submitMapReduce(conf));
1972 }
1973 catch (OozieClientException ex) {
1974 throw new OozieCLIException(ex.toString(), ex);
1975 }
1976 }
1977
1978 private String getFirstMissingDependencies(CoordinatorAction action) {
1979 StringBuilder allDeps = new StringBuilder();
1980 String missingDep = action.getMissingDependencies();
1981 boolean depExists = false;
1982 if (missingDep != null && !missingDep.isEmpty()) {
1983 allDeps.append(missingDep.split(INSTANCE_SEPARATOR)[0]);
1984 depExists = true;
1985 }
1986 String pushDeps = action.getPushMissingDependencies();
1987 if (pushDeps != null && !pushDeps.isEmpty()) {
1988 if(depExists) {
1989 allDeps.append(INSTANCE_SEPARATOR);
1990 }
1991 allDeps.append(pushDeps.split(INSTANCE_SEPARATOR)[0]);
1992 }
1993 return allDeps.toString();
1994 }
1995
1996 }