001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie;
019
020 import java.io.IOException;
021 import java.io.Writer;
022 import java.text.ParseException;
023 import java.util.ArrayList;
024 import java.util.Date;
025 import java.util.HashMap;
026 import java.util.HashSet;
027 import java.util.List;
028 import java.util.Map;
029 import java.util.Set;
030 import java.util.StringTokenizer;
031
032 import org.apache.commons.lang.StringUtils;
033 import org.apache.hadoop.conf.Configuration;
034 import org.apache.oozie.client.CoordinatorAction;
035 import org.apache.oozie.client.CoordinatorJob;
036 import org.apache.oozie.client.Job;
037 import org.apache.oozie.client.OozieClient;
038 import org.apache.oozie.client.WorkflowJob;
039 import org.apache.oozie.client.rest.BulkResponseImpl;
040 import org.apache.oozie.command.BulkJobsXCommand;
041 import org.apache.oozie.command.CommandException;
042 import org.apache.oozie.command.bundle.BundleJobChangeXCommand;
043 import org.apache.oozie.command.bundle.BundleJobResumeXCommand;
044 import org.apache.oozie.command.bundle.BundleJobSuspendXCommand;
045 import org.apache.oozie.command.bundle.BundleJobXCommand;
046 import org.apache.oozie.command.bundle.BundleJobsXCommand;
047 import org.apache.oozie.command.bundle.BundleKillXCommand;
048 import org.apache.oozie.command.bundle.BundleRerunXCommand;
049 import org.apache.oozie.command.bundle.BundleStartXCommand;
050 import org.apache.oozie.command.bundle.BundleSubmitXCommand;
051 import org.apache.oozie.service.DagXLogInfoService;
052 import org.apache.oozie.service.Services;
053 import org.apache.oozie.service.XLogStreamingService;
054 import org.apache.oozie.util.DateUtils;
055 import org.apache.oozie.util.XLogFilter;
056 import org.apache.oozie.util.XLogUserFilterParam;
057 import org.apache.oozie.util.ParamChecker;
058 import org.apache.oozie.util.XLog;
059
060 import com.google.common.annotations.VisibleForTesting;
061
062 public class BundleEngine extends BaseEngine {
063 /**
064 * Create a system Bundle engine, with no user and no group.
065 */
066 public BundleEngine() {
067 }
068
069 /**
070 * Create a Bundle engine to perform operations on behave of a user.
071 *
072 * @param user user name.
073 */
074 public BundleEngine(String user) {
075 this.user = ParamChecker.notEmpty(user, "user");
076 }
077
078 /* (non-Javadoc)
079 * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String)
080 */
081 @Override
082 public void change(String jobId, String changeValue) throws BundleEngineException {
083 try {
084 BundleJobChangeXCommand change = new BundleJobChangeXCommand(jobId, changeValue);
085 change.call();
086 }
087 catch (CommandException ex) {
088 throw new BundleEngineException(ex);
089 }
090 }
091
092 /* (non-Javadoc)
093 * @see org.apache.oozie.BaseEngine#dryRunSubmit(org.apache.hadoop.conf.Configuration)
094 */
095 @Override
096 public String dryRunSubmit(Configuration conf) throws BundleEngineException {
097 BundleSubmitXCommand submit = new BundleSubmitXCommand(true, conf);
098 try {
099 String jobId = submit.call();
100 return jobId;
101 }
102 catch (CommandException ex) {
103 throw new BundleEngineException(ex);
104 }
105 }
106
107 /* (non-Javadoc)
108 * @see org.apache.oozie.BaseEngine#getCoordJob(java.lang.String)
109 */
110 @Override
111 public CoordinatorJob getCoordJob(String jobId) throws BundleEngineException {
112 throw new BundleEngineException(new XException(ErrorCode.E0301, "cannot get a coordinator job from BundleEngine"));
113 }
114
115 public BundleJobBean getBundleJob(String jobId) throws BundleEngineException {
116 try {
117 return new BundleJobXCommand(jobId).call();
118 }
119 catch (CommandException ex) {
120 throw new BundleEngineException(ex);
121 }
122 }
123
124 /* (non-Javadoc)
125 * @see org.apache.oozie.BaseEngine#getCoordJob(java.lang.String, int, int)
126 */
127 @Override
128 public CoordinatorJob getCoordJob(String jobId, String filter, int start, int length, boolean desc)
129 throws BundleEngineException {
130 throw new BundleEngineException(new XException(ErrorCode.E0301,
131 "cannot get a coordinator job from BundleEngine"));
132 }
133
134 /* (non-Javadoc)
135 * @see org.apache.oozie.BaseEngine#getDefinition(java.lang.String)
136 */
137 @Override
138 public String getDefinition(String jobId) throws BundleEngineException {
139 BundleJobBean job;
140 try {
141 job = new BundleJobXCommand(jobId).call();
142 }
143 catch (CommandException ex) {
144 throw new BundleEngineException(ex);
145 }
146 return job.getOrigJobXml();
147 }
148
149 /* (non-Javadoc)
150 * @see org.apache.oozie.BaseEngine#getJob(java.lang.String)
151 */
152 @Override
153 public WorkflowJob getJob(String jobId) throws BundleEngineException {
154 throw new BundleEngineException(new XException(ErrorCode.E0301, "cannot get a workflow job from BundleEngine"));
155 }
156
157 /* (non-Javadoc)
158 * @see org.apache.oozie.BaseEngine#getJob(java.lang.String, int, int)
159 */
160 @Override
161 public WorkflowJob getJob(String jobId, int start, int length) throws BundleEngineException {
162 throw new BundleEngineException(new XException(ErrorCode.E0301, "cannot get a workflow job from BundleEngine"));
163 }
164
165 /* (non-Javadoc)
166 * @see org.apache.oozie.BaseEngine#getJobIdForExternalId(java.lang.String)
167 */
168 @Override
169 public String getJobIdForExternalId(String externalId) throws BundleEngineException {
170 return null;
171 }
172
173 /* (non-Javadoc)
174 * @see org.apache.oozie.BaseEngine#kill(java.lang.String)
175 */
176 @Override
177 public void kill(String jobId) throws BundleEngineException {
178 try {
179 new BundleKillXCommand(jobId).call();
180 }
181 catch (CommandException e) {
182 throw new BundleEngineException(e);
183 }
184 }
185
186 /* (non-Javadoc)
187 * @see org.apache.oozie.BaseEngine#reRun(java.lang.String, org.apache.hadoop.conf.Configuration)
188 */
189 @Override
190 @Deprecated
191 public void reRun(String jobId, Configuration conf) throws BundleEngineException {
192 throw new BundleEngineException(new XException(ErrorCode.E0301, "rerun"));
193 }
194
195 /**
196 * Rerun Bundle actions for given rerunType
197 *
198 * @param jobId bundle job id
199 * @param coordScope the rerun scope for coordinator job names separated by ","
200 * @param dateScope the rerun scope for coordinator nominal times separated by ","
201 * @param refresh true if user wants to refresh input/outpur dataset urls
202 * @param noCleanup false if user wants to cleanup output events for given rerun actions
203 * @throws BaseEngineException thrown if failed to rerun
204 */
205 public void reRun(String jobId, String coordScope, String dateScope, boolean refresh, boolean noCleanup)
206 throws BaseEngineException {
207 try {
208 new BundleRerunXCommand(jobId, coordScope, dateScope, refresh, noCleanup).call();
209 }
210 catch (CommandException ex) {
211 throw new BaseEngineException(ex);
212 }
213 }
214
215 /* (non-Javadoc)
216 * @see org.apache.oozie.BaseEngine#resume(java.lang.String)
217 */
218 @Override
219 public void resume(String jobId) throws BundleEngineException {
220 BundleJobResumeXCommand resume = new BundleJobResumeXCommand(jobId);
221 try {
222 resume.call();
223 }
224 catch (CommandException ex) {
225 throw new BundleEngineException(ex);
226 }
227 }
228
229 /* (non-Javadoc)
230 * @see org.apache.oozie.BaseEngine#start(java.lang.String)
231 */
232 @Override
233 public void start(String jobId) throws BundleEngineException {
234 try {
235 new BundleStartXCommand(jobId).call();
236 }
237 catch (CommandException e) {
238 throw new BundleEngineException(e);
239 }
240 }
241
242 /* (non-Javadoc)
243 * @see org.apache.oozie.BaseEngine#streamLog(java.lang.String, java.io.Writer)
244 */
245 @Override
246 public void streamLog(String jobId, Writer writer, Map<String, String[]> params) throws IOException,
247 BundleEngineException {
248
249 BundleJobBean job;
250 try {
251 XLogFilter filter = new XLogFilter(new XLogUserFilterParam(params));
252 filter.setParameter(DagXLogInfoService.JOB, jobId);
253 job = new BundleJobXCommand(jobId).call();
254 Date lastTime = null;
255 if (job.isTerminalStatus()) {
256 lastTime = job.getLastModifiedTime();
257 }
258 if (lastTime == null) {
259 lastTime = new Date();
260 }
261 Services.get().get(XLogStreamingService.class).streamLog(filter, job.getCreatedTime(), lastTime, writer, params);
262 }
263 catch (Exception ex) {
264 throw new IOException(ex);
265 }
266 }
267
268 /* (non-Javadoc)
269 * @see org.apache.oozie.BaseEngine#submitJob(org.apache.hadoop.conf.Configuration, boolean)
270 */
271 @Override
272 public String submitJob(Configuration conf, boolean startJob) throws BundleEngineException {
273 try {
274 String jobId = new BundleSubmitXCommand(conf).call();
275
276 if (startJob) {
277 start(jobId);
278 }
279 return jobId;
280 }
281 catch (CommandException ex) {
282 throw new BundleEngineException(ex);
283 }
284 }
285
286 /* (non-Javadoc)
287 * @see org.apache.oozie.BaseEngine#suspend(java.lang.String)
288 */
289 @Override
290 public void suspend(String jobId) throws BundleEngineException {
291 BundleJobSuspendXCommand suspend = new BundleJobSuspendXCommand(jobId);
292 try {
293 suspend.call();
294 }
295 catch (CommandException ex) {
296 throw new BundleEngineException(ex);
297 }
298 }
299
300 private static final Set<String> FILTER_NAMES = new HashSet<String>();
301
302 static {
303 FILTER_NAMES.add(OozieClient.FILTER_USER);
304 FILTER_NAMES.add(OozieClient.FILTER_NAME);
305 FILTER_NAMES.add(OozieClient.FILTER_GROUP);
306 FILTER_NAMES.add(OozieClient.FILTER_STATUS);
307 FILTER_NAMES.add(OozieClient.FILTER_ID);
308 }
309
310 /**
311 * Get bundle jobs
312 *
313 * @param filter the filter string
314 * @param start start location for paging
315 * @param len total length to get
316 * @return bundle job info
317 * @throws BundleEngineException thrown if failed to get bundle job info
318 */
319 public BundleJobInfo getBundleJobs(String filter, int start, int len) throws BundleEngineException {
320 Map<String, List<String>> filterList = parseFilter(filter);
321
322 try {
323 return new BundleJobsXCommand(filterList, start, len).call();
324 }
325 catch (CommandException ex) {
326 throw new BundleEngineException(ex);
327 }
328 }
329
330 /**
331 * Parse filter string to a map with key = filter name and values = filter values
332 *
333 * @param filter the filter string
334 * @return filter key and value map
335 * @throws CoordinatorEngineException thrown if failed to parse filter string
336 */
337 @VisibleForTesting
338 Map<String, List<String>> parseFilter(String filter) throws BundleEngineException {
339 Map<String, List<String>> map = new HashMap<String, List<String>>();
340 if (filter != null) {
341 StringTokenizer st = new StringTokenizer(filter, ";");
342 while (st.hasMoreTokens()) {
343 String token = st.nextToken();
344 if (token.contains("=")) {
345 String[] pair = token.split("=");
346 if (pair.length != 2) {
347 throw new BundleEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
348 }
349 if (!FILTER_NAMES.contains(pair[0])) {
350 throw new BundleEngineException(ErrorCode.E0420, filter, XLog.format("invalid name [{0}]",
351 pair[0]));
352 }
353 if (pair[0].equals("status")) {
354 try {
355 Job.Status.valueOf(pair[1]);
356 }
357 catch (IllegalArgumentException ex) {
358 throw new BundleEngineException(ErrorCode.E0420, filter, XLog.format(
359 "invalid status [{0}]", pair[1]));
360 }
361 }
362 List<String> list = map.get(pair[0]);
363 if (list == null) {
364 list = new ArrayList<String>();
365 map.put(pair[0], list);
366 }
367 list.add(pair[1]);
368 }
369 else {
370 throw new BundleEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
371 }
372 }
373 }
374 return map;
375 }
376
377 /**
378 * Get bulk job response
379 *
380 * @param filter the filter string
381 * @param start start location for paging
382 * @param len total length to get
383 * @return bulk job info
384 * @throws BundleEngineException thrown if failed to get bulk job info
385 */
386 public BulkResponseInfo getBulkJobs(String bulkFilter, int start, int len) throws BundleEngineException {
387 Map<String,List<String>> bulkRequestMap = parseBulkFilter(bulkFilter);
388 try {
389 return new BulkJobsXCommand(bulkRequestMap, start, len).call();
390 }
391 catch (CommandException ex) {
392 throw new BundleEngineException(ex);
393 }
394 }
395
396 /**
397 * Parse filter string to a map with key = filter name and values = filter values
398 * Allowed keys are defined as constants on top
399 *
400 * @param filter the filter string
401 * @return filter key-value pair map
402 * @throws BundleEngineException thrown if failed to parse filter string
403 */
404 public static Map<String,List<String>> parseBulkFilter(String bulkParams) throws BundleEngineException {
405
406 Map<String,List<String>> bulkFilter = new HashMap<String,List<String>>();
407 // Functionality can be extended to different job levels - TODO extend filter parser and query
408 // E.g. String filterlevel = "coordinatoraction"; BulkResponseImpl.BULK_FILTER_LEVEL
409 if (bulkFilter != null) {
410 StringTokenizer st = new StringTokenizer(bulkParams, ";");
411 while (st.hasMoreTokens()) {
412 String token = st.nextToken();
413 if (token.contains("=")) {
414 String[] pair = token.split("=");
415 if (pair.length != 2) {
416 throw new BundleEngineException(ErrorCode.E0420, token,
417 "elements must be name=value pairs");
418 }
419 pair[0] = pair[0].toLowerCase();
420 String[] values = pair[1].split(",");
421 if (!BulkResponseImpl.BULK_FILTER_NAMES.contains(pair[0])) {
422 throw new BundleEngineException(ErrorCode.E0420, token, XLog.format("invalid parameter name [{0}]",
423 pair[0]));
424 }
425 // special check and processing for time related params
426 if (pair[0].contains("time")) {
427 try {
428 DateUtils.parseDateUTC(pair[1]);
429 }
430 catch (ParseException e) {
431 throw new BundleEngineException(ErrorCode.E0420, token, XLog.format(
432 "invalid value [{0}] for time. A datetime value of pattern [{1}] is expected", pair[1],
433 DateUtils.ISO8601_UTC_MASK));
434 }
435 }
436 // special check for action status param
437 // TODO: when extended for levels other than coord action, check against corresponding level's Status values
438 if (pair[0].equals(BulkResponseImpl.BULK_FILTER_STATUS)) {
439 for(String value : values) {
440 try {
441 CoordinatorAction.Status.valueOf(value);
442 }
443 catch (IllegalArgumentException ex) {
444 throw new BundleEngineException(ErrorCode.E0420, token, XLog.format(
445 "invalid action status [{0}]", value));
446 }
447 }
448 }
449 // eventually adding into map for all cases e.g. names, times, status
450 List<String> list = bulkFilter.get(pair[0]);
451 if (list == null) {
452 list = new ArrayList<String>();
453 bulkFilter.put(pair[0], list);
454 }
455 for(String value : values) {
456 value = value.trim();
457 if(value.isEmpty()) {
458 throw new BundleEngineException(ErrorCode.E0420, token, "value is empty or whitespace");
459 }
460 list.add(value);
461 }
462 } else {
463 throw new BundleEngineException(ErrorCode.E0420, token, "elements must be name=value pairs");
464 }
465 }
466 if (!bulkFilter.containsKey(BulkResponseImpl.BULK_FILTER_BUNDLE)) {
467 throw new BundleEngineException(ErrorCode.E0305, BulkResponseImpl.BULK_FILTER_BUNDLE);
468 }
469 }
470 return bulkFilter;
471 }
472 }