001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.command;
019
020 import org.apache.oozie.ErrorCode;
021 import org.apache.oozie.FaultInjection;
022 import org.apache.oozie.XException;
023 import org.apache.oozie.service.CallableQueueService;
024 import org.apache.oozie.service.EventHandlerService;
025 import org.apache.oozie.service.InstrumentationService;
026 import org.apache.oozie.service.MemoryLocksService;
027 import org.apache.oozie.service.Services;
028 import org.apache.oozie.util.Instrumentation;
029 import org.apache.oozie.lock.LockToken;
030 import org.apache.oozie.util.XCallable;
031 import org.apache.oozie.util.XLog;
032
033 import java.util.ArrayList;
034 import java.util.HashMap;
035 import java.util.List;
036 import java.util.Map;
037 import java.util.Set;
038 import java.util.UUID;
039 import java.util.concurrent.atomic.AtomicBoolean;
040
041 /**
042 * Base class for synchronous and asynchronous commands.
043 * <p/>
044 * It enables by API the following pattern:
045 * <p/>
046 * <ul>
047 * <li>single execution: a command instance can be executed only once</li>
048 * <li>eager data loading: loads data for eager precondition check</li>
049 * <li>eager precondition check: verify precondition before obtaining lock</li>
050 * <li>data loading: loads data for precondition check and execution</li>
051 * <li>precondition check: verifies precondition for execution is still met</li>
052 * <li>locking: obtains exclusive lock on key before executing the command</li>
053 * <li>execution: command logic</li>
054 * </ul>
055 * <p/>
056 * It has built in instrumentation and logging.
057 */
058 public abstract class XCommand<T> implements XCallable<T> {
059 public static final String DEFAULT_LOCK_TIMEOUT = "oozie.command.default.lock.timeout";
060
061 public static final String INSTRUMENTATION_GROUP = "commands";
062
063 public static final Long DEFAULT_REQUEUE_DELAY = 10L;
064
065 public XLog LOG = XLog.getLog(getClass());
066
067 private String key;
068 private String name;
069 private int priority;
070 private String type;
071 private long createdTime;
072 private LockToken lock;
073 private AtomicBoolean used = new AtomicBoolean(false);
074 private boolean inInterrupt = false;
075 private boolean isSynchronous = false;
076
077 private Map<Long, List<XCommand<?>>> commandQueue;
078 protected boolean dryrun = false;
079 protected Instrumentation instrumentation;
080
081 protected XLog.Info logInfo;
082 protected static EventHandlerService eventService;
083
084 /**
085 * Create a command.
086 *
087 * @param name command name.
088 * @param type command type.
089 * @param priority command priority.
090 */
091 public XCommand(String name, String type, int priority) {
092 this.name = name;
093 this.type = type;
094 this.priority = priority;
095 this.key = name + "_" + UUID.randomUUID();
096 createdTime = System.currentTimeMillis();
097 logInfo = new XLog.Info();
098 instrumentation = Services.get().get(InstrumentationService.class).get();
099 eventService = Services.get().get(EventHandlerService.class);
100 }
101
102 /**
103 * @param name command name.
104 * @param type command type.
105 * @param priority command priority.
106 * @param dryrun indicates if dryrun option is enabled. if enabled bundle will show a diagnostic output without
107 * really running the job
108 */
109 public XCommand(String name, String type, int priority, boolean dryrun) {
110 this(name, type, priority);
111 this.dryrun = dryrun;
112 }
113
114 /**
115 * Return the command name.
116 *
117 * @return the command name.
118 */
119 @Override
120 public String getName() {
121 return name;
122 }
123
124 /**
125 * Return the callable type.
126 * <p/>
127 * The command type is used for concurrency throttling in the {@link CallableQueueService}.
128 *
129 * @return the command type.
130 */
131 @Override
132 public String getType() {
133 return type;
134 }
135
136 /**
137 * Return the priority of the command.
138 *
139 * @return the command priority.
140 */
141 @Override
142 public int getPriority() {
143 return priority;
144 }
145
146 /**
147 * Returns the creation time of the command.
148 *
149 * @return the command creation time, in milliseconds.
150 */
151 @Override
152 public long getCreatedTime() {
153 return createdTime;
154 }
155
156 /**
157 * Queue a command for execution after the current command execution completes.
158 * <p/>
159 * All commands queued during the execution of the current command will be queued for a single serial execution.
160 * <p/>
161 * If the command execution throws an exception, no command will be effectively queued.
162 *
163 * @param command command to queue.
164 */
165 protected void queue(XCommand<?> command) {
166 queue(command, 0);
167 }
168
169 /**
170 * Queue a command for delayed execution after the current command execution completes.
171 * <p/>
172 * All commands queued during the execution of the current command with the same delay will be queued for a single
173 * serial execution.
174 * <p/>
175 * If the command execution throws an exception, no command will be effectively queued.
176 *
177 * @param command command to queue.
178 * @param msDelay delay in milliseconds.
179 */
180 protected void queue(XCommand<?> command, long msDelay) {
181 if (commandQueue == null) {
182 commandQueue = new HashMap<Long, List<XCommand<?>>>();
183 }
184 List<XCommand<?>> list = commandQueue.get(msDelay);
185 if (list == null) {
186 list = new ArrayList<XCommand<?>>();
187 commandQueue.put(msDelay, list);
188 }
189 list.add(command);
190 }
191
192 /**
193 * Obtain an exclusive lock on the {link #getEntityKey}.
194 * <p/>
195 * A timeout of {link #getLockTimeOut} is used when trying to obtain the lock.
196 *
197 * @throws InterruptedException thrown if an interruption happened while trying to obtain the lock
198 * @throws CommandException thrown i the lock could not be obtained.
199 */
200 private void acquireLock() throws InterruptedException, CommandException {
201 if (getEntityKey() == null) {
202 // no lock for null entity key
203 return;
204 }
205 lock = Services.get().get(MemoryLocksService.class).getWriteLock(getEntityKey(), getLockTimeOut());
206 if (lock == null) {
207 instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".lockTimeOut", 1);
208 if (isReQueueRequired()) {
209 //if not acquire the lock, re-queue itself with default delay
210 queue(this, getRequeueDelay());
211 LOG.debug("Could not get lock [{0}], timed out [{1}]ms, and requeue itself [{2}]", this.toString(), getLockTimeOut(), getName());
212 } else {
213 throw new CommandException(ErrorCode.E0606, this.toString(), getLockTimeOut());
214 }
215 } else {
216 LOG.debug("Acquired lock for [{0}] in [{1}]", getEntityKey(), getName());
217 }
218 }
219
220 /**
221 * Release the lock on the {link #getEntityKey}.
222 */
223 private void releaseLock() {
224 if (lock != null) {
225 lock.release();
226 LOG.debug("Released lock for [{0}] in [{1}]", getEntityKey(), getName());
227 }
228 }
229
230 /**
231 * Implements the XCommand life-cycle.
232 *
233 * @return the {link #execute} return value.
234 * @throws Exception thrown if the command could not be executed.
235 */
236 @Override
237 public final T call() throws CommandException {
238 if (CallableQueueService.INTERRUPT_TYPES.contains(this.getType()) && used.get()) {
239 LOG.debug("Command [{0}] key [{1}] already used for [{2}]", getName(), getEntityKey(), this.toString());
240 return null;
241 }
242
243 commandQueue = null;
244 instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".executions", 1);
245 Instrumentation.Cron callCron = new Instrumentation.Cron();
246 try {
247 callCron.start();
248 if (!isSynchronous) {
249 eagerLoadState();
250 LOG = XLog.resetPrefix(LOG);
251 eagerVerifyPrecondition();
252 }
253 try {
254 T ret = null;
255 if (!isSynchronous && isLockRequired() && !this.inInterruptMode()) {
256 Instrumentation.Cron acquireLockCron = new Instrumentation.Cron();
257 acquireLockCron.start();
258 acquireLock();
259 acquireLockCron.stop();
260 instrumentation.addCron(INSTRUMENTATION_GROUP, getName() + ".acquireLock", acquireLockCron);
261 }
262 // executing interrupts only in case of the lock required commands
263 if (lock != null) {
264 this.executeInterrupts();
265 }
266
267 if (isSynchronous || !isLockRequired() || (lock != null) || this.inInterruptMode()) {
268 if (CallableQueueService.INTERRUPT_TYPES.contains(this.getType())
269 && !used.compareAndSet(false, true)) {
270 LOG.debug("Command [{0}] key [{1}] already executed for [{2}]", getName(), getEntityKey(), this.toString());
271 return null;
272 }
273 LOG.trace("Load state for [{0}]", getEntityKey());
274 loadState();
275 LOG = XLog.resetPrefix(LOG);
276 LOG.trace("Precondition check for command [{0}] key [{1}]", getName(), getEntityKey());
277 verifyPrecondition();
278 LOG.debug("Execute command [{0}] key [{1}]", getName(), getEntityKey());
279 Instrumentation.Cron executeCron = new Instrumentation.Cron();
280 executeCron.start();
281 ret = execute();
282 executeCron.stop();
283 instrumentation.addCron(INSTRUMENTATION_GROUP, getName() + ".execute", executeCron);
284 }
285 if (commandQueue != null) {
286 CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
287 for (Map.Entry<Long, List<XCommand<?>>> entry : commandQueue.entrySet()) {
288 LOG.debug("Queuing [{0}] commands with delay [{1}]ms", entry.getValue().size(), entry.getKey());
289 if (!callableQueueService.queueSerial(entry.getValue(), entry.getKey())) {
290 LOG.warn("Could not queue [{0}] commands with delay [{1}]ms, queue full", entry.getValue()
291 .size(), entry.getKey());
292 }
293 }
294 }
295 return ret;
296 }
297 finally {
298 if (!isSynchronous && isLockRequired() && !this.inInterruptMode()) {
299 releaseLock();
300 }
301 }
302 }
303 catch(PreconditionException pex){
304 LOG.warn(pex.getMessage().toString() + ", Error Code: " + pex.getErrorCode().toString());
305 instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".preconditionfailed", 1);
306 return null;
307 }
308 catch (XException ex) {
309 LOG.error("XException, ", ex);
310 instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".xexceptions", 1);
311 if (ex instanceof CommandException) {
312 throw (CommandException) ex;
313 }
314 else {
315 throw new CommandException(ex);
316 }
317 }
318 catch (Exception ex) {
319 LOG.error("Exception, ", ex);
320 instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".exceptions", 1);
321 throw new CommandException(ErrorCode.E0607, getName(), ex.getMessage(), ex);
322 }
323 catch (Error er) {
324 LOG.error("Error, ", er);
325 instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".errors", 1);
326 throw er;
327 }
328 finally {
329 FaultInjection.deactivate("org.apache.oozie.command.SkipCommitFaultInjection");
330 callCron.stop();
331 instrumentation.addCron(INSTRUMENTATION_GROUP, getName() + ".call", callCron);
332 }
333 }
334
335 /**
336 * Call this command synchronously from its caller. This benefits faster
337 * execution of command lifecycle for control nodes and kicking off
338 * subsequent actions
339 *
340 * @param callerEntityKey
341 * @return the {link #execute} return value.
342 * @throws CommandException
343 */
344 public final T call(String callerEntityKey) throws CommandException {
345 if (!callerEntityKey.equals(this.getEntityKey())) {
346 throw new CommandException(ErrorCode.E0607, "Entity Keys mismatch during synchronous call", "caller="
347 + callerEntityKey + ", callee=" + getEntityKey());
348 }
349 isSynchronous = true; //setting to true so lock acquiring and release is not repeated
350 LOG.trace("Executing synchronously command [{0}] on job [{1}]", this.getName(), this.getKey());
351 return call();
352 }
353
354 /**
355 * Check for the existence of interrupts for the same lock key
356 * Execute them if exist.
357 *
358 */
359 protected void executeInterrupts() {
360 CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
361 // getting all the list of interrupts to be executed
362 Set<XCallable<?>> callables = callableQueueService.checkInterrupts(this.getEntityKey());
363
364 if (callables != null) {
365 // executing the list of interrupts in the given order of insertion
366 // in the list
367 for (XCallable<?> callable : callables) {
368 LOG.trace("executing interrupt callable [{0}]", callable.getName());
369 try {
370 // executing the callable in interrupt mode
371 callable.setInterruptMode(true);
372 callable.call();
373 LOG.trace("executed interrupt callable [{0}]", callable.getName());
374 }
375 catch (Exception ex) {
376 LOG.warn("exception interrupt callable [{0}], {1}", callable.getName(), ex.getMessage(), ex);
377 }
378 finally {
379 // reseting the interrupt mode to false after the command is
380 // executed
381 callable.setInterruptMode(false);
382 }
383 }
384 }
385 }
386
387 /**
388 * Return the time out when acquiring a lock.
389 * <p/>
390 * The value is loaded from the Oozie configuration, the property {link #DEFAULT_LOCK_TIMEOUT}.
391 * <p/>
392 * Subclasses should override this method if they want to use a different time out.
393 *
394 * @return the lock time out in milliseconds.
395 */
396 protected long getLockTimeOut() {
397 return Services.get().getConf().getLong(DEFAULT_LOCK_TIMEOUT, 5 * 1000);
398 }
399
400 /**
401 * Indicate if the the command requires locking.
402 * <p/>
403 * Subclasses should override this method if they require locking.
404 *
405 * @return <code>true/false</code>
406 */
407 protected abstract boolean isLockRequired();
408
409 /**
410 * Return the entity key for the command.
411 * <p/>
412 *
413 * @return the entity key for the command.
414 */
415 public abstract String getEntityKey();
416
417 /**
418 * Indicate if the the command requires to requeue itself if the lock is not acquired.
419 * <p/>
420 * Subclasses should override this method if they don't want to requeue.
421 * <p/>
422 * Default is true.
423 *
424 * @return <code>true/false</code>
425 */
426 protected boolean isReQueueRequired() {
427 return true;
428 }
429
430 /**
431 * Load the necessary state to perform an eager precondition check.
432 * <p/>
433 * This implementation does a NOP.
434 * <p/>
435 * Subclasses should override this method and load the state needed to do an eager precondition check.
436 * <p/>
437 * A trivial implementation is calling {link #loadState}.
438 */
439 protected void eagerLoadState() throws CommandException{
440 }
441
442 /**
443 * Verify the precondition for the command before obtaining a lock.
444 * <p/>
445 * This implementation does a NOP.
446 * <p/>
447 * A trivial implementation is calling {link #verifyPrecondition}.
448 *
449 * @throws CommandException thrown if the precondition is not met.
450 */
451 protected void eagerVerifyPrecondition() throws CommandException,PreconditionException {
452 }
453
454 /**
455 * Load the necessary state to perform the precondition check and to execute the command.
456 * <p/>
457 * Subclasses must implement this method and load the state needed to do the precondition check and execute the
458 * command.
459 */
460 protected abstract void loadState() throws CommandException;
461
462 /**
463 * Verify the precondition for the command after a lock has been obtain, just before executing the command.
464 * <p/>
465 *
466 * @throws CommandException thrown if the precondition is not met.
467 */
468 protected abstract void verifyPrecondition() throws CommandException,PreconditionException;
469
470 /**
471 * Command execution body.
472 * <p/>
473 * This method will be invoked after the {link #loadState} and {link #verifyPrecondition} methods.
474 * <p/>
475 * If the command requires locking, this method will be invoked ONLY if the lock has been acquired.
476 *
477 * @return a return value from the execution of the command, only meaningful if the command is executed
478 * synchronously.
479 * @throws CommandException thrown if the command execution failed.
480 */
481 protected abstract T execute() throws CommandException;
482
483
484 /**
485 * Return the {@link Instrumentation} instance in use.
486 *
487 * @return the {@link Instrumentation} instance in use.
488 */
489 protected Instrumentation getInstrumentation() {
490 return instrumentation;
491 }
492
493 /**
494 * @param used set false to the used
495 */
496 public void resetUsed() {
497 this.used.set(false);
498 }
499
500
501 /**
502 * Return the delay time for requeue
503 *
504 * @return delay time when requeue itself
505 */
506 protected Long getRequeueDelay() {
507 return DEFAULT_REQUEUE_DELAY;
508 }
509
510 /**
511 * Get command key
512 *
513 * @return command key
514 */
515 @Override
516 public String getKey(){
517 return this.key;
518 }
519
520 /**
521 * set the mode of execution for the callable. True if in interrupt, false
522 * if not
523 */
524 public void setInterruptMode(boolean mode) {
525 this.inInterrupt = mode;
526 }
527
528 /**
529 * @return the mode of execution. true if it is executed as an Interrupt,
530 * false otherwise
531 */
532 public boolean inInterruptMode() {
533 return this.inInterrupt;
534 }
535
536 /**
537 * Get XLog log
538 *
539 * @return XLog
540 */
541 public XLog getLog() {
542 return LOG;
543 }
544
545 /**
546 * String for the command - key
547 * @return String
548 */
549 @Override
550 public String toString() {
551 return getKey();
552 }
553 }