001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.command.coord;
019
020 import java.util.ArrayList;
021 import java.util.Date;
022 import java.util.HashSet;
023 import java.util.List;
024 import java.util.Map;
025 import java.util.Map.Entry;
026 import java.util.Set;
027
028 import org.apache.commons.lang.StringUtils;
029 import org.apache.oozie.CoordinatorActionBean;
030 import org.apache.oozie.CoordinatorJobBean;
031 import org.apache.oozie.ErrorCode;
032 import org.apache.oozie.XException;
033 import org.apache.oozie.client.CoordinatorAction;
034 import org.apache.oozie.client.CoordinatorJob;
035 import org.apache.oozie.client.Job;
036 import org.apache.oozie.client.OozieClient;
037 import org.apache.oozie.client.rest.JsonBean;
038 import org.apache.oozie.command.CommandException;
039 import org.apache.oozie.command.PreconditionException;
040 import org.apache.oozie.command.bundle.BundleStatusUpdateXCommand;
041 import org.apache.oozie.executor.jpa.CoordActionGetJPAExecutor;
042 import org.apache.oozie.executor.jpa.CoordJobGetActionByActionNumberJPAExecutor;
043 import org.apache.oozie.executor.jpa.CoordJobGetJPAExecutor;
044 import org.apache.oozie.executor.jpa.SLARegistrationQueryExecutor;
045 import org.apache.oozie.executor.jpa.SLARegistrationQueryExecutor.SLARegQuery;
046 import org.apache.oozie.executor.jpa.SLASummaryQueryExecutor;
047 import org.apache.oozie.executor.jpa.CoordJobQueryExecutor.CoordJobQuery;
048 import org.apache.oozie.executor.jpa.SLASummaryQueryExecutor.SLASummaryQuery;
049 import org.apache.oozie.executor.jpa.JPAExecutorException;
050 import org.apache.oozie.executor.jpa.BatchQueryExecutor;
051 import org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry;
052 import org.apache.oozie.service.JPAService;
053 import org.apache.oozie.service.Services;
054 import org.apache.oozie.sla.SLARegistrationBean;
055 import org.apache.oozie.sla.SLASummaryBean;
056 import org.apache.oozie.sla.service.SLAService;
057 import org.apache.oozie.util.DateUtils;
058 import org.apache.oozie.util.JobUtils;
059 import org.apache.oozie.util.LogUtils;
060 import org.apache.oozie.util.ParamChecker;
061 import org.apache.oozie.util.StatusUtils;
062
063 public class CoordChangeXCommand extends CoordinatorXCommand<Void> {
064 private final String jobId;
065 private Date newEndTime = null;
066 private Integer newConcurrency = null;
067 private Date newPauseTime = null;
068 private Date oldPauseTime = null;
069 private boolean resetPauseTime = false;
070 private CoordinatorJob.Status jobStatus = null;
071 private CoordinatorJobBean coordJob;
072 private JPAService jpaService = null;
073 private Job.Status prevStatus;
074 private List<UpdateEntry> updateList = new ArrayList<UpdateEntry>();
075 private List<JsonBean> deleteList = new ArrayList<JsonBean>();
076
077 private static final Set<String> ALLOWED_CHANGE_OPTIONS = new HashSet<String>();
078 static {
079 ALLOWED_CHANGE_OPTIONS.add("endtime");
080 ALLOWED_CHANGE_OPTIONS.add("concurrency");
081 ALLOWED_CHANGE_OPTIONS.add("pausetime");
082 ALLOWED_CHANGE_OPTIONS.add(OozieClient.CHANGE_VALUE_STATUS);
083
084 }
085
086 /**
087 * This command is used to update the Coordinator job with the new values Update the coordinator job bean and update
088 * that to database.
089 *
090 * @param id Coordinator job id.
091 * @param changeValue This the changed value in the form key=value.
092 * @throws CommandException thrown if changeValue cannot be parsed properly.
093 */
094 public CoordChangeXCommand(String id, String changeValue) throws CommandException {
095 super("coord_change", "coord_change", 0);
096 this.jobId = ParamChecker.notEmpty(id, "id");
097 ParamChecker.notEmpty(changeValue, "value");
098
099 validateChangeValue(changeValue);
100 }
101
102 /**
103 * @param changeValue change value.
104 * @throws CommandException thrown if changeValue cannot be parsed properly.
105 */
106 private void validateChangeValue(String changeValue) throws CommandException {
107 Map<String, String> map = JobUtils.parseChangeValue(changeValue);
108
109 if (map.size() > ALLOWED_CHANGE_OPTIONS.size()) {
110 throw new CommandException(ErrorCode.E1015, changeValue, "must change endtime|concurrency|pausetime|status");
111 }
112
113 java.util.Iterator<Entry<String, String>> iter = map.entrySet().iterator();
114 while (iter.hasNext()) {
115 Entry<String, String> entry = iter.next();
116 String key = entry.getKey();
117 String value = entry.getValue();
118
119 if (!ALLOWED_CHANGE_OPTIONS.contains(key)) {
120 throw new CommandException(ErrorCode.E1015, changeValue, "must change endtime|concurrency|pausetime|status");
121 }
122
123 if (!key.equals(OozieClient.CHANGE_VALUE_PAUSETIME) && value.equalsIgnoreCase("")) {
124 throw new CommandException(ErrorCode.E1015, changeValue, "value on " + key + " can not be empty");
125 }
126 }
127
128 if (map.containsKey(OozieClient.CHANGE_VALUE_ENDTIME)) {
129 String value = map.get(OozieClient.CHANGE_VALUE_ENDTIME);
130 try {
131 newEndTime = DateUtils.parseDateOozieTZ(value);
132 }
133 catch (Exception ex) {
134 throw new CommandException(ErrorCode.E1015, value, "must be a valid date");
135 }
136 }
137
138 if (map.containsKey(OozieClient.CHANGE_VALUE_CONCURRENCY)) {
139 String value = map.get(OozieClient.CHANGE_VALUE_CONCURRENCY);
140 try {
141 newConcurrency = Integer.parseInt(value);
142 }
143 catch (NumberFormatException ex) {
144 throw new CommandException(ErrorCode.E1015, value, "must be a valid integer");
145 }
146 }
147
148 if (map.containsKey(OozieClient.CHANGE_VALUE_PAUSETIME)) {
149 String value = map.get(OozieClient.CHANGE_VALUE_PAUSETIME);
150 if (value.equals("")) { // this is to reset pause time to null;
151 resetPauseTime = true;
152 }
153 else {
154 try {
155 newPauseTime = DateUtils.parseDateOozieTZ(value);
156 }
157 catch (Exception ex) {
158 throw new CommandException(ErrorCode.E1015, value, "must be a valid date");
159 }
160 }
161 }
162
163 if (map.containsKey(OozieClient.CHANGE_VALUE_STATUS)) {
164 String value = map.get(OozieClient.CHANGE_VALUE_STATUS);
165 if (!StringUtils.isEmpty(value)) {
166 jobStatus = CoordinatorJob.Status.valueOf(value);
167 }
168 }
169 }
170
171 /**
172 * Check if new end time is valid.
173 *
174 * @param coordJob coordinator job id.
175 * @param newEndTime new end time.
176 * @throws CommandException thrown if new end time is not valid.
177 */
178 private void checkEndTime(CoordinatorJobBean coordJob, Date newEndTime) throws CommandException {
179 //It's ok to set end date before start date.
180 }
181
182 /**
183 * Check if new pause time is valid.
184 *
185 * @param coordJob coordinator job id.
186 * @param newPauseTime new pause time.
187 * @param newEndTime new end time, can be null meaning no change on end time.
188 * @throws CommandException thrown if new pause time is not valid.
189 */
190 private void checkPauseTime(CoordinatorJobBean coordJob, Date newPauseTime)
191 throws CommandException {
192 //no check.
193 }
194
195 /**
196 * Check if status change is valid.
197 *
198 * @param coordJob the coord job
199 * @param jobStatus the job status
200 * @throws CommandException the command exception
201 */
202 private void checkStatusChange(CoordinatorJobBean coordJob, CoordinatorJob.Status jobStatus)
203 throws CommandException {
204 if (!jobStatus.equals(CoordinatorJob.Status.RUNNING) && !jobStatus.equals(CoordinatorJob.Status.IGNORED)) {
205 throw new CommandException(ErrorCode.E1015, jobStatus, " must be RUNNING or IGNORED");
206 }
207
208 if (jobStatus.equals(CoordinatorJob.Status.RUNNING)) {
209 if (!(coordJob.getStatus().equals(CoordinatorJob.Status.FAILED) || coordJob.getStatus().equals(
210 CoordinatorJob.Status.KILLED) || coordJob.getStatus().equals(CoordinatorJob.Status.IGNORED))) {
211 throw new CommandException(ErrorCode.E1015, jobStatus,
212 " Only FAILED, KILLED, IGNORED job can be changed to RUNNING. Current job status is "
213 + coordJob.getStatus());
214 }
215 }
216 else {
217 if (!(coordJob.getStatus().equals(CoordinatorJob.Status.FAILED) || coordJob.getStatus().equals(
218 CoordinatorJob.Status.KILLED))
219 || coordJob.isPending()) {
220 throw new CommandException(ErrorCode.E1015, jobStatus,
221 " Only FAILED or KILLED non-pending job can be changed to IGNORED. Current job status is "
222 + coordJob.getStatus() + " and pending status is " + coordJob.isPending());
223 }
224 }
225 }
226
227 /**
228 * Process lookahead created actions that become invalid because of the new pause time,
229 * These actions will be deleted from DB, also the coordinator job will be updated accordingly
230 *
231 * @param coordJob coordinator job
232 * @param newPauseTime new pause time
233 * @throws JPAExecutorException, CommandException
234 */
235 private void processLookaheadActions(CoordinatorJobBean coordJob, Date newTime) throws CommandException,
236 JPAExecutorException {
237 int lastActionNumber = coordJob.getLastActionNumber();
238 Date lastActionTime = null;
239 Date tempDate = null;
240
241 while ((tempDate = deleteAction(lastActionNumber, newTime)) != null) {
242 lastActionNumber--;
243 lastActionTime = tempDate;
244 }
245 if (lastActionTime != null) {
246 LOG.debug("New pause/end date is : " + newTime + " and last action number is : " + lastActionNumber);
247 coordJob.setLastActionNumber(lastActionNumber);
248 coordJob.setLastActionTime(lastActionTime);
249 coordJob.setNextMaterializedTime(lastActionTime);
250 coordJob.resetDoneMaterialization();
251 }
252 }
253
254 /**
255 * Delete coordinator action
256 *
257 * @param actionNum coordinator action number
258 */
259 private Date deleteAction(int actionNum, Date afterDate) throws CommandException {
260 try {
261 if (actionNum <= 0) {
262 return null;
263 }
264
265 String actionId = jpaService.execute(new CoordJobGetActionByActionNumberJPAExecutor(jobId, actionNum));
266 CoordinatorActionBean bean = jpaService.execute(new CoordActionGetJPAExecutor(actionId));
267 if (afterDate.compareTo(bean.getNominalTime()) <= 0) {
268 // delete SLA registration entry (if any) for action
269 if (SLAService.isEnabled()) {
270 Services.get().get(SLAService.class).removeRegistration(actionId);
271 }
272 SLARegistrationBean slaReg = SLARegistrationQueryExecutor.getInstance().get(SLARegQuery.GET_SLA_REG_ALL, actionId);
273 if (slaReg != null) {
274 LOG.debug("Deleting registration bean corresponding to action " + slaReg.getId());
275 deleteList.add(slaReg);
276 }
277 SLASummaryBean slaSummaryBean = SLASummaryQueryExecutor.getInstance().get(
278 SLASummaryQuery.GET_SLA_SUMMARY, actionId);
279 if (slaSummaryBean != null) {
280 LOG.debug("Deleting summary bean corresponding to action " + slaSummaryBean.getId());
281 deleteList.add(slaSummaryBean);
282 }
283 if (bean.getStatus() == CoordinatorAction.Status.WAITING
284 || bean.getStatus() == CoordinatorAction.Status.READY) {
285 deleteList.add(bean);
286 }
287 else {
288 throw new CommandException(ErrorCode.E1022, bean.getId());
289 }
290 return bean.getNominalTime();
291 }
292 else {
293 return null;
294 }
295
296 }
297 catch (JPAExecutorException e) {
298 throw new CommandException(e);
299 }
300 }
301
302 /**
303 * Check if new end time, new concurrency, new pause time are valid.
304 *
305 * @param coordJob coordinator job id.
306 * @param newEndTime new end time.
307 * @param newConcurrency new concurrency.
308 * @param newPauseTime new pause time.
309 * @throws CommandException thrown if new values are not valid.
310 */
311 private void check(CoordinatorJobBean coordJob, Date newEndTime, Integer newConcurrency, Date newPauseTime,
312 CoordinatorJob.Status jobStatus) throws CommandException {
313
314 if (coordJob.getStatus() == CoordinatorJob.Status.KILLED
315 || coordJob.getStatus() == CoordinatorJob.Status.IGNORED) {
316 if (jobStatus == null || (newEndTime != null || newConcurrency != null || newPauseTime != null)) {
317 throw new CommandException(ErrorCode.E1016);
318 }
319 }
320
321 if (newEndTime != null) {
322 checkEndTime(coordJob, newEndTime);
323 }
324
325 if (newPauseTime != null) {
326 checkPauseTime(coordJob, newPauseTime);
327 }
328 if (jobStatus != null) {
329 checkStatusChange(coordJob, jobStatus);
330 }
331 }
332
333 /* (non-Javadoc)
334 * @see org.apache.oozie.command.XCommand#execute()
335 */
336 @Override
337 protected Void execute() throws CommandException {
338 LOG.info("STARTED CoordChangeXCommand for jobId=" + jobId);
339
340 try {
341 if (newEndTime != null) {
342 // during coord materialization, nextMaterializedTime is set to
343 // startTime + n(actions materialized) * frequency and this can be AFTER endTime,
344 // while doneMaterialization is true. Hence the following checks
345 // for newEndTime being in the middle of endTime and nextMatdTime.
346 // Since job is already done materialization so no need to change
347 boolean dontChange = coordJob.getEndTime().before(newEndTime)
348 && coordJob.getNextMaterializedTime() != null
349 && coordJob.getNextMaterializedTime().after(newEndTime);
350 if (!dontChange) {
351 coordJob.setEndTime(newEndTime);
352 // OOZIE-1703, we should SUCCEEDED the coord, if it's in PREP and new endtime is before start time
353 if (coordJob.getStartTime().compareTo(newEndTime) >= 0) {
354 if (coordJob.getStatus() != CoordinatorJob.Status.PREP) {
355 processLookaheadActions(coordJob, newEndTime);
356 }
357 if (coordJob.getStatus() == CoordinatorJob.Status.PREP
358 || coordJob.getStatus() == CoordinatorJob.Status.RUNNING) {
359 LOG.info("Changing coord status to SUCCEEDED, because it's in " + coordJob.getStatus()
360 + " and new end time is before start time. Startime is " + coordJob.getStartTime()
361 + " and new end time is " + newEndTime);
362
363 coordJob.setStatus(CoordinatorJob.Status.SUCCEEDED);
364 coordJob.resetPending();
365 }
366 coordJob.setDoneMaterialization();
367 }
368 else {
369 // move it to running iff new end time is after starttime.
370 if (coordJob.getStatus() == CoordinatorJob.Status.SUCCEEDED) {
371 coordJob.setStatus(CoordinatorJob.Status.RUNNING);
372 }
373 if (coordJob.getStatus() == CoordinatorJob.Status.DONEWITHERROR
374 || coordJob.getStatus() == CoordinatorJob.Status.FAILED) {
375 // Check for backward compatibility for Oozie versions (3.2 and before)
376 // when RUNNINGWITHERROR, SUSPENDEDWITHERROR and
377 // PAUSEDWITHERROR is not supported
378 coordJob.setStatus(StatusUtils
379 .getStatusIfBackwardSupportTrue(CoordinatorJob.Status.RUNNINGWITHERROR));
380 }
381 coordJob.setPending();
382 coordJob.resetDoneMaterialization();
383 processLookaheadActions(coordJob, newEndTime);
384 }
385 }
386
387 else {
388 LOG.info("Didn't change endtime. Endtime is in between coord end time and next materialization time."
389 + "Coord endTime = " + DateUtils.formatDateOozieTZ(newEndTime)
390 + " next materialization time ="
391 + DateUtils.formatDateOozieTZ(coordJob.getNextMaterializedTime()));
392 }
393 }
394
395 if (newConcurrency != null) {
396 this.coordJob.setConcurrency(newConcurrency);
397 }
398
399 if (newPauseTime != null || resetPauseTime == true) {
400 this.coordJob.setPauseTime(newPauseTime);
401 if (oldPauseTime != null && newPauseTime != null) {
402 if (oldPauseTime.before(newPauseTime)) {
403 if (this.coordJob.getStatus() == Job.Status.PAUSED) {
404 this.coordJob.setStatus(Job.Status.RUNNING);
405 }
406 else if (this.coordJob.getStatus() == Job.Status.PAUSEDWITHERROR) {
407 this.coordJob.setStatus(Job.Status.RUNNINGWITHERROR);
408 }
409 }
410 }
411 else if (oldPauseTime != null && newPauseTime == null) {
412 if (this.coordJob.getStatus() == Job.Status.PAUSED) {
413 this.coordJob.setStatus(Job.Status.RUNNING);
414 }
415 else if (this.coordJob.getStatus() == Job.Status.PAUSEDWITHERROR) {
416 this.coordJob.setStatus(Job.Status.RUNNINGWITHERROR);
417 }
418 }
419 if (!resetPauseTime) {
420 processLookaheadActions(coordJob, newPauseTime);
421 }
422 }
423 if (jobStatus != null) {
424 coordJob.setStatus(jobStatus);
425 LOG.info("Coord status is changed to " + jobStatus + " from " + prevStatus);
426 if (jobStatus.equals(CoordinatorJob.Status.RUNNING)) {
427 coordJob.setPending();
428 if (coordJob.getNextMaterializedTime() != null
429 && coordJob.getEndTime().after(coordJob.getNextMaterializedTime())) {
430 coordJob.resetDoneMaterialization();
431 }
432 } else if (jobStatus.equals(CoordinatorJob.Status.IGNORED)) {
433 coordJob.resetPending();
434 coordJob.setDoneMaterialization();
435 }
436 }
437
438 if (coordJob.getNextMaterializedTime() != null && coordJob.getEndTime().compareTo(coordJob.getNextMaterializedTime()) <= 0) {
439 LOG.info("[" + coordJob.getId() + "]: all actions have been materialized, job status = " + coordJob.getStatus()
440 + ", set pending to true");
441 // set doneMaterialization to true when materialization is done
442 coordJob.setDoneMaterialization();
443 }
444
445 coordJob.setLastModifiedTime(new Date());
446 updateList.add(new UpdateEntry<CoordJobQuery>(CoordJobQuery.UPDATE_COORD_JOB_CHANGE, coordJob));
447 BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(null, updateList, deleteList);
448
449 return null;
450 }
451 catch (XException ex) {
452 throw new CommandException(ex);
453 }
454 finally {
455 LOG.info("ENDED CoordChangeXCommand for jobId=" + jobId);
456 // update bundle action
457 if (coordJob.getBundleId() != null) {
458 //ignore pending as it'sync command
459 BundleStatusUpdateXCommand bundleStatusUpdate = new BundleStatusUpdateXCommand(coordJob, prevStatus, true);
460 bundleStatusUpdate.call();
461 }
462 }
463 }
464
465 /* (non-Javadoc)
466 * @see org.apache.oozie.command.XCommand#getEntityKey()
467 */
468 @Override
469 public String getEntityKey() {
470 return this.jobId;
471 }
472
473 /* (non-Javadoc)
474 * @see org.apache.oozie.command.XCommand#loadState()
475 */
476 @Override
477 protected void loadState() throws CommandException{
478 jpaService = Services.get().get(JPAService.class);
479
480 if (jpaService == null) {
481 throw new CommandException(ErrorCode.E0610);
482 }
483
484 try {
485 this.coordJob = jpaService.execute(new CoordJobGetJPAExecutor(jobId));
486 oldPauseTime = coordJob.getPauseTime();
487 prevStatus = coordJob.getStatus();
488 }
489 catch (JPAExecutorException e) {
490 throw new CommandException(e);
491 }
492
493 LogUtils.setLogInfo(this.coordJob, logInfo);
494 }
495
496 /* (non-Javadoc)
497 * @see org.apache.oozie.command.XCommand#verifyPrecondition()
498 */
499 @Override
500 protected void verifyPrecondition() throws CommandException,PreconditionException {
501 check(this.coordJob, newEndTime, newConcurrency, newPauseTime, jobStatus);
502 }
503
504 /* (non-Javadoc)
505 * @see org.apache.oozie.command.XCommand#isLockRequired()
506 */
507 @Override
508 protected boolean isLockRequired() {
509 return true;
510 }
511 }