001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.sla;
019
020 import java.sql.Timestamp;
021 import java.util.ArrayList;
022 import java.util.Collections;
023 import java.util.Date;
024 import java.util.HashSet;
025 import java.util.Iterator;
026 import java.util.List;
027 import java.util.Map;
028 import java.util.Set;
029 import java.util.concurrent.ConcurrentHashMap;
030
031 import org.apache.hadoop.conf.Configuration;
032 import org.apache.oozie.AppType;
033 import org.apache.oozie.CoordinatorActionBean;
034 import org.apache.oozie.CoordinatorJobBean;
035 import org.apache.oozie.ErrorCode;
036 import org.apache.oozie.WorkflowActionBean;
037 import org.apache.oozie.WorkflowJobBean;
038 import org.apache.oozie.XException;
039 import org.apache.oozie.client.CoordinatorAction;
040 import org.apache.oozie.client.WorkflowAction;
041 import org.apache.oozie.client.WorkflowJob;
042 import org.apache.oozie.client.event.JobEvent;
043 import org.apache.oozie.client.event.SLAEvent.EventStatus;
044 import org.apache.oozie.client.event.SLAEvent.SLAStatus;
045 import org.apache.oozie.client.rest.JsonBean;
046 import org.apache.oozie.executor.jpa.BatchQueryExecutor;
047 import org.apache.oozie.executor.jpa.CoordActionGetForSLAJPAExecutor;
048 import org.apache.oozie.executor.jpa.CoordActionQueryExecutor;
049 import org.apache.oozie.executor.jpa.CoordActionQueryExecutor.CoordActionQuery;
050 import org.apache.oozie.executor.jpa.CoordJobQueryExecutor;
051 import org.apache.oozie.executor.jpa.CoordJobQueryExecutor.CoordJobQuery;
052 import org.apache.oozie.executor.jpa.JPAExecutorException;
053 import org.apache.oozie.executor.jpa.SLARegistrationQueryExecutor;
054 import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor;
055 import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor;
056 import org.apache.oozie.executor.jpa.SLARegistrationQueryExecutor.SLARegQuery;
057 import org.apache.oozie.executor.jpa.SLASummaryQueryExecutor;
058 import org.apache.oozie.executor.jpa.WorkflowActionGetForSLAJPAExecutor;
059 import org.apache.oozie.executor.jpa.WorkflowJobGetForSLAJPAExecutor;
060 import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor.WorkflowActionQuery;
061 import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery;
062 import org.apache.oozie.executor.jpa.sla.SLASummaryGetRecordsOnRestartJPAExecutor;
063 import org.apache.oozie.executor.jpa.SLASummaryQueryExecutor.SLASummaryQuery;
064 import org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry;
065 import org.apache.oozie.lock.LockToken;
066 import org.apache.oozie.service.EventHandlerService;
067 import org.apache.oozie.service.JPAService;
068 import org.apache.oozie.service.JobsConcurrencyService;
069 import org.apache.oozie.service.MemoryLocksService;
070 import org.apache.oozie.service.SchedulerService;
071 import org.apache.oozie.service.ServiceException;
072 import org.apache.oozie.service.Services;
073 import org.apache.oozie.sla.service.SLAService;
074 import org.apache.oozie.util.DateUtils;
075 import org.apache.oozie.util.LogUtils;
076 import org.apache.oozie.util.XLog;
077
078 import com.google.common.annotations.VisibleForTesting;
079
080
081 /**
082 * Implementation class for SLACalculator that calculates SLA related to
083 * start/end/duration of jobs using a memory-based map
084 */
085 public class SLACalculatorMemory implements SLACalculator {
086
087 private static XLog LOG = XLog.getLog(SLACalculatorMemory.class);
088 // TODO optimization priority based insertion/processing/bumping up-down
089 protected Map<String, SLACalcStatus> slaMap;
090 protected Set<String> historySet;
091 private static int capacity;
092 private static JPAService jpaService;
093 protected EventHandlerService eventHandler;
094 private static int modifiedAfter;
095 private static long jobEventLatency;
096
097 @Override
098 public void init(Configuration conf) throws ServiceException {
099 capacity = conf.getInt(SLAService.CONF_CAPACITY, 5000);
100 jobEventLatency = conf.getInt(SLAService.CONF_JOB_EVENT_LATENCY, 90 * 1000);
101 slaMap = new ConcurrentHashMap<String, SLACalcStatus>();
102 historySet = Collections.synchronizedSet(new HashSet<String>());
103 jpaService = Services.get().get(JPAService.class);
104 eventHandler = Services.get().get(EventHandlerService.class);
105 // load events modified after
106 modifiedAfter = conf.getInt(SLAService.CONF_EVENTS_MODIFIED_AFTER, 7);
107 loadOnRestart();
108 Runnable purgeThread = new HistoryPurgeWorker();
109 // schedule runnable by default 1 day
110 Services.get()
111 .get(SchedulerService.class)
112 .schedule(purgeThread, 86400, Services.get().getConf().getInt(SLAService.CONF_SLA_HISTORY_PURGE_INTERVAL, 86400),
113 SchedulerService.Unit.SEC);
114 }
115
116 public class HistoryPurgeWorker implements Runnable {
117
118 public HistoryPurgeWorker() {
119 }
120
121 @Override
122 public void run() {
123 if (Thread.currentThread().isInterrupted()) {
124 return;
125 }
126 Iterator<String> jobItr = historySet.iterator();
127 while (jobItr.hasNext()) {
128 String jobId = jobItr.next();
129
130 if (jobId.endsWith("-W")) {
131 WorkflowJobBean wfJob = null;
132 try {
133 wfJob = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW_STATUS, jobId);
134 }
135 catch (JPAExecutorException e) {
136 if (e.getErrorCode().equals(ErrorCode.E0604)) {
137 jobItr.remove();
138 }
139 else {
140 LOG.info("Failed to fetch the workflow job: " + jobId, e);
141 }
142 }
143 if (wfJob != null && wfJob.inTerminalState()) {
144 try {
145 updateSLASummary(wfJob.getId(), wfJob.getStartTime(), wfJob.getEndTime());
146 jobItr.remove();
147 }
148 catch (JPAExecutorException e) {
149 LOG.info("Failed to update SLASummaryBean when purging history set entry for " + jobId, e);
150 }
151
152 }
153 }
154 else if (jobId.contains("-W@")) {
155 WorkflowActionBean wfAction = null;
156 try {
157 wfAction = WorkflowActionQueryExecutor.getInstance().get(
158 WorkflowActionQuery.GET_ACTION_COMPLETED, jobId);
159 }
160 catch (JPAExecutorException e) {
161 if (e.getErrorCode().equals(ErrorCode.E0605)) {
162 jobItr.remove();
163 }
164 else {
165 LOG.info("Failed to fetch the workflow action: " + jobId, e);
166 }
167 }
168 if (wfAction != null && (wfAction.isComplete() || wfAction.isTerminalWithFailure())) {
169 try {
170 updateSLASummary(wfAction.getId(), wfAction.getStartTime(), wfAction.getEndTime());
171 jobItr.remove();
172 }
173 catch (JPAExecutorException e) {
174 LOG.info("Failed to update SLASummaryBean when purging history set entry for " + jobId, e);
175 }
176 }
177 }
178 else if (jobId.contains("-C@")) {
179 CoordinatorActionBean cAction = null;
180 try {
181 cAction = CoordActionQueryExecutor.getInstance().get(CoordActionQuery.GET_COORD_ACTION, jobId);
182 }
183 catch (JPAExecutorException e) {
184 if (e.getErrorCode().equals(ErrorCode.E0605)) {
185 jobItr.remove();
186 }
187 else {
188 LOG.info("Failed to fetch the coord action: " + jobId, e);
189 }
190 }
191 if (cAction != null && cAction.isTerminalStatus()) {
192 try {
193 updateSLASummaryForCoordAction(cAction);
194 jobItr.remove();
195 }
196 catch (JPAExecutorException e) {
197 XLog.getLog(SLACalculatorMemory.class).info(
198 "Failed to update SLASummaryBean when purging history set entry for " + jobId, e);
199 }
200
201 }
202 }
203 else if (jobId.endsWith("-C")) {
204 CoordinatorJobBean cJob = null;
205 try {
206 cJob = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB_STATUS_PARENTID,
207 jobId);
208 }
209 catch (JPAExecutorException e) {
210 if (e.getErrorCode().equals(ErrorCode.E0604)) {
211 jobItr.remove();
212 }
213 else {
214 LOG.info("Failed to fetch the coord job: " + jobId, e);
215 }
216 }
217 if (cJob != null && cJob.isTerminalStatus()) {
218 try {
219 updateSLASummary(cJob.getId(), cJob.getStartTime(), cJob.getEndTime());
220 jobItr.remove();
221 }
222 catch (JPAExecutorException e) {
223 LOG.info("Failed to update SLASummaryBean when purging history set entry for " + jobId, e);
224 }
225
226 }
227 }
228 }
229 }
230
231 private void updateSLASummary(String id, Date startTime, Date endTime) throws JPAExecutorException {
232 SLASummaryBean sla = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, id);
233 if (sla != null) {
234 sla.setActualStart(startTime);
235 sla.setActualEnd(endTime);
236 if (startTime != null && endTime != null) {
237 sla.setActualDuration(endTime.getTime() - startTime.getTime());
238 }
239 sla.setLastModifiedTime(new Date());
240 sla.setEventProcessed(8);
241 SLASummaryQueryExecutor.getInstance().executeUpdate(
242 SLASummaryQuery.UPDATE_SLA_SUMMARY_FOR_ACTUAL_TIMES, sla);
243 }
244 }
245
246 private void updateSLASummaryForCoordAction(CoordinatorActionBean bean) throws JPAExecutorException {
247 String wrkflowId = bean.getExternalId();
248 if (wrkflowId != null) {
249 WorkflowJobBean wrkflow = WorkflowJobQueryExecutor.getInstance().get(
250 WorkflowJobQuery.GET_WORKFLOW_START_END_TIME, wrkflowId);
251 if (wrkflow != null) {
252 updateSLASummary(bean.getId(), wrkflow.getStartTime(), wrkflow.getEndTime());
253 }
254 }
255 }
256 }
257
258 private void loadOnRestart() {
259 boolean isJobModified = false;
260 try {
261 long slaPendingCount = 0;
262 long statusPendingCount = 0;
263 List<SLASummaryBean> summaryBeans = jpaService.execute(new SLASummaryGetRecordsOnRestartJPAExecutor(
264 modifiedAfter));
265 for (SLASummaryBean summaryBean : summaryBeans) {
266 String jobId = summaryBean.getId();
267 LockToken lock = null;
268 switch (summaryBean.getAppType()) {
269 case COORDINATOR_ACTION:
270 isJobModified = processSummaryBeanForCoordAction(summaryBean, jobId);
271 break;
272 case WORKFLOW_ACTION:
273 isJobModified = processSummaryBeanForWorkflowAction(summaryBean, jobId);
274 break;
275 case WORKFLOW_JOB:
276 isJobModified = processSummaryBeanForWorkflowJob(summaryBean, jobId);
277 break;
278 default:
279 break;
280 }
281 if (isJobModified) {
282 try {
283 boolean update = true;
284 if (Services.get().get(JobsConcurrencyService.class).isHighlyAvailableMode()) {
285 lock = Services
286 .get()
287 .get(MemoryLocksService.class)
288 .getWriteLock(
289 SLACalcStatus.SLA_ENTITYKEY_PREFIX + jobId,
290 Services.get().getConf()
291 .getLong(SLAService.CONF_SLA_CALC_LOCK_TIMEOUT, 5 * 1000));
292 if (lock == null) {
293 update = false;
294 }
295 }
296 if (update) {
297 summaryBean.setLastModifiedTime(new Date());
298 SLASummaryQueryExecutor.getInstance().executeUpdate(
299 SLASummaryQuery.UPDATE_SLA_SUMMARY_FOR_STATUS_ACTUAL_TIMES, summaryBean);
300 }
301 }
302 catch (Exception e) {
303 LOG.warn("Failed to load records for " + jobId, e);
304 }
305 finally {
306 if (lock != null) {
307 lock.release();
308 lock = null;
309 }
310 }
311 }
312 try {
313 if (summaryBean.getEventProcessed() == 7) {
314 historySet.add(jobId);
315 statusPendingCount++;
316 }
317 else if (summaryBean.getEventProcessed() <= 7) {
318 SLARegistrationBean slaRegBean = SLARegistrationQueryExecutor.getInstance().get(
319 SLARegQuery.GET_SLA_REG_ON_RESTART, jobId);
320 SLACalcStatus slaCalcStatus = new SLACalcStatus(summaryBean, slaRegBean);
321 slaMap.put(jobId, slaCalcStatus);
322 slaPendingCount++;
323 }
324 }
325 catch (Exception e) {
326 LOG.warn("Failed to fetch/update records for " + jobId, e);
327 }
328
329 }
330 LOG.info("Loaded SLASummary pendingSLA=" + slaPendingCount + ", pendingStatusUpdate=" + statusPendingCount);
331
332 }
333 catch (Exception e) {
334 LOG.warn("Failed to retrieve SLASummary records on restart", e);
335 }
336 }
337
338 private boolean processSummaryBeanForCoordAction(SLASummaryBean summaryBean, String jobId)
339 throws JPAExecutorException {
340 boolean isJobModified = false;
341 CoordinatorActionBean coordAction = null;
342 coordAction = jpaService.execute(new CoordActionGetForSLAJPAExecutor(jobId));
343 if (!coordAction.getStatusStr().equals(summaryBean.getJobStatus())) {
344 LOG.trace("Coordinator action status is " + coordAction.getStatusStr() + " and summary bean status is "
345 + summaryBean.getJobStatus());
346 isJobModified = true;
347 summaryBean.setJobStatus(coordAction.getStatusStr());
348 if (coordAction.isTerminalStatus()) {
349 WorkflowJobBean wfJob = jpaService.execute(new WorkflowJobGetForSLAJPAExecutor(coordAction
350 .getExternalId()));
351 setEndForSLASummaryBean(summaryBean, wfJob.getStartTime(), coordAction.getLastModifiedTime(),
352 coordAction.getStatusStr());
353 }
354 else if (coordAction.getStatus() != CoordinatorAction.Status.WAITING) {
355 WorkflowJobBean wfJob = jpaService.execute(new WorkflowJobGetForSLAJPAExecutor(coordAction
356 .getExternalId()));
357 setStartForSLASummaryBean(summaryBean, summaryBean.getEventProcessed(), wfJob.getStartTime());
358 }
359 }
360 return isJobModified;
361 }
362
363 private boolean processSummaryBeanForWorkflowAction(SLASummaryBean summaryBean, String jobId)
364 throws JPAExecutorException {
365 boolean isJobModified = false;
366 WorkflowActionBean wfAction = null;
367 wfAction = jpaService.execute(new WorkflowActionGetForSLAJPAExecutor(jobId));
368 if (!wfAction.getStatusStr().equals(summaryBean.getJobStatus())) {
369 LOG.trace("Workflow action status is " + wfAction.getStatusStr() + "and summary bean status is "
370 + summaryBean.getJobStatus());
371 isJobModified = true;
372 summaryBean.setJobStatus(wfAction.getStatusStr());
373 if (wfAction.inTerminalState()) {
374 setEndForSLASummaryBean(summaryBean, wfAction.getStartTime(), wfAction.getEndTime(), wfAction.getStatusStr());
375 }
376 else if (wfAction.getStatus() != WorkflowAction.Status.PREP) {
377 setStartForSLASummaryBean(summaryBean, summaryBean.getEventProcessed(), wfAction.getStartTime());
378 }
379 }
380 return isJobModified;
381 }
382
383 private boolean processSummaryBeanForWorkflowJob(SLASummaryBean summaryBean, String jobId)
384 throws JPAExecutorException {
385 boolean isJobModified = false;
386 WorkflowJobBean wfJob = null;
387 wfJob = jpaService.execute(new WorkflowJobGetForSLAJPAExecutor(jobId));
388 if (!wfJob.getStatusStr().equals(summaryBean.getJobStatus())) {
389 LOG.trace("Workflow job status is " + wfJob.getStatusStr() + "and summary bean status is "
390 + summaryBean.getJobStatus());
391 isJobModified = true;
392 summaryBean.setJobStatus(wfJob.getStatusStr());
393 if (wfJob.inTerminalState()) {
394 setEndForSLASummaryBean(summaryBean, wfJob.getStartTime(), wfJob.getEndTime(), wfJob.getStatusStr());
395 }
396 else if (wfJob.getStatus() != WorkflowJob.Status.PREP) {
397 setStartForSLASummaryBean(summaryBean, summaryBean.getEventProcessed(), wfJob.getStartTime());
398 }
399 }
400 return isJobModified;
401 }
402
403 private void setEndForSLASummaryBean(SLASummaryBean summaryBean, Date startTime, Date endTime, String status) {
404 byte eventProc = summaryBean.getEventProcessed();
405 summaryBean.setEventProcessed(8);
406 summaryBean.setActualStart(startTime);
407 summaryBean.setActualEnd(endTime);
408 long actualDuration = endTime.getTime() - startTime.getTime();
409 summaryBean.setActualDuration(actualDuration);
410 if (eventProc < 4) {
411 if (status.equals(WorkflowJob.Status.SUCCEEDED.name()) || status.equals(WorkflowAction.Status.OK.name())
412 || status.equals(CoordinatorAction.Status.SUCCEEDED.name())) {
413 if (endTime.getTime() <= summaryBean.getExpectedEnd().getTime()) {
414 summaryBean.setSLAStatus(SLAStatus.MET);
415 }
416 else {
417 summaryBean.setSLAStatus(SLAStatus.MISS);
418 }
419 }
420 else {
421 summaryBean.setSLAStatus(SLAStatus.MISS);
422 }
423 }
424
425 }
426
427 private void setStartForSLASummaryBean(SLASummaryBean summaryBean, byte eventProc, Date startTime) {
428 if (((eventProc & 1) == 0)) {
429 eventProc += 1;
430 summaryBean.setEventProcessed(eventProc);
431 }
432 if (summaryBean.getSLAStatus().equals(SLAStatus.NOT_STARTED)) {
433 summaryBean.setSLAStatus(SLAStatus.IN_PROCESS);
434 }
435 summaryBean.setActualStart(startTime);
436 }
437
438 @Override
439 public int size() {
440 return slaMap.size();
441 }
442
443 @Override
444 public SLACalcStatus get(String jobId) throws JPAExecutorException {
445 SLACalcStatus memObj;
446 memObj = slaMap.get(jobId);
447 if (memObj == null && historySet.contains(jobId)) {
448 memObj = new SLACalcStatus(SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId),
449 SLARegistrationQueryExecutor.getInstance().get(SLARegQuery.GET_SLA_REG_ON_RESTART, jobId));
450 }
451 return memObj;
452 }
453
454 @Override
455 public Iterator<String> iterator() {
456 return slaMap.keySet().iterator();
457 }
458
459 @Override
460 public boolean isEmpty() {
461 return slaMap.isEmpty();
462 }
463
464 @Override
465 public void clear() {
466 slaMap.clear();
467 historySet.clear();
468 }
469
470 /**
471 * Invoked via periodic run, update the SLA for registered jobs
472 */
473 protected void updateJobSla(String jobId) throws Exception {
474 SLACalcStatus slaCalc = slaMap.get(jobId);
475 synchronized (slaCalc) {
476 boolean change = false;
477 // get eventProcessed on DB for validation in HA
478 Object eventProcObj = ((SLASummaryQueryExecutor) SLASummaryQueryExecutor.getInstance()).getSingleValue(
479 SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED, jobId);
480 byte eventProc = ((Byte) eventProcObj).byteValue();
481 if (eventProc >= 7) {
482 if (eventProc == 7) {
483 historySet.add(jobId);
484 }
485 slaMap.remove(jobId);
486 LOG.trace("Removed Job [{0}] from map as SLA processed", jobId);
487 }
488 else {
489 slaCalc.setEventProcessed(eventProc);
490 SLARegistrationBean reg = slaCalc.getSLARegistrationBean();
491 // calculation w.r.t current time and status
492 if ((eventProc & 1) == 0) { // first bit (start-processed) unset
493 if (reg.getExpectedStart() != null) {
494 if (reg.getExpectedStart().getTime() + jobEventLatency < System.currentTimeMillis()) {
495 confirmWithDB(slaCalc);
496 eventProc = slaCalc.getEventProcessed();
497 if (eventProc != 8 && (eventProc & 1) == 0) {
498 // Some DB exception
499 slaCalc.setEventStatus(EventStatus.START_MISS);
500 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
501 eventProc++;
502 }
503 change = true;
504 }
505 }
506 else {
507 eventProc++; // disable further processing for optional start sla condition
508 change = true;
509 }
510 }
511 // check if second bit (duration-processed) is unset
512 if (eventProc != 8 && ((eventProc >> 1) & 1) == 0) {
513 if (reg.getExpectedDuration() == -1) {
514 eventProc += 2;
515 change = true;
516 }
517 else if (slaCalc.getActualStart() != null) {
518 if ((reg.getExpectedDuration() + jobEventLatency) < (System.currentTimeMillis() - slaCalc
519 .getActualStart().getTime())) {
520 slaCalc.setEventProcessed(eventProc);
521 confirmWithDB(slaCalc);
522 eventProc = slaCalc.getEventProcessed();
523 if (eventProc != 8 && ((eventProc >> 1) & 1) == 0) {
524 // Some DB exception
525 slaCalc.setEventStatus(EventStatus.DURATION_MISS);
526 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
527 eventProc += 2;
528 }
529 change = true;
530 }
531 }
532 }
533 if (eventProc < 4) {
534 if (reg.getExpectedEnd().getTime() + jobEventLatency < System.currentTimeMillis()) {
535 slaCalc.setEventProcessed(eventProc);
536 confirmWithDB(slaCalc);
537 eventProc = slaCalc.getEventProcessed();
538 change = true;
539 }
540 }
541 if (change) {
542 try {
543 boolean locked = true;
544 slaCalc.acquireLock();
545 locked = slaCalc.isLocked();
546 if (locked) {
547 // no more processing, no transfer to history set
548 if (slaCalc.getEventProcessed() >= 8) {
549 eventProc = 8;
550 // Should not be > 8. But to handle any corner cases
551 slaCalc.setEventProcessed(8);
552 slaMap.remove(jobId);
553 }
554 else {
555 slaCalc.setEventProcessed(eventProc);
556 }
557 SLASummaryBean slaSummaryBean = new SLASummaryBean();
558 slaSummaryBean.setId(slaCalc.getId());
559 slaSummaryBean.setEventProcessed(eventProc);
560 slaSummaryBean.setSLAStatus(slaCalc.getSLAStatus());
561 slaSummaryBean.setEventStatus(slaCalc.getEventStatus());
562 slaSummaryBean.setActualEnd(slaCalc.getActualEnd());
563 slaSummaryBean.setActualStart(slaCalc.getActualStart());
564 slaSummaryBean.setActualDuration(slaCalc.getActualDuration());
565 slaSummaryBean.setJobStatus(slaCalc.getJobStatus());
566 slaSummaryBean.setLastModifiedTime(new Date());
567 SLASummaryQueryExecutor.getInstance().executeUpdate(
568 SLASummaryQuery.UPDATE_SLA_SUMMARY_FOR_STATUS_ACTUAL_TIMES, slaSummaryBean);
569 if (eventProc == 7) {
570 historySet.add(jobId);
571 slaMap.remove(jobId);
572 LOG.trace("Removed Job [{0}] from map after End-processed", jobId);
573 }
574 }
575 }
576 catch (InterruptedException e) {
577 throw new XException(ErrorCode.E0606, slaCalc.getId(), slaCalc.getLockTimeOut());
578 }
579 finally {
580 slaCalc.releaseLock();
581 }
582 }
583 }
584 }
585 }
586
587 /**
588 * Periodically run by the SLAService worker threads to update SLA status by
589 * iterating through all the jobs in the map
590 */
591 @Override
592 public void updateAllSlaStatus() {
593 LOG.info("Running periodic SLA check");
594 Iterator<String> iterator = slaMap.keySet().iterator();
595 while (iterator.hasNext()) {
596 String jobId = iterator.next();
597 try {
598 LOG.trace("Processing SLA for jobid={0}", jobId);
599 updateJobSla(jobId);
600 }
601 catch (Exception e) {
602 setLogPrefix(jobId);
603 LOG.error("Exception in SLA processing for job [{0}]", jobId, e);
604 LogUtils.clearLogPrefix();
605 }
606 }
607 }
608
609 /**
610 * Register a new job into the map for SLA tracking
611 */
612 @Override
613 public boolean addRegistration(String jobId, SLARegistrationBean reg) throws JPAExecutorException {
614 try {
615 if (slaMap.size() < capacity) {
616 SLACalcStatus slaCalc = new SLACalcStatus(reg);
617 slaCalc.setSLAStatus(SLAStatus.NOT_STARTED);
618 slaCalc.setJobStatus(getJobStatus(reg.getAppType()));
619 slaMap.put(jobId, slaCalc);
620 List<JsonBean> insertList = new ArrayList<JsonBean>();
621 final SLASummaryBean summaryBean = new SLASummaryBean(slaCalc);
622 final Timestamp currentTime = DateUtils.convertDateToTimestamp(new Date());
623 reg.setCreatedTimestamp(currentTime);
624 summaryBean.setCreatedTimestamp(currentTime);
625 insertList.add(reg);
626 insertList.add(summaryBean);
627 BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, null, null);
628 LOG.trace("SLA Registration Event - Job:" + jobId);
629 return true;
630 }
631 else {
632 setLogPrefix(reg.getId());
633 LOG.error(
634 "SLACalculator memory capacity reached. Cannot add or update new SLA Registration entry for job [{0}]",
635 reg.getId());
636 LogUtils.clearLogPrefix();
637 }
638 }
639 catch (JPAExecutorException jpa) {
640 throw jpa;
641 }
642 return false;
643 }
644
645 private String getJobStatus(AppType appType) {
646 String status = null;
647 switch (appType) {
648 case COORDINATOR_ACTION:
649 status = CoordinatorAction.Status.WAITING.name();
650 break;
651 case WORKFLOW_ACTION:
652 status = WorkflowAction.Status.PREP.name();
653 break;
654 case WORKFLOW_JOB:
655 status = WorkflowJob.Status.PREP.name();
656 break;
657 default:
658 break;
659 }
660 return status;
661 }
662
663 /**
664 * Update job into the map for SLA tracking
665 */
666 @Override
667 public boolean updateRegistration(String jobId, SLARegistrationBean reg) throws JPAExecutorException {
668 try {
669 if (slaMap.size() < capacity) {
670 SLACalcStatus slaCalc = new SLACalcStatus(reg);
671 slaCalc.setSLAStatus(SLAStatus.NOT_STARTED);
672 slaCalc.setJobStatus(getJobStatus(reg.getAppType()));
673 slaMap.put(jobId, slaCalc);
674 List<UpdateEntry> updateList = new ArrayList<UpdateEntry>();
675 updateList.add(new UpdateEntry<SLARegQuery>(SLARegQuery.UPDATE_SLA_REG_ALL, reg));
676 updateList.add(new UpdateEntry<SLASummaryQuery>(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL,
677 new SLASummaryBean(slaCalc)));
678 BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(null, updateList, null);
679 LOG.trace("SLA Registration Event - Job:" + jobId);
680 return true;
681 }
682 else {
683 setLogPrefix(reg.getId());
684 LOG.error(
685 "SLACalculator memory capacity reached. Cannot add or update new SLA Registration entry for job [{0}]",
686 reg.getId());
687 LogUtils.clearLogPrefix();
688 }
689 }
690 catch (JPAExecutorException jpa) {
691 throw jpa;
692 }
693 return false;
694 }
695
696 /**
697 * Remove job from being tracked in map
698 */
699 @Override
700 public void removeRegistration(String jobId) {
701 if (slaMap.remove(jobId) == null) {
702 historySet.remove(jobId);
703 }
704 }
705
706 /**
707 * Triggered after receiving Job status change event, update SLA status
708 * accordingly
709 */
710 @Override
711 public boolean addJobStatus(String jobId, String jobStatus, JobEvent.EventStatus jobEventStatus, Date startTime,
712 Date endTime) throws JPAExecutorException, ServiceException {
713 SLACalcStatus slaCalc = slaMap.get(jobId);
714 SLASummaryBean slaInfo = null;
715 boolean hasSla = false;
716 if (slaCalc == null) {
717 if (historySet.contains(jobId)) {
718 slaInfo = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
719 if (slaInfo == null) {
720 throw new JPAExecutorException(ErrorCode.E0604, jobId);
721 }
722 slaInfo.setJobStatus(jobStatus);
723 slaInfo.setActualStart(startTime);
724 slaInfo.setActualEnd(endTime);
725 if (endTime != null) {
726 slaInfo.setActualDuration(endTime.getTime() - startTime.getTime());
727 }
728 slaInfo.setEventProcessed(8);
729 historySet.remove(jobId);
730 slaInfo.setLastModifiedTime(new Date());
731 SLASummaryQueryExecutor.getInstance().executeUpdate(
732 SLASummaryQuery.UPDATE_SLA_SUMMARY_FOR_STATUS_ACTUAL_TIMES, slaInfo);
733 hasSla = true;
734 }
735 else if (Services.get().get(JobsConcurrencyService.class).isHighlyAvailableMode()) {
736 // jobid might not exist in slaMap in HA Setting
737 SLARegistrationBean slaRegBean = SLARegistrationQueryExecutor.getInstance().get(
738 SLARegQuery.GET_SLA_REG_ALL, jobId);
739 if (slaRegBean != null) { // filter out jobs picked by SLA job event listener
740 // but not actually configured for SLA
741 SLASummaryBean slaSummaryBean = SLASummaryQueryExecutor.getInstance().get(
742 SLASummaryQuery.GET_SLA_SUMMARY, jobId);
743 if (slaSummaryBean.getEventProcessed() < 7) {
744 slaCalc = new SLACalcStatus(slaSummaryBean, slaRegBean);
745 slaMap.put(jobId, slaCalc);
746 }
747 }
748 }
749 }
750 if (slaCalc != null) {
751 synchronized (slaCalc) {
752 try {
753 // only get ZK lock when multiple servers running
754 boolean locked = true;
755 slaCalc.acquireLock();
756 locked = slaCalc.isLocked();
757 if (locked) {
758 // get eventProcessed on DB for validation in HA
759 Object eventProcObj = ((SLASummaryQueryExecutor) SLASummaryQueryExecutor.getInstance())
760 .getSingleValue(SLASummaryQuery.GET_SLA_SUMMARY_EVENTPROCESSED, jobId);
761 byte eventProc = ((Byte) eventProcObj).byteValue();
762 slaCalc.setEventProcessed(eventProc);
763 slaCalc.setJobStatus(jobStatus);
764 switch (jobEventStatus) {
765 case STARTED:
766 slaInfo = processJobStartSLA(slaCalc, startTime);
767 break;
768 case SUCCESS:
769 slaInfo = processJobEndSuccessSLA(slaCalc, startTime, endTime);
770 break;
771 case FAILURE:
772 slaInfo = processJobEndFailureSLA(slaCalc, startTime, endTime);
773 break;
774 default:
775 LOG.debug("Unknown Job Status for SLA purpose[{0}]", jobEventStatus);
776 slaInfo = getSLASummaryBean(slaCalc);
777 }
778 if (slaCalc.getEventProcessed() == 7) {
779 slaInfo.setEventProcessed(8);
780 slaMap.remove(jobId);
781 }
782 slaInfo.setLastModifiedTime(new Date());
783 SLASummaryQueryExecutor.getInstance().executeUpdate(
784 SLASummaryQuery.UPDATE_SLA_SUMMARY_FOR_STATUS_ACTUAL_TIMES, slaInfo);
785 hasSla = true;
786 }
787 }
788 catch (InterruptedException e) {
789 throw new ServiceException(ErrorCode.E0606, slaCalc.getEntityKey(), slaCalc.getLockTimeOut());
790 }
791 finally {
792 slaCalc.releaseLock();
793 }
794 }
795 LOG.trace("SLA Status Event - Job:" + jobId + " Status:" + slaCalc.getSLAStatus());
796 }
797
798 return hasSla;
799 }
800
801 /**
802 * Process SLA for jobs that started running. Also update actual-start time
803 *
804 * @param slaCalc
805 * @param actualStart
806 * @return SLASummaryBean
807 */
808 private SLASummaryBean processJobStartSLA(SLACalcStatus slaCalc, Date actualStart) {
809 slaCalc.setActualStart(actualStart);
810 if (slaCalc.getSLAStatus().equals(SLAStatus.NOT_STARTED)) {
811 slaCalc.setSLAStatus(SLAStatus.IN_PROCESS);
812 }
813 SLARegistrationBean reg = slaCalc.getSLARegistrationBean();
814 Date expecStart = reg.getExpectedStart();
815 byte eventProc = slaCalc.getEventProcessed();
816 // set event proc here
817 if (((eventProc & 1) == 0)) {
818 if (expecStart != null) {
819 if (actualStart.getTime() > expecStart.getTime()) {
820 slaCalc.setEventStatus(EventStatus.START_MISS);
821 }
822 else {
823 slaCalc.setEventStatus(EventStatus.START_MET);
824 }
825 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
826 }
827 eventProc += 1;
828 slaCalc.setEventProcessed(eventProc);
829 }
830 return getSLASummaryBean(slaCalc);
831 }
832
833 /**
834 * Process SLA for jobs that ended successfully. Also update actual-start
835 * and end time
836 *
837 * @param slaCalc
838 * @param actualStart
839 * @param actualEnd
840 * @return SLASummaryBean
841 * @throws JPAExecutorException
842 */
843 private SLASummaryBean processJobEndSuccessSLA(SLACalcStatus slaCalc, Date actualStart, Date actualEnd) throws JPAExecutorException {
844 SLARegistrationBean reg = slaCalc.getSLARegistrationBean();
845 slaCalc.setActualStart(actualStart);
846 slaCalc.setActualEnd(actualEnd);
847 long expectedDuration = reg.getExpectedDuration();
848 long actualDuration = actualEnd.getTime() - actualStart.getTime();
849 slaCalc.setActualDuration(actualDuration);
850 //check event proc
851 byte eventProc = slaCalc.getEventProcessed();
852 if (((eventProc >> 1) & 1) == 0) {
853 processDurationSLA(expectedDuration, actualDuration, slaCalc);
854 eventProc += 2;
855 slaCalc.setEventProcessed(eventProc);
856 }
857
858 if (eventProc < 4) {
859 Date expectedEnd = reg.getExpectedEnd();
860 if (actualEnd.getTime() > expectedEnd.getTime()) {
861 slaCalc.setEventStatus(EventStatus.END_MISS);
862 slaCalc.setSLAStatus(SLAStatus.MISS);
863 }
864 else {
865 slaCalc.setEventStatus(EventStatus.END_MET);
866 slaCalc.setSLAStatus(SLAStatus.MET);
867 }
868 eventProc += 4;
869 slaCalc.setEventProcessed(eventProc);
870 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
871 }
872 return getSLASummaryBean(slaCalc);
873 }
874
875 /**
876 * Process SLA for jobs that ended in failure. Also update actual-start and
877 * end time
878 *
879 * @param slaCalc
880 * @param actualStart
881 * @param actualEnd
882 * @return SLASummaryBean
883 * @throws JPAExecutorException
884 */
885 private SLASummaryBean processJobEndFailureSLA(SLACalcStatus slaCalc, Date actualStart, Date actualEnd) throws JPAExecutorException {
886 slaCalc.setActualStart(actualStart);
887 slaCalc.setActualEnd(actualEnd);
888 if (actualStart == null) { // job failed before starting
889 if (slaCalc.getEventProcessed() < 4) {
890 slaCalc.setEventStatus(EventStatus.END_MISS);
891 slaCalc.setSLAStatus(SLAStatus.MISS);
892 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
893 slaCalc.setEventProcessed(7);
894 return getSLASummaryBean(slaCalc);
895 }
896 }
897 SLARegistrationBean reg = slaCalc.getSLARegistrationBean();
898 long expectedDuration = reg.getExpectedDuration();
899 long actualDuration = actualEnd.getTime() - actualStart.getTime();
900 slaCalc.setActualDuration(actualDuration);
901
902 byte eventProc = slaCalc.getEventProcessed();
903 if (((eventProc >> 1) & 1) == 0) {
904 if (expectedDuration != -1) {
905 slaCalc.setEventStatus(EventStatus.DURATION_MISS);
906 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
907 }
908 eventProc += 2;
909 slaCalc.setEventProcessed(eventProc);
910 }
911 if (eventProc < 4) {
912 slaCalc.setEventStatus(EventStatus.END_MISS);
913 slaCalc.setSLAStatus(SLAStatus.MISS);
914 eventProc += 4;
915 slaCalc.setEventProcessed(eventProc);
916 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
917 }
918 return getSLASummaryBean(slaCalc);
919 }
920
921 private SLASummaryBean getSLASummaryBean (SLACalcStatus slaCalc) {
922 SLASummaryBean slaSummaryBean = new SLASummaryBean();
923 slaSummaryBean.setActualStart(slaCalc.getActualStart());
924 slaSummaryBean.setActualEnd(slaCalc.getActualEnd());
925 slaSummaryBean.setActualDuration(slaCalc.getActualDuration());
926 slaSummaryBean.setSLAStatus(slaCalc.getSLAStatus());
927 slaSummaryBean.setEventStatus(slaCalc.getEventStatus());
928 slaSummaryBean.setEventProcessed(slaCalc.getEventProcessed());
929 slaSummaryBean.setId(slaCalc.getId());
930 slaSummaryBean.setJobStatus(slaCalc.getJobStatus());
931 return slaSummaryBean;
932 }
933
934 private void processDurationSLA(long expected, long actual, SLACalcStatus slaCalc) {
935 if (expected != -1 && actual > expected) {
936 slaCalc.setEventStatus(EventStatus.DURATION_MISS);
937 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
938 }
939 else if (expected != -1 && actual <= expected) {
940 slaCalc.setEventStatus(EventStatus.DURATION_MET);
941 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
942 }
943 }
944
945 /*
946 * Confirm alerts against source of truth - DB. Also required in case of High Availability
947 */
948 private void confirmWithDB(SLACalcStatus slaCalc) {
949 boolean ended = false, isEndMiss = false;
950 try {
951 switch (slaCalc.getAppType()) {
952 case WORKFLOW_JOB:
953 WorkflowJobBean wf = jpaService.execute(new WorkflowJobGetForSLAJPAExecutor(slaCalc.getId()));
954 if (wf.getEndTime() != null) {
955 ended = true;
956 if (wf.getStatus() == WorkflowJob.Status.KILLED || wf.getStatus() == WorkflowJob.Status.FAILED
957 || wf.getEndTime().getTime() > slaCalc.getExpectedEnd().getTime()) {
958 isEndMiss = true;
959 }
960 }
961 slaCalc.setActualStart(wf.getStartTime());
962 slaCalc.setActualEnd(wf.getEndTime());
963 slaCalc.setJobStatus(wf.getStatusStr());
964 break;
965 case WORKFLOW_ACTION:
966 WorkflowActionBean wa = jpaService.execute(new WorkflowActionGetForSLAJPAExecutor(slaCalc.getId()));
967 if (wa.getEndTime() != null) {
968 ended = true;
969 if (wa.isTerminalWithFailure()
970 || wa.getEndTime().getTime() > slaCalc.getExpectedEnd().getTime()) {
971 isEndMiss = true;
972 }
973 }
974 slaCalc.setActualStart(wa.getStartTime());
975 slaCalc.setActualEnd(wa.getEndTime());
976 slaCalc.setJobStatus(wa.getStatusStr());
977 break;
978 case COORDINATOR_ACTION:
979 CoordinatorActionBean ca = jpaService.execute(new CoordActionGetForSLAJPAExecutor(slaCalc.getId()));
980 if (ca.isTerminalWithFailure()) {
981 isEndMiss = ended = true;
982 slaCalc.setActualEnd(ca.getLastModifiedTime());
983 }
984 if (ca.getExternalId() != null) {
985 wf = jpaService.execute(new WorkflowJobGetForSLAJPAExecutor(ca.getExternalId()));
986 if (wf.getEndTime() != null) {
987 ended = true;
988 if (wf.getEndTime().getTime() > slaCalc.getExpectedEnd().getTime()) {
989 isEndMiss = true;
990 }
991 }
992 slaCalc.setActualEnd(wf.getEndTime());
993 slaCalc.setActualStart(wf.getStartTime());
994 }
995 slaCalc.setJobStatus(ca.getStatusStr());
996 break;
997 default:
998 LOG.debug("Unsupported App-type for SLA - " + slaCalc.getAppType());
999 }
1000
1001 byte eventProc = slaCalc.getEventProcessed();
1002 if (ended) {
1003 if (isEndMiss) {
1004 slaCalc.setSLAStatus(SLAStatus.MISS);
1005 }
1006 else {
1007 slaCalc.setSLAStatus(SLAStatus.MET);
1008 }
1009 if (slaCalc.getActualStart() != null) {
1010 if ((eventProc & 1) == 0) {
1011 if (slaCalc.getExpectedStart().getTime() < slaCalc.getActualStart().getTime()) {
1012 slaCalc.setEventStatus(EventStatus.START_MISS);
1013 }
1014 else {
1015 slaCalc.setEventStatus(EventStatus.START_MET);
1016 }
1017 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
1018 }
1019 slaCalc.setActualDuration(slaCalc.getActualEnd().getTime() - slaCalc.getActualStart().getTime());
1020 if (((eventProc >> 1) & 1) == 0) {
1021 processDurationSLA(slaCalc.getExpectedDuration(), slaCalc.getActualDuration(), slaCalc);
1022 }
1023 }
1024 if (eventProc < 4) {
1025 if (isEndMiss) {
1026 slaCalc.setEventStatus(EventStatus.END_MISS);
1027 }
1028 else {
1029 slaCalc.setEventStatus(EventStatus.END_MET);
1030 }
1031 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
1032 }
1033 slaCalc.setEventProcessed(8);
1034 }
1035 else {
1036 if (slaCalc.getActualStart() != null) {
1037 slaCalc.setSLAStatus(SLAStatus.IN_PROCESS);
1038 }
1039 if ((eventProc & 1) == 0) {
1040 if (slaCalc.getActualStart() != null) {
1041 if (slaCalc.getExpectedStart().getTime() < slaCalc.getActualStart().getTime()) {
1042 slaCalc.setEventStatus(EventStatus.START_MISS);
1043 }
1044 else {
1045 slaCalc.setEventStatus(EventStatus.START_MET);
1046 }
1047 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
1048 eventProc++;
1049 }
1050 else if (slaCalc.getExpectedStart().getTime() < System.currentTimeMillis()) {
1051 slaCalc.setEventStatus(EventStatus.START_MISS);
1052 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
1053 eventProc++;
1054 }
1055 }
1056 if (((eventProc >> 1) & 1) == 0 && slaCalc.getActualStart() != null
1057 && slaCalc.getExpectedDuration() != -1) {
1058 if (System.currentTimeMillis() - slaCalc.getActualStart().getTime() > slaCalc.getExpectedDuration()) {
1059 slaCalc.setEventStatus(EventStatus.DURATION_MISS);
1060 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
1061 eventProc += 2;
1062 }
1063 }
1064 if (eventProc < 4 && slaCalc.getExpectedEnd().getTime() < System.currentTimeMillis()) {
1065 slaCalc.setEventStatus(EventStatus.END_MISS);
1066 slaCalc.setSLAStatus(SLAStatus.MISS);
1067 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
1068 eventProc += 4;
1069 }
1070 slaCalc.setEventProcessed(eventProc);
1071 }
1072 }
1073 catch (Exception e) {
1074 LOG.warn("Error while confirming SLA against DB for jobid= " + slaCalc.getId() + ". Exception is "
1075 + e.getClass().getName() + ": " + e.getMessage());
1076 if (slaCalc.getEventProcessed() < 4 && slaCalc.getExpectedEnd().getTime() < System.currentTimeMillis()) {
1077 slaCalc.setEventStatus(EventStatus.END_MISS);
1078 slaCalc.setSLAStatus(SLAStatus.MISS);
1079 eventHandler.queueEvent(new SLACalcStatus(slaCalc));
1080 slaCalc.setEventProcessed(slaCalc.getEventProcessed() + 4);
1081 }
1082 }
1083 }
1084
1085 @VisibleForTesting
1086 public boolean isJobIdInSLAMap(String jobId) {
1087 return this.slaMap.containsKey(jobId);
1088 }
1089
1090 @VisibleForTesting
1091 public boolean isJobIdInHistorySet(String jobId) {
1092 return this.historySet.contains(jobId);
1093 }
1094
1095 private void setLogPrefix(String jobId) {
1096 LOG = LogUtils.setLogInfo(LOG, jobId, null, null);
1097 }
1098 }