001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.command;
019
020 import java.util.ArrayList;
021 import java.util.Collections;
022 import java.util.Iterator;
023 import java.util.List;
024
025 import org.apache.oozie.WorkflowJobBean;
026 import org.apache.oozie.ErrorCode;
027 import org.apache.oozie.XException;
028 import org.apache.oozie.executor.jpa.BundleJobsDeleteJPAExecutor;
029 import org.apache.oozie.executor.jpa.BundleJobsGetForPurgeJPAExecutor;
030 import org.apache.oozie.executor.jpa.CoordActionsDeleteJPAExecutor;
031 import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor;
032 import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery;
033 import org.apache.oozie.executor.jpa.CoordJobsCountNotForPurgeFromParentIdJPAExecutor;
034 import org.apache.oozie.executor.jpa.CoordJobsDeleteJPAExecutor;
035 import org.apache.oozie.executor.jpa.CoordJobsGetForPurgeJPAExecutor;
036 import org.apache.oozie.executor.jpa.CoordJobsGetFromParentIdJPAExecutor;
037 import org.apache.oozie.executor.jpa.JPAExecutorException;
038 import org.apache.oozie.executor.jpa.WorkflowJobsCountNotForPurgeFromCoordParentIdJPAExecutor;
039 import org.apache.oozie.executor.jpa.WorkflowJobsCountNotForPurgeFromWorkflowParentIdJPAExecutor;
040 import org.apache.oozie.executor.jpa.WorkflowJobsDeleteJPAExecutor;
041 import org.apache.oozie.executor.jpa.WorkflowJobsGetFromWorkflowParentIdJPAExecutor;
042 import org.apache.oozie.executor.jpa.WorkflowJobsGetForPurgeJPAExecutor;
043 import org.apache.oozie.executor.jpa.WorkflowJobsGetFromCoordParentIdJPAExecutor;
044 import org.apache.oozie.service.JPAService;
045 import org.apache.oozie.service.Services;
046 import org.apache.oozie.service.XLogCopyService;
047
048 /**
049 * This class is used to purge workflows, coordinators, and bundles. It takes into account the relationships between workflows and
050 * coordinators, and coordinators and bundles. It also only acts on 'limit' number of items at a time to not overtax the DB and in
051 * case something gets rolled back. Also, children are always deleted before their parents in case of a rollback.
052 */
053 public class PurgeXCommand extends XCommand<Void> {
054 private JPAService jpaService = null;
055 private int wfOlderThan;
056 private int coordOlderThan;
057 private int bundleOlderThan;
058 private boolean purgeOldCoordAction = false;
059 private final int limit;
060 private List<String> wfList;
061 private List<String> coordActionList;
062 private List<String> coordList;
063 private List<String> bundleList;
064 private int wfDel;
065 private int coordDel;
066 private int coordActionDel;
067 private int bundleDel;
068
069 public PurgeXCommand(int wfOlderThan, int coordOlderThan, int bundleOlderThan, int limit) {
070 this(wfOlderThan, coordOlderThan, bundleOlderThan, limit, false);
071 }
072
073 public PurgeXCommand(int wfOlderThan, int coordOlderThan, int bundleOlderThan, int limit, boolean purgeOldCoordAction) {
074 super("purge", "purge", 0);
075 this.wfOlderThan = wfOlderThan;
076 this.coordOlderThan = coordOlderThan;
077 this.bundleOlderThan = bundleOlderThan;
078 this.purgeOldCoordAction = purgeOldCoordAction;
079 this.limit = limit;
080 wfList = new ArrayList<String>();
081 coordActionList = new ArrayList<String>();
082 coordList = new ArrayList<String>();
083 bundleList = new ArrayList<String>();
084 wfDel = 0;
085 coordDel = 0;
086 bundleDel = 0;
087 }
088
089 /* (non-Javadoc)
090 * @see org.apache.oozie.command.XCommand#loadState()
091 */
092 @Override
093 protected void loadState() throws CommandException {
094 try {
095 jpaService = Services.get().get(JPAService.class);
096
097 if (jpaService != null) {
098 // Get the lists of workflows, coordinators, and bundles that can be purged (and have no parents)
099 int size;
100 do {
101 size = wfList.size();
102 wfList.addAll(jpaService.execute(new WorkflowJobsGetForPurgeJPAExecutor(wfOlderThan, wfList.size(), limit)));
103 } while(size != wfList.size());
104 if (purgeOldCoordAction) {
105 LOG.debug("Purging workflows of long running coordinators is turned on");
106 do {
107 size = coordActionList.size();
108 long olderThan = wfOlderThan;
109 List<WorkflowJobBean> jobBeans = WorkflowJobQueryExecutor.getInstance().getList(
110 WorkflowJobQuery.GET_COMPLETED_COORD_WORKFLOWS_OLDER_THAN, olderThan,
111 coordActionList.size(), limit);
112 for (WorkflowJobBean bean : jobBeans) {
113 coordActionList.add(bean.getParentId());
114 wfList.add(bean.getId());
115 }
116 } while(size != coordActionList.size());
117 }
118 do {
119 size = coordList.size();
120 coordList.addAll(jpaService.execute(
121 new CoordJobsGetForPurgeJPAExecutor(coordOlderThan, coordList.size(), limit)));
122 } while(size != coordList.size());
123 do {
124 size = bundleList.size();
125 bundleList.addAll(jpaService.execute(
126 new BundleJobsGetForPurgeJPAExecutor(bundleOlderThan, bundleList.size(), limit)));
127 } while(size != bundleList.size());
128 }
129 else {
130 throw new CommandException(ErrorCode.E0610);
131 }
132 }
133 catch (XException ex) {
134 throw new CommandException(ex);
135 }
136 }
137
138 /* (non-Javadoc)
139 * @see org.apache.oozie.command.XCommand#execute()
140 */
141 @Override
142 protected Void execute() throws CommandException {
143 LOG.info("STARTED Purge to purge Workflow Jobs older than [{0}] days, Coordinator Jobs older than [{1}] days, and Bundle"
144 + "jobs older than [{2}] days.", wfOlderThan, coordOlderThan, bundleOlderThan);
145
146 // Process parentless workflows to purge them and their children
147 if (!wfList.isEmpty()) {
148 try {
149 processWorkflows(wfList);
150 }
151 catch (JPAExecutorException je) {
152 throw new CommandException(je);
153 }
154 }
155
156 // Process coordinator actions of long running coordinators and purge them
157 if (!coordActionList.isEmpty()) {
158 try {
159 purgeCoordActions(coordActionList);
160 }
161 catch (JPAExecutorException je) {
162 throw new CommandException(je);
163 }
164 }
165 // Processs parentless coordinators to purge them and their children
166 if (!coordList.isEmpty()) {
167 try {
168 processCoordinators(coordList);
169 }
170 catch (JPAExecutorException je) {
171 throw new CommandException(je);
172 }
173 }
174
175 // Process bundles to purge them and their children
176 if (!bundleList.isEmpty()) {
177 try {
178 processBundles(bundleList);
179 }
180 catch (JPAExecutorException je) {
181 throw new CommandException(je);
182 }
183 }
184
185 LOG.info("ENDED Purge deleted [{0}] workflows, [{1}] coordinatorActions, [{2}] coordinators, [{3}] bundles",
186 wfDel, coordActionDel, coordDel, bundleDel);
187
188 if (XLogCopyService.IS_LOG_PURGING_ENABLED){
189 queue(new XLogPurgeXCommand(wfList, coordList, bundleList));
190 }
191 return null;
192 }
193
194 /**
195 * Process workflows to purge them and their children. Uses the processWorkflowsHelper method to help via recursion to make
196 * sure that the workflow children are deleted before their parents.
197 *
198 * @param wfs List of workflows to process
199 * @throws JPAExecutorException If a JPA executor has a problem
200 */
201 private void processWorkflows(List<String> wfs) throws JPAExecutorException {
202 List<String> wfsToPurge = processWorkflowsHelper(wfs);
203 for (String id: wfsToPurge) {
204 LOG.debug("Purging workflow " + id);
205 }
206 purgeWorkflows(wfsToPurge);
207 }
208
209 /**
210 * Used by the processWorkflows method and via recursion.
211 *
212 * @param wfs List of workflows to process
213 * @return List of workflows to purge
214 * @throws JPAExecutorException If a JPA executor has a problem
215 */
216 private List<String> processWorkflowsHelper(List<String> wfs) throws JPAExecutorException {
217 // If the list is empty, then we've finished recursing
218 if (wfs.isEmpty()) {
219 return wfs;
220 }
221 List<String> subwfs = new ArrayList<String>();
222 List<String> wfsToPurge = new ArrayList<String>();
223 for (String wfId : wfs) {
224 // We only purge the workflow and its children if they are all ready to be purged
225 long numChildrenNotReady = jpaService.execute(
226 new WorkflowJobsCountNotForPurgeFromWorkflowParentIdJPAExecutor(wfOlderThan, wfId));
227 if (numChildrenNotReady == 0) {
228 wfsToPurge.add(wfId);
229 // Get all of the direct children for this workflow
230 List<String> children = new ArrayList<String>();
231 int size;
232 do {
233 size = children.size();
234 children.addAll(jpaService.execute(
235 new WorkflowJobsGetFromWorkflowParentIdJPAExecutor(wfId, children.size(), limit)));
236 } while (size != children.size());
237 subwfs.addAll(children);
238 }
239 }
240 // Recurse on the children we just found to process their children
241 wfsToPurge.addAll(processWorkflowsHelper(subwfs));
242 return wfsToPurge;
243 }
244
245 /**
246 * Process coordinators to purge them and their children.
247 *
248 * @param coords List of coordinators to process
249 * @throws JPAExecutorException If a JPA executor has a problem
250 */
251 private void processCoordinators(List<String> coords) throws JPAExecutorException {
252 List<String> wfsToPurge = new ArrayList<String>();
253 List<String> coordsToPurge = new ArrayList<String>();
254 for (String coordId : coords) {
255 // We only purge the coord and its children if they are all ready to be purged
256 long numChildrenNotReady = jpaService.execute(
257 new WorkflowJobsCountNotForPurgeFromCoordParentIdJPAExecutor(wfOlderThan, coordId));
258 if (numChildrenNotReady == 0) {
259 coordsToPurge.add(coordId);
260 LOG.debug("Purging coordinator " + coordId);
261 // Get all of the direct children for this coord
262 List<String> children = new ArrayList<String>();
263 int size;
264 do {
265 size = children.size();
266 children.addAll(jpaService.execute(
267 new WorkflowJobsGetFromCoordParentIdJPAExecutor(coordId, children.size(), limit)));
268 } while (size != children.size());
269 wfsToPurge.addAll(children);
270 }
271 }
272 // Process the children
273 processWorkflows(wfsToPurge);
274 // Now that all children have been purged, we can purge the coordinators
275 purgeCoordinators(coordsToPurge);
276 }
277
278 /**
279 * Process bundles to purge them and their children
280 *
281 * @param bundles List of bundles to process
282 * @throws JPAExecutorException If a JPA executor has a problem
283 */
284 private void processBundles(List<String> bundles) throws JPAExecutorException {
285 List<String> coordsToPurge = new ArrayList<String>();
286 List<String> bundlesToPurge = new ArrayList<String>();
287 for (Iterator<String> it = bundles.iterator(); it.hasNext(); ) {
288 String bundleId = it.next();
289 // We only purge the bundle and its children if they are all ready to be purged
290 long numChildrenNotReady = jpaService.execute(
291 new CoordJobsCountNotForPurgeFromParentIdJPAExecutor(coordOlderThan, bundleId));
292 if (numChildrenNotReady == 0) {
293 bundlesToPurge.add(bundleId);
294 LOG.debug("Purging bundle " + bundleId);
295 // Get all of the direct children for this bundle
296 List<String> children = new ArrayList<String>();
297 int size;
298 do {
299 size = children.size();
300 children.addAll(jpaService.execute(new CoordJobsGetFromParentIdJPAExecutor(bundleId, children.size(), limit)));
301 } while (size != children.size());
302 coordsToPurge.addAll(children);
303 }
304 }
305 // Process the children
306 processCoordinators(coordsToPurge);
307 // Now that all children have been purged, we can purge the bundles
308 purgeBundles(bundlesToPurge);
309 }
310
311 /**
312 * Purge the workflows in REVERSE order in batches of size 'limit' (this must be done in reverse order so that children are
313 * purged before their parents)
314 *
315 * @param wfs List of workflows to purge
316 * @throws JPAExecutorException If a JPA executor has a problem
317 */
318 private void purgeWorkflows(List<String> wfs) throws JPAExecutorException {
319 wfDel += wfs.size();
320 Collections.reverse(wfs);
321 for (int startIndex = 0; startIndex < wfs.size(); ) {
322 int endIndex = (startIndex + limit < wfs.size()) ? (startIndex + limit) : wfs.size();
323 jpaService.execute(new WorkflowJobsDeleteJPAExecutor(wfs.subList(startIndex, endIndex)));
324 startIndex = endIndex;
325 }
326 }
327
328 /**
329 * Purge coordActions of long running coordinators and purge them
330 *
331 * @param coordActions List of coordActions to purge
332 * @throws JPAExecutorException If a JPA executor has a problem
333 */
334 private void purgeCoordActions(List<String> coordActions) throws JPAExecutorException {
335 coordActionDel = coordActions.size();
336 for (int startIndex = 0; startIndex < coordActions.size(); ) {
337 int endIndex = (startIndex + limit < coordActions.size()) ? (startIndex + limit) : coordActions.size();
338 jpaService.execute(new CoordActionsDeleteJPAExecutor(coordActions.subList(startIndex, endIndex)));
339 startIndex = endIndex;
340 }
341 }
342 /**
343 * Purge the coordinators in SOME order in batches of size 'limit' (its in reverse order only for convenience)
344 *
345 * @param coords List of coordinators to purge
346 * @throws JPAExecutorException If a JPA executor has a problem
347 */
348 private void purgeCoordinators(List<String> coords) throws JPAExecutorException {
349 coordDel += coords.size();
350 for (int startIndex = 0; startIndex < coords.size(); ) {
351 int endIndex = (startIndex + limit < coords.size()) ? (startIndex + limit) : coords.size();
352 jpaService.execute(new CoordJobsDeleteJPAExecutor(coords.subList(startIndex, endIndex)));
353 startIndex = endIndex;
354 }
355 }
356
357 /**
358 * Purge the bundles in SOME order in batches of size 'limit' (its in reverse order only for convenience)
359 *
360 * @param bundles List of bundles to purge
361 * @throws JPAExecutorException If a JPA executor has a problem
362 */
363 private void purgeBundles(List<String> bundles) throws JPAExecutorException {
364 bundleDel += bundles.size();
365 for (int startIndex = 0; startIndex < bundles.size(); ) {
366 int endIndex = (startIndex + limit < bundles.size()) ? (startIndex + limit) : bundles.size();
367 jpaService.execute(new BundleJobsDeleteJPAExecutor(bundles.subList(startIndex, endIndex)));
368 startIndex = endIndex;
369 }
370 }
371
372 /* (non-Javadoc)
373 * @see org.apache.oozie.command.XCommand#getEntityKey()
374 */
375 @Override
376 public String getEntityKey() {
377 return null;
378 }
379
380 /* (non-Javadoc)
381 * @see org.apache.oozie.command.XCommand#isLockRequired()
382 */
383 @Override
384 protected boolean isLockRequired() {
385 return false;
386 }
387
388 /* (non-Javadoc)
389 * @see org.apache.oozie.command.XCommand#verifyPrecondition()
390 */
391 @Override
392 protected void verifyPrecondition() throws CommandException, PreconditionException {
393 }
394 }