001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.command;
019    
020    import java.util.ArrayList;
021    import java.util.Collections;
022    import java.util.Iterator;
023    import java.util.List;
024    
025    import org.apache.oozie.WorkflowJobBean;
026    import org.apache.oozie.ErrorCode;
027    import org.apache.oozie.XException;
028    import org.apache.oozie.executor.jpa.BundleJobsDeleteJPAExecutor;
029    import org.apache.oozie.executor.jpa.BundleJobsGetForPurgeJPAExecutor;
030    import org.apache.oozie.executor.jpa.CoordActionsDeleteJPAExecutor;
031    import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor;
032    import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery;
033    import org.apache.oozie.executor.jpa.CoordJobsCountNotForPurgeFromParentIdJPAExecutor;
034    import org.apache.oozie.executor.jpa.CoordJobsDeleteJPAExecutor;
035    import org.apache.oozie.executor.jpa.CoordJobsGetForPurgeJPAExecutor;
036    import org.apache.oozie.executor.jpa.CoordJobsGetFromParentIdJPAExecutor;
037    import org.apache.oozie.executor.jpa.JPAExecutorException;
038    import org.apache.oozie.executor.jpa.WorkflowJobsCountNotForPurgeFromCoordParentIdJPAExecutor;
039    import org.apache.oozie.executor.jpa.WorkflowJobsCountNotForPurgeFromWorkflowParentIdJPAExecutor;
040    import org.apache.oozie.executor.jpa.WorkflowJobsDeleteJPAExecutor;
041    import org.apache.oozie.executor.jpa.WorkflowJobsGetFromWorkflowParentIdJPAExecutor;
042    import org.apache.oozie.executor.jpa.WorkflowJobsGetForPurgeJPAExecutor;
043    import org.apache.oozie.executor.jpa.WorkflowJobsGetFromCoordParentIdJPAExecutor;
044    import org.apache.oozie.service.JPAService;
045    import org.apache.oozie.service.Services;
046    import org.apache.oozie.service.XLogCopyService;
047    
048    /**
049     * This class is used to purge workflows, coordinators, and bundles.  It takes into account the relationships between workflows and
050     * coordinators, and coordinators and bundles.  It also only acts on 'limit' number of items at a time to not overtax the DB and in
051     * case something gets rolled back.  Also, children are always deleted before their parents in case of a rollback.
052     */
053    public class PurgeXCommand extends XCommand<Void> {
054        private JPAService jpaService = null;
055        private int wfOlderThan;
056        private int coordOlderThan;
057        private int bundleOlderThan;
058        private boolean purgeOldCoordAction = false;
059        private final int limit;
060        private List<String> wfList;
061        private List<String> coordActionList;
062        private List<String> coordList;
063        private List<String> bundleList;
064        private int wfDel;
065        private int coordDel;
066        private int coordActionDel;
067        private int bundleDel;
068    
069        public PurgeXCommand(int wfOlderThan, int coordOlderThan, int bundleOlderThan, int limit) {
070            this(wfOlderThan, coordOlderThan, bundleOlderThan, limit, false);
071        }
072    
073        public PurgeXCommand(int wfOlderThan, int coordOlderThan, int bundleOlderThan, int limit, boolean purgeOldCoordAction) {
074            super("purge", "purge", 0);
075            this.wfOlderThan = wfOlderThan;
076            this.coordOlderThan = coordOlderThan;
077            this.bundleOlderThan = bundleOlderThan;
078            this.purgeOldCoordAction = purgeOldCoordAction;
079            this.limit = limit;
080            wfList = new ArrayList<String>();
081            coordActionList = new ArrayList<String>();
082            coordList = new ArrayList<String>();
083            bundleList = new ArrayList<String>();
084            wfDel = 0;
085            coordDel = 0;
086            bundleDel = 0;
087        }
088    
089        /* (non-Javadoc)
090         * @see org.apache.oozie.command.XCommand#loadState()
091         */
092        @Override
093        protected void loadState() throws CommandException {
094            try {
095                jpaService = Services.get().get(JPAService.class);
096    
097                if (jpaService != null) {
098                    // Get the lists of workflows, coordinators, and bundles that can be purged (and have no parents)
099                    int size;
100                    do {
101                        size = wfList.size();
102                        wfList.addAll(jpaService.execute(new WorkflowJobsGetForPurgeJPAExecutor(wfOlderThan, wfList.size(), limit)));
103                    } while(size != wfList.size());
104                    if (purgeOldCoordAction) {
105                        LOG.debug("Purging workflows of long running coordinators is turned on");
106                        do {
107                            size = coordActionList.size();
108                            long olderThan = wfOlderThan;
109                            List<WorkflowJobBean> jobBeans = WorkflowJobQueryExecutor.getInstance().getList(
110                                    WorkflowJobQuery.GET_COMPLETED_COORD_WORKFLOWS_OLDER_THAN, olderThan,
111                                    coordActionList.size(), limit);
112                            for (WorkflowJobBean bean : jobBeans) {
113                                coordActionList.add(bean.getParentId());
114                                wfList.add(bean.getId());
115                            }
116                        } while(size != coordActionList.size());
117                    }
118                    do {
119                        size = coordList.size();
120                        coordList.addAll(jpaService.execute(
121                                new CoordJobsGetForPurgeJPAExecutor(coordOlderThan, coordList.size(), limit)));
122                    } while(size != coordList.size());
123                    do {
124                        size = bundleList.size();
125                        bundleList.addAll(jpaService.execute(
126                                new BundleJobsGetForPurgeJPAExecutor(bundleOlderThan, bundleList.size(), limit)));
127                    } while(size != bundleList.size());
128                }
129                else {
130                    throw new CommandException(ErrorCode.E0610);
131                }
132            }
133            catch (XException ex) {
134                throw new CommandException(ex);
135            }
136        }
137    
138        /* (non-Javadoc)
139         * @see org.apache.oozie.command.XCommand#execute()
140         */
141        @Override
142        protected Void execute() throws CommandException {
143            LOG.info("STARTED Purge to purge Workflow Jobs older than [{0}] days, Coordinator Jobs older than [{1}] days, and Bundle"
144                    + "jobs older than [{2}] days.", wfOlderThan, coordOlderThan, bundleOlderThan);
145    
146            // Process parentless workflows to purge them and their children
147            if (!wfList.isEmpty()) {
148                try {
149                    processWorkflows(wfList);
150                }
151                catch (JPAExecutorException je) {
152                    throw new CommandException(je);
153                }
154            }
155    
156            // Process coordinator actions of long running coordinators and purge them
157            if (!coordActionList.isEmpty()) {
158                try {
159                    purgeCoordActions(coordActionList);
160                }
161                catch (JPAExecutorException je) {
162                    throw new CommandException(je);
163                }
164            }
165            // Processs parentless coordinators to purge them and their children
166            if (!coordList.isEmpty()) {
167                try {
168                    processCoordinators(coordList);
169                }
170                catch (JPAExecutorException je) {
171                    throw new CommandException(je);
172                }
173            }
174    
175            // Process bundles to purge them and their children
176            if (!bundleList.isEmpty()) {
177                try {
178                    processBundles(bundleList);
179                }
180                catch (JPAExecutorException je) {
181                    throw new CommandException(je);
182                }
183            }
184    
185            LOG.info("ENDED Purge deleted [{0}] workflows, [{1}] coordinatorActions, [{2}] coordinators, [{3}] bundles",
186                    wfDel, coordActionDel, coordDel, bundleDel);
187    
188            if (XLogCopyService.IS_LOG_PURGING_ENABLED){
189                queue(new XLogPurgeXCommand(wfList, coordList, bundleList));
190            }
191            return null;
192        }
193    
194        /**
195         * Process workflows to purge them and their children.  Uses the processWorkflowsHelper method to help via recursion to make
196         * sure that the workflow children are deleted before their parents.
197         *
198         * @param wfs List of workflows to process
199         * @throws JPAExecutorException If a JPA executor has a problem
200         */
201        private void processWorkflows(List<String> wfs) throws JPAExecutorException {
202            List<String> wfsToPurge = processWorkflowsHelper(wfs);
203            for (String id: wfsToPurge) {
204                LOG.debug("Purging workflow " + id);
205            }
206            purgeWorkflows(wfsToPurge);
207        }
208    
209        /**
210         * Used by the processWorkflows method and via recursion.
211         *
212         * @param wfs List of workflows to process
213         * @return List of workflows to purge
214         * @throws JPAExecutorException If a JPA executor has a problem
215         */
216        private List<String> processWorkflowsHelper(List<String> wfs) throws JPAExecutorException {
217            // If the list is empty, then we've finished recursing
218            if (wfs.isEmpty()) {
219                return wfs;
220            }
221            List<String> subwfs = new ArrayList<String>();
222            List<String> wfsToPurge = new ArrayList<String>();
223            for (String wfId : wfs) {
224                // We only purge the workflow and its children if they are all ready to be purged
225                long numChildrenNotReady = jpaService.execute(
226                        new WorkflowJobsCountNotForPurgeFromWorkflowParentIdJPAExecutor(wfOlderThan, wfId));
227                if (numChildrenNotReady == 0) {
228                    wfsToPurge.add(wfId);
229                    // Get all of the direct children for this workflow
230                    List<String> children = new ArrayList<String>();
231                    int size;
232                    do {
233                        size = children.size();
234                        children.addAll(jpaService.execute(
235                                new WorkflowJobsGetFromWorkflowParentIdJPAExecutor(wfId, children.size(), limit)));
236                    } while (size != children.size());
237                    subwfs.addAll(children);
238                }
239            }
240            // Recurse on the children we just found to process their children
241            wfsToPurge.addAll(processWorkflowsHelper(subwfs));
242            return wfsToPurge;
243        }
244    
245        /**
246         * Process coordinators to purge them and their children.
247         *
248         * @param coords List of coordinators to process
249         * @throws JPAExecutorException If a JPA executor has a problem
250         */
251        private void processCoordinators(List<String> coords) throws JPAExecutorException {
252            List<String> wfsToPurge = new ArrayList<String>();
253            List<String> coordsToPurge = new ArrayList<String>();
254            for (String coordId : coords) {
255                // We only purge the coord and its children if they are all ready to be purged
256                long numChildrenNotReady = jpaService.execute(
257                        new WorkflowJobsCountNotForPurgeFromCoordParentIdJPAExecutor(wfOlderThan, coordId));
258                if (numChildrenNotReady == 0) {
259                    coordsToPurge.add(coordId);
260                    LOG.debug("Purging coordinator " + coordId);
261                    // Get all of the direct children for this coord
262                    List<String> children = new ArrayList<String>();
263                    int size;
264                    do {
265                        size = children.size();
266                        children.addAll(jpaService.execute(
267                                new WorkflowJobsGetFromCoordParentIdJPAExecutor(coordId, children.size(), limit)));
268                    } while (size != children.size());
269                    wfsToPurge.addAll(children);
270                }
271            }
272            // Process the children
273            processWorkflows(wfsToPurge);
274            // Now that all children have been purged, we can purge the coordinators
275            purgeCoordinators(coordsToPurge);
276        }
277    
278        /**
279         * Process bundles to purge them and their children
280         *
281         * @param bundles List of bundles to process
282         * @throws JPAExecutorException If a JPA executor has a problem
283         */
284        private void processBundles(List<String> bundles) throws JPAExecutorException {
285            List<String> coordsToPurge = new ArrayList<String>();
286            List<String> bundlesToPurge = new ArrayList<String>();
287            for (Iterator<String> it = bundles.iterator(); it.hasNext(); ) {
288                String bundleId = it.next();
289                // We only purge the bundle and its children if they are all ready to be purged
290                long numChildrenNotReady = jpaService.execute(
291                        new CoordJobsCountNotForPurgeFromParentIdJPAExecutor(coordOlderThan, bundleId));
292                if (numChildrenNotReady == 0) {
293                    bundlesToPurge.add(bundleId);
294                    LOG.debug("Purging bundle " + bundleId);
295                    // Get all of the direct children for this bundle
296                    List<String> children = new ArrayList<String>();
297                    int size;
298                    do {
299                        size = children.size();
300                        children.addAll(jpaService.execute(new CoordJobsGetFromParentIdJPAExecutor(bundleId, children.size(), limit)));
301                    } while (size != children.size());
302                    coordsToPurge.addAll(children);
303                }
304            }
305            // Process the children
306            processCoordinators(coordsToPurge);
307            // Now that all children have been purged, we can purge the bundles
308            purgeBundles(bundlesToPurge);
309        }
310    
311        /**
312         * Purge the workflows in REVERSE order in batches of size 'limit' (this must be done in reverse order so that children are
313         * purged before their parents)
314         *
315         * @param wfs List of workflows to purge
316         * @throws JPAExecutorException If a JPA executor has a problem
317         */
318        private void purgeWorkflows(List<String> wfs) throws JPAExecutorException {
319            wfDel += wfs.size();
320            Collections.reverse(wfs);
321            for (int startIndex = 0; startIndex < wfs.size(); ) {
322                int endIndex = (startIndex + limit < wfs.size()) ? (startIndex + limit) : wfs.size();
323                jpaService.execute(new WorkflowJobsDeleteJPAExecutor(wfs.subList(startIndex, endIndex)));
324                startIndex = endIndex;
325            }
326        }
327    
328        /**
329         * Purge coordActions of long running coordinators and purge them
330         *
331         * @param coordActions List of coordActions to purge
332         * @throws JPAExecutorException If a JPA executor has a problem
333         */
334        private void purgeCoordActions(List<String> coordActions) throws JPAExecutorException {
335            coordActionDel = coordActions.size();
336            for (int startIndex = 0; startIndex < coordActions.size(); ) {
337                int endIndex = (startIndex + limit < coordActions.size()) ? (startIndex + limit) : coordActions.size();
338                jpaService.execute(new CoordActionsDeleteJPAExecutor(coordActions.subList(startIndex, endIndex)));
339                startIndex = endIndex;
340            }
341        }
342        /**
343         * Purge the coordinators in SOME order in batches of size 'limit' (its in reverse order only for convenience)
344         *
345         * @param coords List of coordinators to purge
346         * @throws JPAExecutorException If a JPA executor has a problem
347         */
348        private void purgeCoordinators(List<String> coords) throws JPAExecutorException {
349            coordDel += coords.size();
350            for (int startIndex = 0; startIndex < coords.size(); ) {
351                int endIndex = (startIndex + limit < coords.size()) ? (startIndex + limit) : coords.size();
352                jpaService.execute(new CoordJobsDeleteJPAExecutor(coords.subList(startIndex, endIndex)));
353                startIndex = endIndex;
354            }
355        }
356    
357        /**
358         * Purge the bundles in SOME order in batches of size 'limit' (its in reverse order only for convenience)
359         *
360         * @param bundles List of bundles to purge
361         * @throws JPAExecutorException If a JPA executor has a problem
362         */
363        private void purgeBundles(List<String> bundles) throws JPAExecutorException {
364            bundleDel += bundles.size();
365            for (int startIndex = 0; startIndex < bundles.size(); ) {
366                int endIndex = (startIndex + limit < bundles.size()) ? (startIndex + limit) : bundles.size();
367                jpaService.execute(new BundleJobsDeleteJPAExecutor(bundles.subList(startIndex, endIndex)));
368                startIndex = endIndex;
369            }
370        }
371    
372        /* (non-Javadoc)
373         * @see org.apache.oozie.command.XCommand#getEntityKey()
374         */
375        @Override
376        public String getEntityKey() {
377            return null;
378        }
379    
380        /* (non-Javadoc)
381         * @see org.apache.oozie.command.XCommand#isLockRequired()
382         */
383        @Override
384        protected boolean isLockRequired() {
385            return false;
386        }
387    
388        /* (non-Javadoc)
389         * @see org.apache.oozie.command.XCommand#verifyPrecondition()
390         */
391        @Override
392        protected void verifyPrecondition() throws CommandException, PreconditionException {
393        }
394    }