001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.executor.jpa;
019    
020    import java.sql.Timestamp;
021    import java.text.ParseException;
022    import java.util.ArrayList;
023    import java.util.Collections;
024    import java.util.HashMap;
025    import java.util.Iterator;
026    import java.util.List;
027    import java.util.Map;
028    import java.util.Map.Entry;
029    import java.util.regex.Matcher;
030    import java.util.regex.Pattern;
031    
032    import javax.persistence.EntityManager;
033    import javax.persistence.Query;
034    
035    import org.apache.oozie.BundleJobBean;
036    import org.apache.oozie.client.BundleJob;
037    import org.apache.oozie.client.CoordinatorJob;
038    import org.apache.oozie.client.rest.BulkResponseImpl;
039    import org.apache.oozie.BulkResponseInfo;
040    import org.apache.oozie.CoordinatorActionBean;
041    import org.apache.oozie.CoordinatorJobBean;
042    import org.apache.oozie.ErrorCode;
043    import org.apache.oozie.StringBlob;
044    import org.apache.oozie.client.CoordinatorAction;
045    import org.apache.oozie.service.Services;
046    import org.apache.oozie.util.DateUtils;
047    import org.apache.oozie.util.ParamChecker;
048    
049    /**
050     * The query executor class for bulk monitoring queries i.e. debugging bundle ->
051     * coord actions directly
052     */
053    public class BulkJPAExecutor implements JPAExecutor<BulkResponseInfo> {
054        private Map<String, List<String>> bulkFilter;
055        // defaults
056        private int start = 1;
057        private int len = 50;
058        private enum PARAM_TYPE {
059            ID, NAME
060        }
061    
062        public BulkJPAExecutor(Map<String, List<String>> bulkFilter, int start, int len) {
063            ParamChecker.notNull(bulkFilter, "bulkFilter");
064            this.bulkFilter = bulkFilter;
065            this.start = start;
066            this.len = len;
067        }
068    
069        @Override
070        public String getName() {
071            return "BulkJPAExecutor";
072        }
073    
074        @Override
075        public BulkResponseInfo execute(EntityManager em) throws JPAExecutorException {
076            List<BulkResponseImpl> responseList = new ArrayList<BulkResponseImpl>();
077            Map<String, Timestamp> actionTimes = new HashMap<String, Timestamp>();
078    
079            try {
080                // Lightweight Query 1 on Bundle level to fetch the bundle job(s)
081                // corresponding to names or ids
082                List<BundleJobBean> bundleBeans = bundleQuery(em);
083    
084                // Join query between coordinator job and coordinator action tables
085                // to get entries for specific bundleId only
086                String conditions = actionQuery(em, bundleBeans, actionTimes, responseList);
087    
088                // Query to get the count of records
089                long total = countQuery(conditions, em, bundleBeans, actionTimes);
090    
091                BulkResponseInfo bulk = new BulkResponseInfo(responseList, start, len, total);
092                return bulk;
093            }
094            catch (Exception e) {
095                throw new JPAExecutorException(ErrorCode.E0603, e.getMessage(), e);
096            }
097        }
098    
099        /**
100         * build the bundle level query to get bundle beans for the specified ids or appnames
101         * @param em
102         * @return List BundleJobBeans
103         * @throws JPAExecutorException
104         */
105        @SuppressWarnings("unchecked")
106        private List<BundleJobBean> bundleQuery(EntityManager em) throws JPAExecutorException {
107            Query q = em.createNamedQuery("BULK_MONITOR_BUNDLE_QUERY");
108            StringBuilder bundleQuery = new StringBuilder(q.toString());
109    
110            StringBuilder whereClause = null;
111            List<String> bundles = bulkFilter.get(BulkResponseImpl.BULK_FILTER_BUNDLE);
112            if (bundles != null) {
113                PARAM_TYPE type = getParamType(bundles.get(0), 'B');
114                if (type == PARAM_TYPE.NAME) {
115                    whereClause = inClause(bundles, "appName", 'b');
116                }
117                else if (type == PARAM_TYPE.ID) {
118                    whereClause = inClause(bundles, "id", 'b');
119                }
120    
121                // Query: select <columns> from BundleJobBean b where b.id IN (...) _or_ b.appName IN (...)
122                bundleQuery.append(whereClause.replace(whereClause.indexOf("AND"), whereClause.indexOf("AND") + 3, "WHERE"));
123                List<Object[]> bundleObjs = (List<Object[]>) em.createQuery(bundleQuery.toString()).getResultList();
124                if (bundleObjs.isEmpty()) {
125                    throw new JPAExecutorException(ErrorCode.E0603, "No entries found for given bundle(s)");
126                }
127    
128                List<BundleJobBean> bundleBeans = new ArrayList<BundleJobBean>();
129                for (Object[] bundleElem : bundleObjs) {
130                    bundleBeans.add(constructBundleBean(bundleElem));
131                }
132                return bundleBeans;
133            }
134            return null;
135        }
136    
137        /**
138         * Validate and determine whether passed param is job-id or appname
139         * @param id
140         * @param job
141         * @return PARAM_TYPE
142         */
143        private PARAM_TYPE getParamType(String id, char job) {
144            Pattern p = Pattern.compile("\\d{7}-\\d{15}-" + Services.get().getSystemId() + "-" + job);
145            Matcher m = p.matcher(id);
146            if (m.matches()) {
147                return PARAM_TYPE.ID;
148            }
149            return PARAM_TYPE.NAME;
150        }
151    
152        /**
153         * Compose the coord action level query comprising bundle id/appname filter and coord action
154         * status filter (if specified) and start-time or nominal-time filter (if specified)
155         * @param em
156         * @param bundles
157         * @param times
158         * @param responseList
159         * @return Query string
160         * @throws ParseException
161         */
162        @SuppressWarnings("unchecked")
163        private String actionQuery(EntityManager em, List<BundleJobBean> bundles,
164                Map<String, Timestamp> times, List<BulkResponseImpl> responseList) throws ParseException {
165            Query q = em.createNamedQuery("BULK_MONITOR_ACTIONS_QUERY");
166            StringBuilder getActions = new StringBuilder(q.toString());
167            int offset = getActions.indexOf("ORDER");
168            StringBuilder conditionClause = new StringBuilder();
169    
170            List<String> coords = bulkFilter.get(BulkResponseImpl.BULK_FILTER_COORD);
171            // Query: Select <columns> from CoordinatorActionBean a, CoordinatorJobBean c WHERE a.jobId = c.id
172            // AND c.bundleId = :bundleId AND c.appName/id IN (...)
173            if (coords != null) {
174                PARAM_TYPE type = getParamType(coords.get(0), 'C');
175                if (type == PARAM_TYPE.NAME) {
176                    conditionClause.append(inClause(bulkFilter.get(BulkResponseImpl.BULK_FILTER_COORD), "appName", 'c'));
177                }
178                else if (type == PARAM_TYPE.ID) {
179                    conditionClause.append(inClause(bulkFilter.get(BulkResponseImpl.BULK_FILTER_COORD), "id", 'c'));
180                }
181            }
182            // Query: Select <columns> from CoordinatorActionBean a, CoordinatorJobBean c WHERE a.jobId = c.id
183            // AND c.bundleId = :bundleId AND c.appName/id IN (...) AND a.statusStr IN (...)
184            conditionClause.append(statusClause(bulkFilter.get(BulkResponseImpl.BULK_FILTER_STATUS)));
185            offset = getActions.indexOf("ORDER");
186            getActions.insert(offset - 1, conditionClause);
187    
188            // Query: Select <columns> from CoordinatorActionBean a, CoordinatorJobBean c WHERE a.jobId = c.id
189            // AND c.bundleId = :bundleId AND c.appName/id IN (...) AND a.statusStr IN (...)
190            // AND a.createdTimestamp >= startCreated _or_ a.createdTimestamp <= endCreated
191            // AND a.nominalTimestamp >= startNominal _or_ a.nominalTimestamp <= endNominal
192            timesClause(getActions, offset, times);
193            q = em.createQuery(getActions.toString());
194            Iterator<Entry<String, Timestamp>> iter = times.entrySet().iterator();
195            while (iter.hasNext()) {
196                Entry<String, Timestamp> time = iter.next();
197                q.setParameter(time.getKey(), time.getValue());
198            }
199            // pagination
200            q.setFirstResult(start - 1);
201            q.setMaxResults(len);
202            // repeatedly execute above query for each bundle
203            for (BundleJobBean bundle : bundles) {
204                q.setParameter("bundleId", bundle.getId());
205                List<Object[]> response = q.getResultList();
206                for (Object[] r : response) {
207                    BulkResponseImpl br = getResponseFromObject(bundle, r);
208                    responseList.add(br);
209                }
210            }
211            return q.toString();
212        }
213    
214        /**
215         * Get total number of records for use with offset and len in API
216         * @param clause
217         * @param em
218         * @param bundles
219         * @return total count of coord actions
220         */
221        private long countQuery(String clause, EntityManager em, List<BundleJobBean> bundles, Map<String, Timestamp> times) {
222            Query q = em.createNamedQuery("BULK_MONITOR_COUNT_QUERY");
223            StringBuilder getTotal = new StringBuilder(q.toString() + " ");
224            // Query: select COUNT(a) from CoordinatorActionBean a, CoordinatorJobBean c
225            // get entire WHERE clause from above i.e. actionQuery() for all conditions on coordinator job
226            // and action status and times
227            getTotal.append(clause.substring(clause.indexOf("WHERE"), clause.indexOf("ORDER")));
228            int offset = getTotal.indexOf("bundleId");
229            List<String> bundleIds = new ArrayList<String>();
230            for (BundleJobBean bundle : bundles) {
231                bundleIds.add(bundle.getId());
232            }
233            // Query: select COUNT(a) from CoordinatorActionBean a, CoordinatorJobBean c WHERE ...
234            // AND c.bundleId IN (... list of bundle ids) i.e. replace single :bundleId with list
235            getTotal = getTotal.replace(offset - 6, offset + 20, inClause(bundleIds, "bundleId", 'c').toString());
236            q = em.createQuery(getTotal.toString());
237            Iterator<Entry<String, Timestamp>> iter = times.entrySet().iterator();
238            while (iter.hasNext()) {
239                Entry<String, Timestamp> time = iter.next();
240                q.setParameter(time.getKey(), time.getValue());
241            }
242            long total = ((Long) q.getSingleResult()).longValue();
243            return total;
244        }
245    
246        // Form the where clause to filter by coordinator appname/id
247        private StringBuilder inClause(List<String> values, String col, char type) {
248            StringBuilder sb = new StringBuilder();
249            boolean firstVal = true;
250            for (String name : nullToEmpty(values)) {
251                if (firstVal) {
252                    sb.append(" AND " + type + "." + col + " IN (\'" + name + "\'");
253                    firstVal = false;
254                }
255                else {
256                    sb.append(",\'" + name + "\'");
257                }
258            }
259            if (!firstVal) {
260                sb.append(") ");
261            }
262            return sb;
263        }
264    
265        // Form the where clause to filter by coord action status
266        private StringBuilder statusClause(List<String> statuses) {
267            StringBuilder sb = inClause(statuses, "statusStr", 'a');
268            if (sb.length() == 0) { // statuses was null. adding default
269                sb.append(" AND a.statusStr IN ('KILLED', 'FAILED') ");
270            }
271            return sb;
272        }
273    
274        private void timesClause(StringBuilder sb, int offset, Map<String, Timestamp> eachTime) throws ParseException {
275            Timestamp ts = null;
276            List<String> times = bulkFilter.get(BulkResponseImpl.BULK_FILTER_START_CREATED_EPOCH);
277            if (times != null) {
278                ts = new Timestamp(DateUtils.parseDateUTC(times.get(0)).getTime());
279                sb.insert(offset - 1, " AND a.createdTimestamp >= :startCreated");
280                eachTime.put("startCreated", ts);
281            }
282            times = bulkFilter.get(BulkResponseImpl.BULK_FILTER_END_CREATED_EPOCH);
283            if (times != null) {
284                ts = new Timestamp(DateUtils.parseDateUTC(times.get(0)).getTime());
285                sb.insert(offset - 1, " AND a.createdTimestamp <= :endCreated");
286                eachTime.put("endCreated", ts);
287            }
288            times = bulkFilter.get(BulkResponseImpl.BULK_FILTER_START_NOMINAL_EPOCH);
289            if (times != null) {
290                ts = new Timestamp(DateUtils.parseDateUTC(times.get(0)).getTime());
291                sb.insert(offset - 1, " AND a.nominalTimestamp >= :startNominal");
292                eachTime.put("startNominal", ts);
293            }
294            times = bulkFilter.get(BulkResponseImpl.BULK_FILTER_END_NOMINAL_EPOCH);
295            if (times != null) {
296                ts = new Timestamp(DateUtils.parseDateUTC(times.get(0)).getTime());
297                sb.insert(offset - 1, " AND a.nominalTimestamp <= :endNominal");
298                eachTime.put("endNominal", ts);
299            }
300        }
301    
302        private BulkResponseImpl getResponseFromObject(BundleJobBean bundleBean, Object arr[]) {
303            BulkResponseImpl bean = new BulkResponseImpl();
304            CoordinatorJobBean coordBean = new CoordinatorJobBean();
305            CoordinatorActionBean actionBean = new CoordinatorActionBean();
306            if (arr[0] != null) {
307                actionBean.setId((String) arr[0]);
308            }
309            if (arr[1] != null) {
310                actionBean.setActionNumber((Integer) arr[1]);
311            }
312            if (arr[2] != null) {
313                actionBean.setErrorCode((String) arr[2]);
314            }
315            if (arr[3] != null) {
316                actionBean.setErrorMessage((String) arr[3]);
317            }
318            if (arr[4] != null) {
319                actionBean.setExternalId((String) arr[4]);
320            }
321            if (arr[5] != null) {
322                actionBean.setExternalStatus((String) arr[5]);
323            }
324            if (arr[6] != null) {
325                actionBean.setStatus(CoordinatorAction.Status.valueOf((String) arr[6]));
326            }
327            if (arr[7] != null) {
328                actionBean.setCreatedTime(DateUtils.toDate((Timestamp) arr[7]));
329            }
330            if (arr[8] != null) {
331                actionBean.setNominalTime(DateUtils.toDate((Timestamp) arr[8]));
332            }
333            if (arr[9] != null) {
334                actionBean.setMissingDependenciesBlob((StringBlob) arr[9]);
335            }
336            if (arr[10] != null) {
337                coordBean.setId((String) arr[10]);
338                actionBean.setJobId((String) arr[10]);
339            }
340            if (arr[11] != null) {
341                coordBean.setAppName((String) arr[11]);
342            }
343            if (arr[12] != null) {
344                coordBean.setStatus(CoordinatorJob.Status.valueOf((String) arr[12]));
345            }
346            bean.setBundle(bundleBean);
347            bean.setCoordinator(coordBean);
348            bean.setAction(actionBean);
349            return bean;
350        }
351    
352        private BundleJobBean constructBundleBean(Object[] barr) throws JPAExecutorException {
353            BundleJobBean bean = new BundleJobBean();
354            if (barr[0] != null) {
355                bean.setId((String) barr[0]);
356            }
357            else {
358                throw new JPAExecutorException(ErrorCode.E0603,
359                        "bundleId returned by query is null - cannot retrieve bulk results");
360            }
361            if (barr[1] != null) {
362                bean.setAppName((String) barr[1]);
363            }
364            if (barr[2] != null) {
365                bean.setStatus(BundleJob.Status.valueOf((String) barr[2]));
366            }
367            if (barr[3] != null) {
368                bean.setUser((String) barr[3]);
369            }
370            return bean;
371        }
372    
373        // null safeguard
374        public static List<String> nullToEmpty(List<String> input) {
375            return input == null ? Collections.<String> emptyList() : input;
376        }
377    
378    }