001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.command.bundle;
019    
020    import java.io.IOException;
021    import java.io.StringReader;
022    import java.util.Date;
023    import java.util.HashMap;
024    import java.util.List;
025    import java.util.Map;
026    import java.util.Map.Entry;
027    
028    import org.apache.hadoop.conf.Configuration;
029    import org.apache.oozie.BundleActionBean;
030    import org.apache.oozie.BundleJobBean;
031    import org.apache.oozie.ErrorCode;
032    import org.apache.oozie.XException;
033    import org.apache.oozie.action.hadoop.OozieJobInfo;
034    import org.apache.oozie.client.Job;
035    import org.apache.oozie.client.OozieClient;
036    import org.apache.oozie.client.rest.JsonBean;
037    import org.apache.oozie.command.CommandException;
038    import org.apache.oozie.command.PreconditionException;
039    import org.apache.oozie.command.StartTransitionXCommand;
040    import org.apache.oozie.command.coord.CoordSubmitXCommand;
041    import org.apache.oozie.executor.jpa.BatchQueryExecutor;
042    import org.apache.oozie.executor.jpa.BundleJobQueryExecutor;
043    import org.apache.oozie.executor.jpa.BundleJobQueryExecutor.BundleJobQuery;
044    import org.apache.oozie.executor.jpa.JPAExecutorException;
045    import org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry;
046    import org.apache.oozie.util.JobUtils;
047    import org.apache.oozie.util.LogUtils;
048    import org.apache.oozie.util.ParamChecker;
049    import org.apache.oozie.util.XConfiguration;
050    import org.apache.oozie.util.XmlUtils;
051    import org.jdom.Attribute;
052    import org.jdom.Element;
053    import org.jdom.JDOMException;
054    
055    /**
056     * The command to start Bundle job
057     */
058    public class BundleStartXCommand extends StartTransitionXCommand {
059        private final String jobId;
060        private BundleJobBean bundleJob;
061    
062        /**
063         * The constructor for class {@link BundleStartXCommand}
064         *
065         * @param jobId the bundle job id
066         */
067        public BundleStartXCommand(String jobId) {
068            super("bundle_start", "bundle_start", 1);
069            this.jobId = ParamChecker.notEmpty(jobId, "jobId");
070        }
071    
072        /**
073         * The constructor for class {@link BundleStartXCommand}
074         *
075         * @param jobId the bundle job id
076         * @param dryrun true if dryrun is enable
077         */
078        public BundleStartXCommand(String jobId, boolean dryrun) {
079            super("bundle_start", "bundle_start", 1, dryrun);
080            this.jobId = ParamChecker.notEmpty(jobId, "jobId");
081        }
082    
083        /* (non-Javadoc)
084         * @see org.apache.oozie.command.XCommand#getEntityKey()
085         */
086        @Override
087        public String getEntityKey() {
088            return jobId;
089        }
090    
091        @Override
092        public String getKey() {
093            return getName() + "_" + jobId;
094        }
095    
096        /* (non-Javadoc)
097         * @see org.apache.oozie.command.XCommand#isLockRequired()
098         */
099        @Override
100        protected boolean isLockRequired() {
101            return true;
102        }
103    
104        @Override
105        protected void verifyPrecondition() throws CommandException, PreconditionException {
106            if (bundleJob.getStatus() != Job.Status.PREP) {
107                String msg = "Bundle " + bundleJob.getId() + " is not in PREP status. It is in : " + bundleJob.getStatus();
108                LOG.info(msg);
109                throw new PreconditionException(ErrorCode.E1100, msg);
110            }
111        }
112    
113        @Override
114        public void loadState() throws CommandException {
115            try {
116                this.bundleJob = BundleJobQueryExecutor.getInstance().get(BundleJobQuery.GET_BUNDLE_JOB, jobId);
117                LogUtils.setLogInfo(bundleJob, logInfo);
118                super.setJob(bundleJob);
119            }
120            catch (XException ex) {
121                throw new CommandException(ex);
122            }
123        }
124    
125        /* (non-Javadoc)
126         * @see org.apache.oozie.command.StartTransitionXCommand#StartChildren()
127         */
128        @Override
129        public void StartChildren() throws CommandException {
130            LOG.debug("Started coord jobs for the bundle=[{0}]", jobId);
131            insertBundleActions();
132            startCoordJobs();
133            LOG.debug("Ended coord jobs for the bundle=[{0}]", jobId);
134        }
135    
136        /* (non-Javadoc)
137         * @see org.apache.oozie.command.TransitionXCommand#notifyParent()
138         */
139        @Override
140        public void notifyParent() {
141        }
142    
143        /* (non-Javadoc)
144         * @see org.apache.oozie.command.StartTransitionXCommand#performWrites()
145         */
146        @Override
147        public void performWrites() throws CommandException {
148            try {
149                BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null);
150            }
151            catch (JPAExecutorException e) {
152                throw new CommandException(e);
153            }
154        }
155    
156        /**
157         * Insert bundle actions
158         *
159         * @throws CommandException thrown if failed to create bundle actions
160         */
161        @SuppressWarnings("unchecked")
162        private void insertBundleActions() throws CommandException {
163            if (bundleJob != null) {
164                Map<String, Boolean> map = new HashMap<String, Boolean>();
165                try {
166                    Element bAppXml = XmlUtils.parseXml(bundleJob.getJobXml());
167                    List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace());
168                    for (Element elem : coordElems) {
169                        Attribute name = elem.getAttribute("name");
170                        Attribute critical = elem.getAttribute("critical");
171                        if (name != null) {
172                            if (map.containsKey(name.getValue())) {
173                                throw new CommandException(ErrorCode.E1304, name);
174                            }
175                            boolean isCritical = false;
176                            if (critical != null && Boolean.parseBoolean(critical.getValue())) {
177                                isCritical = true;
178                            }
179                            map.put(name.getValue(), isCritical);
180                        }
181                        else {
182                            throw new CommandException(ErrorCode.E1305);
183                        }
184                    }
185                }
186                catch (JDOMException jex) {
187                    throw new CommandException(ErrorCode.E1301, jex.getMessage(), jex);
188                }
189    
190                // if there is no coordinator for this bundle, failed it.
191                if (map.isEmpty()) {
192                    bundleJob.setStatus(Job.Status.FAILED);
193                    bundleJob.resetPending();
194                    try {
195                        BundleJobQueryExecutor.getInstance().executeUpdate(BundleJobQuery.UPDATE_BUNDLE_JOB_STATUS_PENDING, bundleJob);
196                    }
197                    catch (JPAExecutorException jex) {
198                        throw new CommandException(jex);
199                    }
200    
201                    LOG.debug("No coord jobs for the bundle=[{0}], failed it!!", jobId);
202                    throw new CommandException(ErrorCode.E1318, jobId);
203                }
204    
205                for (Entry<String, Boolean> coordName : map.entrySet()) {
206                    BundleActionBean action = createBundleAction(jobId, coordName.getKey(), coordName.getValue());
207                    insertList.add(action);
208                }
209            }
210            else {
211                throw new CommandException(ErrorCode.E0604, jobId);
212            }
213        }
214    
215        private BundleActionBean createBundleAction(String jobId, String coordName, boolean isCritical) {
216            BundleActionBean action = new BundleActionBean();
217            action.setBundleActionId(jobId + "_" + coordName);
218            action.setBundleId(jobId);
219            action.setCoordName(coordName);
220            action.setStatus(Job.Status.PREP);
221            action.setLastModifiedTime(new Date());
222            if (isCritical) {
223                action.setCritical();
224            }
225            else {
226                action.resetCritical();
227            }
228            return action;
229        }
230    
231        /**
232         * Start Coord Jobs
233         *
234         * @throws CommandException thrown if failed to start coord jobs
235         */
236        @SuppressWarnings("unchecked")
237        private void startCoordJobs() throws CommandException {
238            if (bundleJob != null) {
239                try {
240                    Element bAppXml = XmlUtils.parseXml(bundleJob.getJobXml());
241                    List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace());
242                    for (Element coordElem : coordElems) {
243                        Attribute name = coordElem.getAttribute("name");
244                        Configuration coordConf = mergeConfig(coordElem);
245                        coordConf.set(OozieClient.BUNDLE_ID, jobId);
246                        if (OozieJobInfo.isJobInfoEnabled()) {
247                            coordConf.set(OozieJobInfo.BUNDLE_NAME, bundleJob.getAppName());
248                        }
249    
250                        queue(new CoordSubmitXCommand(coordConf, bundleJob.getId(), name.getValue()));
251    
252                    }
253                    updateBundleAction();
254                }
255                catch (JDOMException jex) {
256                    throw new CommandException(ErrorCode.E1301, jex.getMessage(), jex);
257                }
258                catch (JPAExecutorException je) {
259                    throw new CommandException(je);
260                }
261            }
262            else {
263                throw new CommandException(ErrorCode.E0604, jobId);
264            }
265        }
266    
267        private void updateBundleAction() throws JPAExecutorException {
268            for(JsonBean bAction : insertList) {
269                BundleActionBean action = (BundleActionBean) bAction;
270                action.incrementAndGetPending();
271                action.setLastModifiedTime(new Date());
272            }
273        }
274    
275        /**
276         * Merge Bundle job config and the configuration from the coord job to pass
277         * to Coord Engine
278         *
279         * @param coordElem the coordinator configuration
280         * @return Configuration merged configuration
281         * @throws CommandException thrown if failed to merge configuration
282         */
283        private Configuration mergeConfig(Element coordElem) throws CommandException {
284            String jobConf = bundleJob.getConf();
285            // Step 1: runConf = jobConf
286            Configuration runConf = null;
287            try {
288                runConf = new XConfiguration(new StringReader(jobConf));
289            }
290            catch (IOException e1) {
291                LOG.warn("Configuration parse error in:" + jobConf);
292                throw new CommandException(ErrorCode.E1306, e1.getMessage(), e1);
293            }
294            // Step 2: Merge local properties into runConf
295            // extract 'property' tags under 'configuration' block in the coordElem
296            // convert Element to XConfiguration
297            Element localConfigElement = coordElem.getChild("configuration", coordElem.getNamespace());
298    
299            if (localConfigElement != null) {
300                String strConfig = XmlUtils.prettyPrint(localConfigElement).toString();
301                Configuration localConf;
302                try {
303                    localConf = new XConfiguration(new StringReader(strConfig));
304                }
305                catch (IOException e1) {
306                    LOG.warn("Configuration parse error in:" + strConfig);
307                    throw new CommandException(ErrorCode.E1307, e1.getMessage(), e1);
308                }
309    
310                // copy configuration properties in the coordElem to the runConf
311                XConfiguration.copy(localConf, runConf);
312            }
313    
314            // Step 3: Extract value of 'app-path' in coordElem, save it as a
315            // new property called 'oozie.coord.application.path', and normalize.
316            String appPath = coordElem.getChild("app-path", coordElem.getNamespace()).getValue();
317            runConf.set(OozieClient.COORDINATOR_APP_PATH, appPath);
318            // Normalize coordinator appPath here;
319            try {
320                JobUtils.normalizeAppPath(runConf.get(OozieClient.USER_NAME), runConf.get(OozieClient.GROUP_NAME), runConf);
321            }
322            catch (IOException e) {
323                throw new CommandException(ErrorCode.E1001, runConf.get(OozieClient.COORDINATOR_APP_PATH));
324            }
325            return runConf;
326        }
327    
328        /* (non-Javadoc)
329         * @see org.apache.oozie.command.TransitionXCommand#getJob()
330         */
331        @Override
332        public Job getJob() {
333            return bundleJob;
334        }
335    
336        /* (non-Javadoc)
337         * @see org.apache.oozie.command.TransitionXCommand#updateJob()
338         */
339        @Override
340        public void updateJob() throws CommandException {
341            updateList.add(new UpdateEntry<BundleJobQuery>(BundleJobQuery.UPDATE_BUNDLE_JOB_STATUS_PENDING, bundleJob));
342        }
343    }