001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.command.coord;
019    
020    import org.apache.hadoop.conf.Configuration;
021    import org.apache.oozie.action.hadoop.OozieJobInfo;
022    import org.apache.oozie.client.CoordinatorAction;
023    import org.apache.oozie.client.OozieClient;
024    import org.apache.oozie.CoordinatorActionBean;
025    import org.apache.oozie.DagEngineException;
026    import org.apache.oozie.DagEngine;
027    import org.apache.oozie.ErrorCode;
028    import org.apache.oozie.SLAEventBean;
029    import org.apache.oozie.WorkflowJobBean;
030    import org.apache.oozie.command.CommandException;
031    import org.apache.oozie.command.PreconditionException;
032    import org.apache.oozie.service.DagEngineService;
033    import org.apache.oozie.service.EventHandlerService;
034    import org.apache.oozie.service.JPAService;
035    import org.apache.oozie.service.Services;
036    import org.apache.oozie.util.JobUtils;
037    import org.apache.oozie.util.LogUtils;
038    import org.apache.oozie.util.ParamChecker;
039    import org.apache.oozie.util.XLog;
040    import org.apache.oozie.util.XmlUtils;
041    import org.apache.oozie.util.XConfiguration;
042    import org.apache.oozie.util.db.SLADbOperations;
043    import org.apache.oozie.client.SLAEvent.SlaAppType;
044    import org.apache.oozie.client.SLAEvent.Status;
045    import org.apache.oozie.client.rest.JsonBean;
046    import org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry;
047    import org.apache.oozie.executor.jpa.BatchQueryExecutor;
048    import org.apache.oozie.executor.jpa.CoordActionQueryExecutor.CoordActionQuery;
049    import org.apache.oozie.executor.jpa.JPAExecutorException;
050    import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor;
051    import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery;
052    import org.jdom.Element;
053    import org.jdom.JDOMException;
054    
055    import java.io.IOException;
056    import java.io.StringReader;
057    import java.util.ArrayList;
058    import java.util.Date;
059    import java.util.HashMap;
060    import java.util.List;
061    import java.util.Map;
062    
063    @SuppressWarnings("deprecation")
064    public class CoordActionStartXCommand extends CoordinatorXCommand<Void> {
065    
066        public static final String EL_ERROR = "EL_ERROR";
067        public static final String EL_EVAL_ERROR = "EL_EVAL_ERROR";
068        public static final String COULD_NOT_START = "COULD_NOT_START";
069        public static final String START_DATA_MISSING = "START_DATA_MISSING";
070        public static final String EXEC_DATA_MISSING = "EXEC_DATA_MISSING";
071    
072        private final XLog log = getLog();
073        private String actionId = null;
074        private String user = null;
075        private String appName = null;
076        private CoordinatorActionBean coordAction = null;
077        private JPAService jpaService = null;
078        private String jobId = null;
079        private List<UpdateEntry> updateList = new ArrayList<UpdateEntry>();
080        private List<JsonBean> insertList = new ArrayList<JsonBean>();
081    
082        public CoordActionStartXCommand(String id, String user, String appName, String jobId) {
083            //super("coord_action_start", "coord_action_start", 1, XLog.OPS);
084            super("coord_action_start", "coord_action_start", 1);
085            this.actionId = ParamChecker.notEmpty(id, "id");
086            this.user = ParamChecker.notEmpty(user, "user");
087            this.appName = ParamChecker.notEmpty(appName, "appName");
088            this.jobId = jobId;
089        }
090    
091        /**
092         * Create config to pass to WF Engine 1. Get createdConf from coord_actions table 2. Get actionXml from
093         * coord_actions table. Extract all 'property' tags and merge createdConf (overwrite duplicate keys). 3. Extract
094         * 'app-path' from actionXML. Create a new property called 'oozie.wf.application.path' and merge with createdConf
095         * (overwrite duplicate keys) 4. Read contents of config-default.xml in workflow directory. 5. Merge createdConf
096         * with config-default.xml (overwrite duplicate keys). 6. Results is runConf which is saved in coord_actions table.
097         * Merge Action createdConf with actionXml to create new runConf with replaced variables
098         *
099         * @param action CoordinatorActionBean
100         * @return Configuration
101         * @throws CommandException
102         */
103        private Configuration mergeConfig(CoordinatorActionBean action) throws CommandException {
104            String createdConf = action.getCreatedConf();
105            String actionXml = action.getActionXml();
106            Element workflowProperties = null;
107            try {
108                workflowProperties = XmlUtils.parseXml(actionXml);
109            }
110            catch (JDOMException e1) {
111                log.warn("Configuration parse error in:" + actionXml);
112                throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
113            }
114            // generate the 'runConf' for this action
115            // Step 1: runConf = createdConf
116            Configuration runConf = null;
117            try {
118                runConf = new XConfiguration(new StringReader(createdConf));
119            }
120            catch (IOException e1) {
121                log.warn("Configuration parse error in:" + createdConf);
122                throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
123            }
124            // Step 2: Merge local properties into runConf
125            // extract 'property' tags under 'configuration' block in the
126            // coordinator.xml (saved in actionxml column)
127            // convert Element to XConfiguration
128            Element configElement = workflowProperties.getChild("action", workflowProperties.getNamespace())
129                    .getChild("workflow", workflowProperties.getNamespace()).getChild("configuration",
130                                                                                      workflowProperties.getNamespace());
131            if (configElement != null) {
132                String strConfig = XmlUtils.prettyPrint(configElement).toString();
133                Configuration localConf;
134                try {
135                    localConf = new XConfiguration(new StringReader(strConfig));
136                }
137                catch (IOException e1) {
138                    log.warn("Configuration parse error in:" + strConfig);
139                    throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
140                }
141    
142                // copy configuration properties in coordinator.xml to the runConf
143                XConfiguration.copy(localConf, runConf);
144            }
145    
146            // Step 3: Extract value of 'app-path' in actionxml, and save it as a
147            // new property called 'oozie.wf.application.path'
148            // WF Engine requires the path to the workflow.xml to be saved under
149            // this property name
150            String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace()).getChild("workflow",
151                                                                                                               workflowProperties.getNamespace()).getChild("app-path", workflowProperties.getNamespace()).getValue();
152            runConf.set("oozie.wf.application.path", appPath);
153            return runConf;
154        }
155    
156        @Override
157        protected Void execute() throws CommandException {
158            boolean makeFail = true;
159            String errCode = "";
160            String errMsg = "";
161            ParamChecker.notEmpty(user, "user");
162    
163            log.debug("actionid=" + actionId + ", status=" + coordAction.getStatus());
164            if (coordAction.getStatus() == CoordinatorAction.Status.SUBMITTED) {
165                // log.debug("getting.. job id: " + coordAction.getJobId());
166                // create merged runConf to pass to WF Engine
167                Configuration runConf = mergeConfig(coordAction);
168                coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString());
169                // log.debug("%%% merged runconf=" +
170                // XmlUtils.prettyPrint(runConf).toString());
171                DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user);
172                try {
173                    Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf()));
174                    SLAEventBean slaEvent = SLADbOperations.createStatusEvent(coordAction.getSlaXml(), coordAction.getId(), Status.STARTED,
175                            SlaAppType.COORDINATOR_ACTION, log);
176                    if(slaEvent != null) {
177                        insertList.add(slaEvent);
178                    }
179                    if (OozieJobInfo.isJobInfoEnabled()) {
180                        conf.set(OozieJobInfo.COORD_NAME, appName);
181                        conf.set(OozieJobInfo.COORD_NOMINAL_TIME, coordAction.getNominalTimestamp().toString());
182                    }
183                    // Normalize workflow appPath here;
184                    JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf);
185                    String wfId = dagEngine.submitJobFromCoordinator(conf, actionId);
186                    coordAction.setStatus(CoordinatorAction.Status.RUNNING);
187                    coordAction.setExternalId(wfId);
188                    coordAction.incrementAndGetPending();
189    
190                    //store.updateCoordinatorAction(coordAction);
191                    JPAService jpaService = Services.get().get(JPAService.class);
192                    if (jpaService != null) {
193                        log.debug("Updating WF record for WFID :" + wfId + " with parent id: " + actionId);
194                        WorkflowJobBean wfJob = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW_STARTTIME, wfId);
195                        wfJob.setParentId(actionId);
196                        wfJob.setLastModifiedTime(new Date());
197                        BatchQueryExecutor executor = BatchQueryExecutor.getInstance();
198                        updateList.add(new UpdateEntry<WorkflowJobQuery>(
199                                WorkflowJobQuery.UPDATE_WORKFLOW_PARENT_MODIFIED, wfJob));
200                        updateList.add(new UpdateEntry<CoordActionQuery>(
201                                CoordActionQuery.UPDATE_COORD_ACTION_FOR_START, coordAction));
202                        try {
203                            executor.executeBatchInsertUpdateDelete(insertList, updateList, null);
204                            if (EventHandlerService.isEnabled()) {
205                                generateEvent(coordAction, user, appName, wfJob.getStartTime());
206                            }
207                        }
208                        catch (JPAExecutorException je) {
209                            throw new CommandException(je);
210                        }
211                    }
212                    else {
213                        log.error(ErrorCode.E0610);
214                    }
215    
216                    makeFail = false;
217                }
218                catch (DagEngineException dee) {
219                    errMsg = dee.getMessage();
220                    errCode = dee.getErrorCode().toString();
221                    log.warn("can not create DagEngine for submitting jobs", dee);
222                }
223                catch (CommandException ce) {
224                    errMsg = ce.getMessage();
225                    errCode = ce.getErrorCode().toString();
226                    log.warn("command exception occured ", ce);
227                }
228                catch (java.io.IOException ioe) {
229                    errMsg = ioe.getMessage();
230                    errCode = "E1005";
231                    log.warn("Configuration parse error. read from DB :" + coordAction.getRunConf(), ioe);
232                }
233                catch (Exception ex) {
234                    errMsg = ex.getMessage();
235                    errCode = "E1005";
236                    log.warn("can not create DagEngine for submitting jobs", ex);
237                }
238                finally {
239                    if (makeFail == true) { // No DB exception occurs
240                        log.error("Failing the action " + coordAction.getId() + ". Because " + errCode + " : " + errMsg);
241                        coordAction.setStatus(CoordinatorAction.Status.FAILED);
242                        if (errMsg.length() > 254) { // Because table column size is 255
243                            errMsg = errMsg.substring(0, 255);
244                        }
245                        coordAction.setErrorMessage(errMsg);
246                        coordAction.setErrorCode(errCode);
247    
248                        updateList = new ArrayList<UpdateEntry>();
249                        updateList.add(new UpdateEntry<CoordActionQuery>(
250                                        CoordActionQuery.UPDATE_COORD_ACTION_FOR_START, coordAction));
251                        insertList = new ArrayList<JsonBean>();
252    
253                        SLAEventBean slaEvent = SLADbOperations.createStatusEvent(coordAction.getSlaXml(), coordAction.getId(), Status.FAILED,
254                                SlaAppType.COORDINATOR_ACTION, log);
255                        if(slaEvent != null) {
256                            insertList.add(slaEvent); //Update SLA events
257                        }
258                        try {
259                            // call JPAExecutor to do the bulk writes
260                            BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null);
261                            if (EventHandlerService.isEnabled()) {
262                                generateEvent(coordAction, user, appName, null);
263                            }
264                        }
265                        catch (JPAExecutorException je) {
266                            throw new CommandException(je);
267                        }
268                        queue(new CoordActionReadyXCommand(coordAction.getJobId()));
269                    }
270                }
271            }
272            return null;
273        }
274    
275        @Override
276        public String getEntityKey() {
277            return this.jobId;
278        }
279    
280        @Override
281        protected boolean isLockRequired() {
282            return true;
283        }
284    
285        @Override
286        protected void loadState() throws CommandException {
287            jpaService = Services.get().get(JPAService.class);
288            try {
289                coordAction = jpaService.execute(new org.apache.oozie.executor.jpa.CoordActionGetForStartJPAExecutor(
290                        actionId));
291            }
292            catch (JPAExecutorException je) {
293                throw new CommandException(je);
294            }
295            LogUtils.setLogInfo(coordAction, logInfo);
296        }
297    
298        @Override
299        protected void verifyPrecondition() throws PreconditionException {
300            if (coordAction.getStatus() != CoordinatorAction.Status.SUBMITTED) {
301                throw new PreconditionException(ErrorCode.E1100, "The coord action [" + actionId + "] must have status "
302                        + CoordinatorAction.Status.SUBMITTED.name() + " but has status [" + coordAction.getStatus().name() + "]");
303            }
304        }
305    
306        @Override
307        public String getKey(){
308            return getName() + "_" + actionId;
309        }
310    }