001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     * 
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     * 
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.action.oozie;
019    
020    import org.apache.oozie.client.OozieClientException;
021    import org.apache.oozie.action.ActionExecutor;
022    import org.apache.oozie.action.ActionExecutorException;
023    import org.apache.oozie.DagEngine;
024    import org.apache.oozie.LocalOozieClient;
025    import org.apache.oozie.WorkflowJobBean;
026    import org.apache.oozie.service.DagEngineService;
027    import org.apache.oozie.client.WorkflowAction;
028    import org.apache.oozie.client.OozieClient;
029    import org.apache.oozie.client.WorkflowJob;
030    import org.apache.oozie.command.CommandException;
031    import org.apache.oozie.util.ConfigUtils;
032    import org.apache.oozie.util.JobUtils;
033    import org.apache.oozie.util.PropertiesUtils;
034    import org.apache.oozie.util.XmlUtils;
035    import org.apache.oozie.util.XConfiguration;
036    import org.apache.oozie.util.XLog;
037    import org.apache.oozie.service.Services;
038    import org.apache.hadoop.conf.Configuration;
039    import org.jdom.Element;
040    import org.jdom.Namespace;
041    
042    import java.io.StringReader;
043    import java.io.IOException;
044    import java.util.Set;
045    import java.util.HashSet;
046    
047    public class SubWorkflowActionExecutor extends ActionExecutor {
048        public static final String ACTION_TYPE = "sub-workflow";
049        public static final String LOCAL = "local";
050        public static final String PARENT_ID = "oozie.wf.parent.id";
051        public static final String SUBWORKFLOW_MAX_DEPTH = "oozie.action.subworkflow.max.depth";
052        private static final String SUBWORKFLOW_DEPTH = "oozie.action.subworkflow.depth";
053    
054        private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
055    
056        static {
057            String[] badUserProps = {PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES,
058                    PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB,
059                    PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN,
060                    PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS};
061    
062            String[] badDefaultProps = {PropertiesUtils.HADOOP_USER};
063            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
064            PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
065        }
066    
067        protected SubWorkflowActionExecutor() {
068            super(ACTION_TYPE);
069        }
070    
071        public void initActionType() {
072            super.initActionType();
073        }
074    
075        protected OozieClient getWorkflowClient(Context context, String oozieUri) {
076            OozieClient oozieClient;
077            if (oozieUri.equals(LOCAL)) {
078                WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
079                String user = workflow.getUser();
080                String group = workflow.getGroup();
081                DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user);
082                oozieClient = new LocalOozieClient(dagEngine);
083            }
084            else {
085                // TODO we need to add authToken to the WC for the remote case
086                oozieClient = new OozieClient(oozieUri);
087            }
088            return oozieClient;
089        }
090    
091        protected void injectInline(Element eConf, Configuration subWorkflowConf) throws IOException,
092                ActionExecutorException {
093            if (eConf != null) {
094                String strConf = XmlUtils.prettyPrint(eConf).toString();
095                Configuration conf = new XConfiguration(new StringReader(strConf));
096                try {
097                    PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_DEFAULT_PROPERTIES);
098                }
099                catch (CommandException ex) {
100                    throw convertException(ex);
101                }
102                XConfiguration.copy(conf, subWorkflowConf);
103            }
104        }
105    
106        @SuppressWarnings("unchecked")
107        protected void injectCallback(Context context, Configuration conf) {
108            String callback = context.getCallbackUrl("$status");
109            if (conf.get(OozieClient.WORKFLOW_NOTIFICATION_URL) != null) {
110                XLog.getLog(getClass())
111                        .warn("Sub-Workflow configuration has a custom job end notification URI, overriding");
112            }
113            conf.set(OozieClient.WORKFLOW_NOTIFICATION_URL, callback);
114        }
115    
116        protected void injectRecovery(String externalId, Configuration conf) {
117            conf.set(OozieClient.EXTERNAL_ID, externalId);
118        }
119    
120        protected void injectParent(String parentId, Configuration conf) {
121            conf.set(PARENT_ID, parentId);
122        }
123    
124        protected void verifyAndInjectSubworkflowDepth(Configuration parentConf, Configuration conf) throws ActionExecutorException {
125            int depth = conf.getInt(SUBWORKFLOW_DEPTH, 0);
126            int maxDepth = Services.get().getConf().getInt(SUBWORKFLOW_MAX_DEPTH, 50);
127            if (depth >= maxDepth) {
128                throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "SUBWF001",
129                        "Depth [{0}] cannot exceed maximum subworkflow depth [{1}]", (depth + 1), maxDepth);
130            }
131            conf.setInt(SUBWORKFLOW_DEPTH, depth + 1);
132        }
133    
134        protected String checkIfRunning(OozieClient oozieClient, String extId) throws OozieClientException {
135            String jobId = oozieClient.getJobId(extId);
136            if (jobId.equals("")) {
137                return null;
138            }
139            return jobId;
140        }
141    
142        public void start(Context context, WorkflowAction action) throws ActionExecutorException {
143            try {
144                Element eConf = XmlUtils.parseXml(action.getConf());
145                Namespace ns = eConf.getNamespace();
146                Element e = eConf.getChild("oozie", ns);
147                String oozieUri = (e == null) ? LOCAL : e.getTextTrim();
148                OozieClient oozieClient = getWorkflowClient(context, oozieUri);
149                String subWorkflowId = null;
150                String extId = context.getRecoveryId();
151                String runningJobId = null;
152                if (extId != null) {
153                    runningJobId = checkIfRunning(oozieClient, extId);
154                }
155                if (runningJobId == null) {
156                    String appPath = eConf.getChild("app-path", ns).getTextTrim();
157    
158                    XConfiguration subWorkflowConf = new XConfiguration();
159                    Configuration parentConf = new XConfiguration(new StringReader(context.getWorkflow().getConf()));
160                    if (eConf.getChild(("propagate-configuration"), ns) != null) {
161                        XConfiguration.copy(parentConf, subWorkflowConf);
162                    }
163    
164                    // the proto has the necessary credentials
165                    Configuration protoActionConf = context.getProtoActionConf();
166                    XConfiguration.copy(protoActionConf, subWorkflowConf);
167                    subWorkflowConf.set(OozieClient.APP_PATH, appPath);
168                    String group = ConfigUtils.getWithDeprecatedCheck(parentConf, OozieClient.JOB_ACL, OozieClient.GROUP_NAME, null);
169                    if(group != null) {
170                        subWorkflowConf.set(OozieClient.GROUP_NAME, group);
171                    }
172                    injectInline(eConf.getChild("configuration", ns), subWorkflowConf);
173                    injectCallback(context, subWorkflowConf);
174                    injectRecovery(extId, subWorkflowConf);
175                    injectParent(context.getWorkflow().getId(), subWorkflowConf);
176                    verifyAndInjectSubworkflowDepth(parentConf, subWorkflowConf);
177    
178                    //TODO: this has to be refactored later to be done in a single place for REST calls and this
179                    JobUtils.normalizeAppPath(context.getWorkflow().getUser(), context.getWorkflow().getGroup(),
180                                              subWorkflowConf);
181    
182                    subWorkflowId = oozieClient.run(subWorkflowConf.toProperties());
183                }
184                else {
185                    subWorkflowId = runningJobId;
186                }
187                WorkflowJob workflow = oozieClient.getJobInfo(subWorkflowId);
188                String consoleUrl = workflow.getConsoleUrl();
189                context.setStartData(subWorkflowId, oozieUri, consoleUrl);
190                if (runningJobId != null) {
191                    check(context, action);
192                }
193            }
194            catch (Exception ex) {
195                throw convertException(ex);
196            }
197        }
198    
199        public void end(Context context, WorkflowAction action) throws ActionExecutorException {
200            try {
201                String externalStatus = action.getExternalStatus();
202                WorkflowAction.Status status = externalStatus.equals("SUCCEEDED") ? WorkflowAction.Status.OK
203                                               : WorkflowAction.Status.ERROR;
204                context.setEndData(status, getActionSignal(status));
205            }
206            catch (Exception ex) {
207                throw convertException(ex);
208            }
209        }
210    
211        public void check(Context context, WorkflowAction action) throws ActionExecutorException {
212            try {
213                String subWorkflowId = action.getExternalId();
214                String oozieUri = action.getTrackerUri();
215                OozieClient oozieClient = getWorkflowClient(context, oozieUri);
216                WorkflowJob subWorkflow = oozieClient.getJobInfo(subWorkflowId);
217                WorkflowJob.Status status = subWorkflow.getStatus();
218                switch (status) {
219                    case FAILED:
220                    case KILLED:
221                    case SUCCEEDED:
222                        context.setExecutionData(status.toString(), null);
223                        break;
224                    default:
225                        context.setExternalStatus(status.toString());
226                        break;
227                }
228            }
229            catch (Exception ex) {
230                throw convertException(ex);
231            }
232        }
233    
234        public void kill(Context context, WorkflowAction action) throws ActionExecutorException {
235            try {
236                String subWorkflowId = action.getExternalId();
237                String oozieUri = action.getTrackerUri();
238                OozieClient oozieClient = getWorkflowClient(context, oozieUri);
239                oozieClient.kill(subWorkflowId);
240                context.setEndData(WorkflowAction.Status.KILLED, getActionSignal(WorkflowAction.Status.KILLED));
241            }
242            catch (Exception ex) {
243                throw convertException(ex);
244            }
245        }
246    
247        private static Set<String> FINAL_STATUS = new HashSet<String>();
248    
249        static {
250            FINAL_STATUS.add("SUCCEEDED");
251            FINAL_STATUS.add("KILLED");
252            FINAL_STATUS.add("FAILED");
253        }
254    
255        public boolean isCompleted(String externalStatus) {
256            return FINAL_STATUS.contains(externalStatus);
257        }
258    }