001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.action.oozie;
019
020 import org.apache.oozie.client.OozieClientException;
021 import org.apache.oozie.action.ActionExecutor;
022 import org.apache.oozie.action.ActionExecutorException;
023 import org.apache.oozie.DagEngine;
024 import org.apache.oozie.LocalOozieClient;
025 import org.apache.oozie.WorkflowJobBean;
026 import org.apache.oozie.service.DagEngineService;
027 import org.apache.oozie.client.WorkflowAction;
028 import org.apache.oozie.client.OozieClient;
029 import org.apache.oozie.client.WorkflowJob;
030 import org.apache.oozie.command.CommandException;
031 import org.apache.oozie.util.ConfigUtils;
032 import org.apache.oozie.util.JobUtils;
033 import org.apache.oozie.util.PropertiesUtils;
034 import org.apache.oozie.util.XmlUtils;
035 import org.apache.oozie.util.XConfiguration;
036 import org.apache.oozie.util.XLog;
037 import org.apache.oozie.service.Services;
038 import org.apache.hadoop.conf.Configuration;
039 import org.jdom.Element;
040 import org.jdom.Namespace;
041
042 import java.io.StringReader;
043 import java.io.IOException;
044 import java.util.Set;
045 import java.util.HashSet;
046
047 public class SubWorkflowActionExecutor extends ActionExecutor {
048 public static final String ACTION_TYPE = "sub-workflow";
049 public static final String LOCAL = "local";
050 public static final String PARENT_ID = "oozie.wf.parent.id";
051 public static final String SUBWORKFLOW_MAX_DEPTH = "oozie.action.subworkflow.max.depth";
052 private static final String SUBWORKFLOW_DEPTH = "oozie.action.subworkflow.depth";
053
054 private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
055
056 static {
057 String[] badUserProps = {PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES,
058 PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB,
059 PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN,
060 PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS};
061
062 String[] badDefaultProps = {PropertiesUtils.HADOOP_USER};
063 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
064 PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
065 }
066
067 protected SubWorkflowActionExecutor() {
068 super(ACTION_TYPE);
069 }
070
071 public void initActionType() {
072 super.initActionType();
073 }
074
075 protected OozieClient getWorkflowClient(Context context, String oozieUri) {
076 OozieClient oozieClient;
077 if (oozieUri.equals(LOCAL)) {
078 WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
079 String user = workflow.getUser();
080 String group = workflow.getGroup();
081 DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user);
082 oozieClient = new LocalOozieClient(dagEngine);
083 }
084 else {
085 // TODO we need to add authToken to the WC for the remote case
086 oozieClient = new OozieClient(oozieUri);
087 }
088 return oozieClient;
089 }
090
091 protected void injectInline(Element eConf, Configuration subWorkflowConf) throws IOException,
092 ActionExecutorException {
093 if (eConf != null) {
094 String strConf = XmlUtils.prettyPrint(eConf).toString();
095 Configuration conf = new XConfiguration(new StringReader(strConf));
096 try {
097 PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_DEFAULT_PROPERTIES);
098 }
099 catch (CommandException ex) {
100 throw convertException(ex);
101 }
102 XConfiguration.copy(conf, subWorkflowConf);
103 }
104 }
105
106 @SuppressWarnings("unchecked")
107 protected void injectCallback(Context context, Configuration conf) {
108 String callback = context.getCallbackUrl("$status");
109 if (conf.get(OozieClient.WORKFLOW_NOTIFICATION_URL) != null) {
110 XLog.getLog(getClass())
111 .warn("Sub-Workflow configuration has a custom job end notification URI, overriding");
112 }
113 conf.set(OozieClient.WORKFLOW_NOTIFICATION_URL, callback);
114 }
115
116 protected void injectRecovery(String externalId, Configuration conf) {
117 conf.set(OozieClient.EXTERNAL_ID, externalId);
118 }
119
120 protected void injectParent(String parentId, Configuration conf) {
121 conf.set(PARENT_ID, parentId);
122 }
123
124 protected void verifyAndInjectSubworkflowDepth(Configuration parentConf, Configuration conf) throws ActionExecutorException {
125 int depth = conf.getInt(SUBWORKFLOW_DEPTH, 0);
126 int maxDepth = Services.get().getConf().getInt(SUBWORKFLOW_MAX_DEPTH, 50);
127 if (depth >= maxDepth) {
128 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "SUBWF001",
129 "Depth [{0}] cannot exceed maximum subworkflow depth [{1}]", (depth + 1), maxDepth);
130 }
131 conf.setInt(SUBWORKFLOW_DEPTH, depth + 1);
132 }
133
134 protected String checkIfRunning(OozieClient oozieClient, String extId) throws OozieClientException {
135 String jobId = oozieClient.getJobId(extId);
136 if (jobId.equals("")) {
137 return null;
138 }
139 return jobId;
140 }
141
142 public void start(Context context, WorkflowAction action) throws ActionExecutorException {
143 try {
144 Element eConf = XmlUtils.parseXml(action.getConf());
145 Namespace ns = eConf.getNamespace();
146 Element e = eConf.getChild("oozie", ns);
147 String oozieUri = (e == null) ? LOCAL : e.getTextTrim();
148 OozieClient oozieClient = getWorkflowClient(context, oozieUri);
149 String subWorkflowId = null;
150 String extId = context.getRecoveryId();
151 String runningJobId = null;
152 if (extId != null) {
153 runningJobId = checkIfRunning(oozieClient, extId);
154 }
155 if (runningJobId == null) {
156 String appPath = eConf.getChild("app-path", ns).getTextTrim();
157
158 XConfiguration subWorkflowConf = new XConfiguration();
159 Configuration parentConf = new XConfiguration(new StringReader(context.getWorkflow().getConf()));
160 if (eConf.getChild(("propagate-configuration"), ns) != null) {
161 XConfiguration.copy(parentConf, subWorkflowConf);
162 }
163
164 // the proto has the necessary credentials
165 Configuration protoActionConf = context.getProtoActionConf();
166 XConfiguration.copy(protoActionConf, subWorkflowConf);
167 subWorkflowConf.set(OozieClient.APP_PATH, appPath);
168 String group = ConfigUtils.getWithDeprecatedCheck(parentConf, OozieClient.JOB_ACL, OozieClient.GROUP_NAME, null);
169 if(group != null) {
170 subWorkflowConf.set(OozieClient.GROUP_NAME, group);
171 }
172 injectInline(eConf.getChild("configuration", ns), subWorkflowConf);
173 injectCallback(context, subWorkflowConf);
174 injectRecovery(extId, subWorkflowConf);
175 injectParent(context.getWorkflow().getId(), subWorkflowConf);
176 verifyAndInjectSubworkflowDepth(parentConf, subWorkflowConf);
177
178 //TODO: this has to be refactored later to be done in a single place for REST calls and this
179 JobUtils.normalizeAppPath(context.getWorkflow().getUser(), context.getWorkflow().getGroup(),
180 subWorkflowConf);
181
182 subWorkflowId = oozieClient.run(subWorkflowConf.toProperties());
183 }
184 else {
185 subWorkflowId = runningJobId;
186 }
187 WorkflowJob workflow = oozieClient.getJobInfo(subWorkflowId);
188 String consoleUrl = workflow.getConsoleUrl();
189 context.setStartData(subWorkflowId, oozieUri, consoleUrl);
190 if (runningJobId != null) {
191 check(context, action);
192 }
193 }
194 catch (Exception ex) {
195 throw convertException(ex);
196 }
197 }
198
199 public void end(Context context, WorkflowAction action) throws ActionExecutorException {
200 try {
201 String externalStatus = action.getExternalStatus();
202 WorkflowAction.Status status = externalStatus.equals("SUCCEEDED") ? WorkflowAction.Status.OK
203 : WorkflowAction.Status.ERROR;
204 context.setEndData(status, getActionSignal(status));
205 }
206 catch (Exception ex) {
207 throw convertException(ex);
208 }
209 }
210
211 public void check(Context context, WorkflowAction action) throws ActionExecutorException {
212 try {
213 String subWorkflowId = action.getExternalId();
214 String oozieUri = action.getTrackerUri();
215 OozieClient oozieClient = getWorkflowClient(context, oozieUri);
216 WorkflowJob subWorkflow = oozieClient.getJobInfo(subWorkflowId);
217 WorkflowJob.Status status = subWorkflow.getStatus();
218 switch (status) {
219 case FAILED:
220 case KILLED:
221 case SUCCEEDED:
222 context.setExecutionData(status.toString(), null);
223 break;
224 default:
225 context.setExternalStatus(status.toString());
226 break;
227 }
228 }
229 catch (Exception ex) {
230 throw convertException(ex);
231 }
232 }
233
234 public void kill(Context context, WorkflowAction action) throws ActionExecutorException {
235 try {
236 String subWorkflowId = action.getExternalId();
237 String oozieUri = action.getTrackerUri();
238 OozieClient oozieClient = getWorkflowClient(context, oozieUri);
239 oozieClient.kill(subWorkflowId);
240 context.setEndData(WorkflowAction.Status.KILLED, getActionSignal(WorkflowAction.Status.KILLED));
241 }
242 catch (Exception ex) {
243 throw convertException(ex);
244 }
245 }
246
247 private static Set<String> FINAL_STATUS = new HashSet<String>();
248
249 static {
250 FINAL_STATUS.add("SUCCEEDED");
251 FINAL_STATUS.add("KILLED");
252 FINAL_STATUS.add("FAILED");
253 }
254
255 public boolean isCompleted(String externalStatus) {
256 return FINAL_STATUS.contains(externalStatus);
257 }
258 }