001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.action.hadoop;
019
020 import java.io.IOException;
021 import java.io.StringReader;
022 import java.util.ArrayList;
023 import java.util.List;
024
025 import org.apache.hadoop.conf.Configuration;
026 import org.apache.hadoop.fs.FileSystem;
027 import org.apache.hadoop.fs.Path;
028 import org.apache.hadoop.mapred.Counters;
029 import org.apache.hadoop.mapred.JobClient;
030 import org.apache.hadoop.mapred.JobConf;
031 import org.apache.hadoop.mapred.JobID;
032 import org.apache.hadoop.mapred.RunningJob;
033 import org.apache.oozie.action.ActionExecutorException;
034 import org.apache.oozie.client.WorkflowAction;
035 import org.apache.oozie.service.Services;
036 import org.apache.oozie.util.XConfiguration;
037 import org.apache.oozie.util.XLog;
038 import org.apache.oozie.util.XmlUtils;
039 import org.jdom.Element;
040 import org.jdom.Namespace;
041 import org.json.simple.JSONObject;
042
043 public class MapReduceActionExecutor extends JavaActionExecutor {
044
045 public static final String OOZIE_ACTION_EXTERNAL_STATS_WRITE = "oozie.action.external.stats.write";
046 public static final String HADOOP_COUNTERS = "hadoop.counters";
047 public static final String OOZIE_MAPREDUCE_UBER_JAR_ENABLE = "oozie.action.mapreduce.uber.jar.enable";
048 private static final String STREAMING_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.StreamingMain";
049 private XLog log = XLog.getLog(getClass());
050
051 public MapReduceActionExecutor() {
052 super("map-reduce");
053 }
054
055 @SuppressWarnings("rawtypes")
056 @Override
057 public List<Class> getLauncherClasses() {
058 List<Class> classes = new ArrayList<Class>();
059 try {
060 classes.add(Class.forName(STREAMING_MAIN_CLASS_NAME));
061 }
062 catch (ClassNotFoundException e) {
063 throw new RuntimeException("Class not found", e);
064 }
065 return classes;
066 }
067
068 @Override
069 protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
070 String mainClass;
071 Namespace ns = actionXml.getNamespace();
072 if (actionXml.getChild("streaming", ns) != null) {
073 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, STREAMING_MAIN_CLASS_NAME);
074 }
075 else {
076 if (actionXml.getChild("pipes", ns) != null) {
077 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, PipesMain.class.getName());
078 }
079 else {
080 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, MapReduceMain.class.getName());
081 }
082 }
083 return mainClass;
084 }
085
086 @Override
087 Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException {
088 super.setupLauncherConf(conf, actionXml, appPath, context);
089 conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true);
090 return conf;
091 }
092
093 @Override
094 @SuppressWarnings("unchecked")
095 Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
096 throws ActionExecutorException {
097 boolean regularMR = false;
098 Namespace ns = actionXml.getNamespace();
099 if (actionXml.getChild("streaming", ns) != null) {
100 Element streamingXml = actionXml.getChild("streaming", ns);
101 String mapper = streamingXml.getChildTextTrim("mapper", ns);
102 String reducer = streamingXml.getChildTextTrim("reducer", ns);
103 String recordReader = streamingXml.getChildTextTrim("record-reader", ns);
104 List<Element> list = (List<Element>) streamingXml.getChildren("record-reader-mapping", ns);
105 String[] recordReaderMapping = new String[list.size()];
106 for (int i = 0; i < list.size(); i++) {
107 recordReaderMapping[i] = list.get(i).getTextTrim();
108 }
109 list = (List<Element>) streamingXml.getChildren("env", ns);
110 String[] env = new String[list.size()];
111 for (int i = 0; i < list.size(); i++) {
112 env[i] = list.get(i).getTextTrim();
113 }
114 setStreaming(actionConf, mapper, reducer, recordReader, recordReaderMapping, env);
115 }
116 else {
117 if (actionXml.getChild("pipes", ns) != null) {
118 Element pipesXml = actionXml.getChild("pipes", ns);
119 String map = pipesXml.getChildTextTrim("map", ns);
120 String reduce = pipesXml.getChildTextTrim("reduce", ns);
121 String inputFormat = pipesXml.getChildTextTrim("inputformat", ns);
122 String partitioner = pipesXml.getChildTextTrim("partitioner", ns);
123 String writer = pipesXml.getChildTextTrim("writer", ns);
124 String program = pipesXml.getChildTextTrim("program", ns);
125 PipesMain.setPipes(actionConf, map, reduce, inputFormat, partitioner, writer, program, appPath);
126 }
127 else {
128 regularMR = true;
129 }
130 }
131 actionConf = super.setupActionConf(actionConf, context, actionXml, appPath);
132
133 // For "regular" (not streaming or pipes) MR jobs
134 if (regularMR) {
135 // Resolve uber jar path (has to be done after super because oozie.mapreduce.uber.jar is under <configuration>)
136 String uberJar = actionConf.get(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR);
137 if (uberJar != null) {
138 if (!Services.get().getConf().getBoolean(OOZIE_MAPREDUCE_UBER_JAR_ENABLE, false)) {
139 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "MR003",
140 "{0} property is not allowed. Set {1} to true in oozie-site to enable.",
141 MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, OOZIE_MAPREDUCE_UBER_JAR_ENABLE);
142 }
143 String nameNode = actionXml.getChildTextTrim("name-node", ns);
144 if (nameNode != null) {
145 Path uberJarPath = new Path(uberJar);
146 if (uberJarPath.toUri().getScheme() == null || uberJarPath.toUri().getAuthority() == null) {
147 if (uberJarPath.isAbsolute()) { // absolute path without namenode --> prepend namenode
148 Path nameNodePath = new Path(nameNode);
149 String nameNodeSchemeAuthority = nameNodePath.toUri().getScheme()
150 + "://" + nameNodePath.toUri().getAuthority();
151 actionConf.set(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR,
152 new Path(nameNodeSchemeAuthority + uberJarPath).toString());
153 }
154 else { // relative path --> prepend app path
155 actionConf.set(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, new Path(appPath, uberJarPath).toString());
156 }
157 }
158 }
159 }
160 }
161 else {
162 if (actionConf.get(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR) != null) {
163 log.warn("The " + MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR + " property is only applicable for MapReduce (not"
164 + "streaming nor pipes) workflows, ignoring");
165 actionConf.set(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, "");
166 }
167 }
168
169 return actionConf;
170 }
171
172 @Override
173 public void end(Context context, WorkflowAction action) throws ActionExecutorException {
174 super.end(context, action);
175 JobClient jobClient = null;
176 boolean exception = false;
177 try {
178 if (action.getStatus() == WorkflowAction.Status.OK) {
179 Element actionXml = XmlUtils.parseXml(action.getConf());
180 JobConf jobConf = createBaseHadoopConf(context, actionXml);
181 jobClient = createJobClient(context, jobConf);
182 RunningJob runningJob = jobClient.getJob(JobID.forName(action.getExternalChildIDs()));
183 if (runningJob == null) {
184 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR002",
185 "Unknown hadoop job [{0}] associated with action [{1}]. Failing this action!",
186 action.getExternalChildIDs(), action.getId());
187 }
188
189 Counters counters = runningJob.getCounters();
190 if (counters != null) {
191 ActionStats stats = new MRStats(counters);
192 String statsJsonString = stats.toJSON();
193 context.setVar(HADOOP_COUNTERS, statsJsonString);
194
195 // If action stats write property is set to false by user or
196 // size of stats is greater than the maximum allowed size,
197 // do not store the action stats
198 if (Boolean.parseBoolean(evaluateConfigurationProperty(actionXml,
199 OOZIE_ACTION_EXTERNAL_STATS_WRITE, "false"))
200 && (statsJsonString.getBytes().length <= getMaxExternalStatsSize())) {
201 context.setExecutionStats(statsJsonString);
202 log.debug(
203 "Printing stats for Map-Reduce action as a JSON string : [{0}]", statsJsonString);
204 }
205 }
206 else {
207 context.setVar(HADOOP_COUNTERS, "");
208 XLog.getLog(getClass()).warn("Could not find Hadoop Counters for: [{0}]",
209 action.getExternalChildIDs());
210 }
211 }
212 }
213 catch (Exception ex) {
214 exception = true;
215 throw convertException(ex);
216 }
217 finally {
218 if (jobClient != null) {
219 try {
220 jobClient.close();
221 }
222 catch (Exception e) {
223 if (exception) {
224 log.error("JobClient error: ", e);
225 }
226 else {
227 throw convertException(e);
228 }
229 }
230 }
231 }
232 }
233
234 // Return the value of the specified configuration property
235 private String evaluateConfigurationProperty(Element actionConf, String key, String defaultValue) throws ActionExecutorException {
236 try {
237 String ret = defaultValue;
238 if (actionConf != null) {
239 Namespace ns = actionConf.getNamespace();
240 Element e = actionConf.getChild("configuration", ns);
241 if(e != null) {
242 String strConf = XmlUtils.prettyPrint(e).toString();
243 XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
244 ret = inlineConf.get(key, defaultValue);
245 }
246 }
247 return ret;
248 }
249 catch (IOException ex) {
250 throw convertException(ex);
251 }
252 }
253
254 @SuppressWarnings("unchecked")
255 private JSONObject counterstoJson(Counters counters) {
256
257 if (counters == null) {
258 return null;
259 }
260
261 JSONObject groups = new JSONObject();
262 for (String gName : counters.getGroupNames()) {
263 JSONObject group = new JSONObject();
264 for (Counters.Counter counter : counters.getGroup(gName)) {
265 String cName = counter.getName();
266 Long cValue = counter.getCounter();
267 group.put(cName, cValue);
268 }
269 groups.put(gName, group);
270 }
271 return groups;
272 }
273
274 /**
275 * Return the sharelib name for the action.
276 *
277 * @return returns <code>streaming</code> if mapreduce-streaming action, <code>NULL</code> otherwise.
278 * @param actionXml
279 */
280 @Override
281 protected String getDefaultShareLibName(Element actionXml) {
282 Namespace ns = actionXml.getNamespace();
283 return (actionXml.getChild("streaming", ns) != null) ? "mapreduce-streaming" : null;
284 }
285
286 @Override
287 JobConf createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml,
288 Configuration actionConf) throws ActionExecutorException {
289 // If the user is using a regular MapReduce job and specified an uber jar, we need to also set it for the launcher;
290 // so we override createLauncherConf to call super and then to set the uber jar if specified. At this point, checking that
291 // uber jars are enabled and resolving the uber jar path is already done by setupActionConf() when it parsed the actionConf
292 // argument and we can just look up the uber jar in the actionConf argument.
293 JobConf launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf);
294 Namespace ns = actionXml.getNamespace();
295 if (actionXml.getChild("streaming", ns) == null && actionXml.getChild("pipes", ns) == null) {
296 // Set for uber jar
297 String uberJar = actionConf.get(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR);
298 if (uberJar != null && uberJar.trim().length() > 0) {
299 launcherJobConf.setJar(uberJar);
300 }
301 }
302 return launcherJobConf;
303 }
304
305 public static void setStreaming(Configuration conf, String mapper, String reducer, String recordReader,
306 String[] recordReaderMapping, String[] env) {
307 if (mapper != null) {
308 conf.set("oozie.streaming.mapper", mapper);
309 }
310 if (reducer != null) {
311 conf.set("oozie.streaming.reducer", reducer);
312 }
313 if (recordReader != null) {
314 conf.set("oozie.streaming.record-reader", recordReader);
315 }
316 MapReduceMain.setStrings(conf, "oozie.streaming.record-reader-mapping", recordReaderMapping);
317 MapReduceMain.setStrings(conf, "oozie.streaming.env", env);
318 }
319
320 @Override
321 protected RunningJob getRunningJob(Context context, WorkflowAction action, JobClient jobClient) throws Exception{
322
323 RunningJob runningJob;
324 String launcherJobId = action.getExternalId();
325 String childJobId = action.getExternalChildIDs();
326
327 if (childJobId != null && childJobId.length() > 0) {
328 runningJob = jobClient.getJob(JobID.forName(childJobId));
329 }
330 else {
331 runningJob = jobClient.getJob(JobID.forName(launcherJobId));
332 }
333
334 return runningJob;
335 }
336 }