001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.action.hadoop;
019
020 import java.io.BufferedReader;
021 import java.io.IOException;
022 import java.io.InputStream;
023 import java.io.InputStreamReader;
024 import java.io.OutputStream;
025 import java.io.OutputStreamWriter;
026 import java.io.Writer;
027 import java.math.BigInteger;
028 import java.security.MessageDigest;
029 import java.security.NoSuchAlgorithmException;
030 import java.security.PrivilegedExceptionAction;
031 import java.util.ArrayList;
032 import java.util.Collection;
033 import java.util.HashMap;
034 import java.util.List;
035 import java.util.Map;
036 import java.util.Properties;
037
038 import org.apache.hadoop.conf.Configuration;
039 import org.apache.hadoop.fs.FileSystem;
040 import org.apache.hadoop.fs.Path;
041 import org.apache.hadoop.io.SequenceFile;
042 import org.apache.hadoop.io.Text;
043 import org.apache.hadoop.mapred.JobConf;
044 import org.apache.hadoop.mapred.RunningJob;
045 import org.apache.hadoop.mapred.Counters;
046 import org.apache.hadoop.security.UserGroupInformation;
047 import org.apache.oozie.client.OozieClient;
048 import org.apache.oozie.client.WorkflowAction;
049 import org.apache.oozie.service.HadoopAccessorException;
050 import org.apache.oozie.service.HadoopAccessorService;
051 import org.apache.oozie.service.Services;
052 import org.apache.oozie.service.URIHandlerService;
053 import org.apache.oozie.service.UserGroupInformationService;
054 import org.apache.oozie.util.IOUtils;
055 import org.apache.oozie.util.PropertiesUtils;
056
057 public class LauncherMapperHelper {
058
059 public static String getRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId)
060 throws HadoopAccessorException, IOException {
061 String jobId = null;
062 Path recoveryFile = new Path(actionDir, recoveryId);
063 FileSystem fs = Services.get().get(HadoopAccessorService.class)
064 .createFileSystem(launcherConf.get("user.name"),recoveryFile.toUri(), launcherConf);
065
066 if (fs.exists(recoveryFile)) {
067 InputStream is = fs.open(recoveryFile);
068 BufferedReader reader = new BufferedReader(new InputStreamReader(is));
069 jobId = reader.readLine();
070 reader.close();
071 }
072 return jobId;
073
074 }
075
076 public static void setupMainClass(Configuration launcherConf, String javaMainClass) {
077 // Only set the javaMainClass if its not null or empty string, this way the user can override the action's main class via
078 // <configuration> property
079 if (javaMainClass != null && !javaMainClass.equals("")) {
080 launcherConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass);
081 }
082 }
083
084 public static void setupLauncherURIHandlerConf(Configuration launcherConf) {
085 for(Map.Entry<String, String> entry : Services.get().get(URIHandlerService.class).getLauncherConfig()) {
086 launcherConf.set(entry.getKey(), entry.getValue());
087 }
088 }
089
090 public static void setupMainArguments(Configuration launcherConf, String[] args) {
091 launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_COUNT, args.length);
092 for (int i = 0; i < args.length; i++) {
093 launcherConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i, args[i]);
094 }
095 }
096
097 public static void setupMaxOutputData(Configuration launcherConf, int maxOutputData) {
098 launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, maxOutputData);
099 }
100
101 /**
102 * Set the maximum value of stats data
103 *
104 * @param launcherConf the oozie launcher configuration
105 * @param maxStatsData the maximum allowed size of stats data
106 */
107 public static void setupMaxExternalStatsSize(Configuration launcherConf, int maxStatsData){
108 launcherConf.setInt(LauncherMapper.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, maxStatsData);
109 }
110
111 /**
112 * Set the maximum number of globbed files/dirs
113 *
114 * @param launcherConf the oozie launcher configuration
115 * @param fsGlobMax the maximum number of files/dirs for FS operation
116 */
117 public static void setupMaxFSGlob(Configuration launcherConf, int fsGlobMax){
118 launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX, fsGlobMax);
119 }
120
121 public static void setupLauncherInfo(JobConf launcherConf, String jobId, String actionId, Path actionDir,
122 String recoveryId, Configuration actionConf, String prepareXML) throws IOException, HadoopAccessorException {
123
124 launcherConf.setMapperClass(LauncherMapper.class);
125 launcherConf.setSpeculativeExecution(false);
126 launcherConf.setNumMapTasks(1);
127 launcherConf.setNumReduceTasks(0);
128
129 launcherConf.set(LauncherMapper.OOZIE_JOB_ID, jobId);
130 launcherConf.set(LauncherMapper.OOZIE_ACTION_ID, actionId);
131 launcherConf.set(LauncherMapper.OOZIE_ACTION_DIR_PATH, actionDir.toString());
132 launcherConf.set(LauncherMapper.OOZIE_ACTION_RECOVERY_ID, recoveryId);
133 launcherConf.set(LauncherMapper.ACTION_PREPARE_XML, prepareXML);
134
135 actionConf.set(LauncherMapper.OOZIE_JOB_ID, jobId);
136 actionConf.set(LauncherMapper.OOZIE_ACTION_ID, actionId);
137
138 if (Services.get().getConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) {
139 List<String> purgedEntries = new ArrayList<String>();
140 Collection<String> entries = actionConf.getStringCollection("mapreduce.job.cache.files");
141 for (String entry : entries) {
142 if (entry.contains("#")) {
143 purgedEntries.add(entry);
144 }
145 }
146 actionConf.setStrings("mapreduce.job.cache.files", purgedEntries.toArray(new String[purgedEntries.size()]));
147 launcherConf.setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true);
148 }
149
150 FileSystem fs =
151 Services.get().get(HadoopAccessorService.class).createFileSystem(launcherConf.get("user.name"),
152 actionDir.toUri(), launcherConf);
153 fs.mkdirs(actionDir);
154
155 OutputStream os = fs.create(new Path(actionDir, LauncherMapper.ACTION_CONF_XML));
156 try {
157 actionConf.writeXml(os);
158 } finally {
159 IOUtils.closeSafely(os);
160 }
161
162 launcherConf.setInputFormat(OozieLauncherInputFormat.class);
163 launcherConf.set("mapred.output.dir", new Path(actionDir, "output").toString());
164 }
165
166 public static void setupYarnRestartHandling(JobConf launcherJobConf, Configuration actionConf, String actionId)
167 throws NoSuchAlgorithmException {
168 launcherJobConf.setLong("oozie.job.launch.time", System.currentTimeMillis());
169 // Tags are limited to 100 chars so we need to hash them to make sure (the actionId otherwise doesn't have a max length)
170 String tag = getTag(actionId);
171 actionConf.set("mapreduce.job.tags", tag);
172 }
173
174 private static String getTag(String actionId) throws NoSuchAlgorithmException {
175 MessageDigest digest = MessageDigest.getInstance("MD5");
176 digest.update(actionId.getBytes(), 0, actionId.length());
177 String md5 = "oozie-" + new BigInteger(1, digest.digest()).toString(16);
178 return md5;
179 }
180
181 public static boolean isMainDone(RunningJob runningJob) throws IOException {
182 return runningJob.isComplete();
183 }
184
185 public static boolean isMainSuccessful(RunningJob runningJob) throws IOException {
186 boolean succeeded = runningJob.isSuccessful();
187 if (succeeded) {
188 Counters counters = runningJob.getCounters();
189 if (counters != null) {
190 Counters.Group group = counters.getGroup(LauncherMapper.COUNTER_GROUP);
191 if (group != null) {
192 succeeded = group.getCounter(LauncherMapper.COUNTER_LAUNCHER_ERROR) == 0;
193 }
194 }
195 }
196 return succeeded;
197 }
198
199 /**
200 * Determine whether action has external child jobs or not
201 * @param actionData
202 * @return true/false
203 * @throws IOException
204 */
205 public static boolean hasExternalChildJobs(Map<String, String> actionData) throws IOException {
206 return actionData.containsKey(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS);
207 }
208
209 /**
210 * Determine whether action has output data or not
211 * @param actionData
212 * @return true/false
213 * @throws IOException
214 */
215 public static boolean hasOutputData(Map<String, String> actionData) throws IOException {
216 return actionData.containsKey(LauncherMapper.ACTION_DATA_OUTPUT_PROPS);
217 }
218
219 /**
220 * Determine whether action has external stats or not
221 * @param actionData
222 * @return true/false
223 * @throws IOException
224 */
225 public static boolean hasStatsData(Map<String, String> actionData) throws IOException{
226 return actionData.containsKey(LauncherMapper.ACTION_DATA_STATS);
227 }
228
229 /**
230 * Determine whether action has new id (id swap) or not
231 * @param actionData
232 * @return true/false
233 * @throws IOException
234 */
235 public static boolean hasIdSwap(Map<String, String> actionData) throws IOException {
236 return actionData.containsKey(LauncherMapper.ACTION_DATA_NEW_ID);
237 }
238
239 /**
240 * Get the sequence file path storing all action data
241 * @param actionDir
242 * @return
243 */
244 public static Path getActionDataSequenceFilePath(Path actionDir) {
245 return new Path(actionDir, LauncherMapper.ACTION_DATA_SEQUENCE_FILE);
246 }
247
248 /**
249 * Utility function to load the contents of action data sequence file into
250 * memory object
251 *
252 * @param fs Action Filesystem
253 * @param actionDir Path
254 * @param conf Configuration
255 * @return Map action data
256 * @throws IOException
257 * @throws InterruptedException
258 */
259 public static Map<String, String> getActionData(final FileSystem fs, final Path actionDir, final Configuration conf)
260 throws IOException, InterruptedException {
261 UserGroupInformationService ugiService = Services.get().get(UserGroupInformationService.class);
262 UserGroupInformation ugi = ugiService.getProxyUser(conf.get(OozieClient.USER_NAME));
263
264 return ugi.doAs(new PrivilegedExceptionAction<Map<String, String>>() {
265 @Override
266 public Map<String, String> run() throws IOException {
267 Map<String, String> ret = new HashMap<String, String>();
268 Path seqFilePath = getActionDataSequenceFilePath(actionDir);
269 if (fs.exists(seqFilePath)) {
270 SequenceFile.Reader seqFile = new SequenceFile.Reader(fs, seqFilePath, conf);
271 Text key = new Text(), value = new Text();
272 while (seqFile.next(key, value)) {
273 ret.put(key.toString(), value.toString());
274 }
275 seqFile.close();
276 }
277 else { // maintain backward-compatibility. to be deprecated
278 org.apache.hadoop.fs.FileStatus[] files = fs.listStatus(actionDir);
279 InputStream is;
280 BufferedReader reader = null;
281 Properties props;
282 if (files != null && files.length > 0) {
283 for (int x = 0; x < files.length; x++) {
284 Path file = files[x].getPath();
285 if (file.equals(new Path(actionDir, "externalChildIds.properties"))) {
286 is = fs.open(file);
287 reader = new BufferedReader(new InputStreamReader(is));
288 ret.put(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS,
289 IOUtils.getReaderAsString(reader, -1));
290 }
291 else if (file.equals(new Path(actionDir, "newId.properties"))) {
292 is = fs.open(file);
293 reader = new BufferedReader(new InputStreamReader(is));
294 props = PropertiesUtils.readProperties(reader, -1);
295 ret.put(LauncherMapper.ACTION_DATA_NEW_ID, props.getProperty("id"));
296 }
297 else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_OUTPUT_PROPS))) {
298 int maxOutputData = conf.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA,
299 2 * 1024);
300 is = fs.open(file);
301 reader = new BufferedReader(new InputStreamReader(is));
302 ret.put(LauncherMapper.ACTION_DATA_OUTPUT_PROPS, PropertiesUtils
303 .propertiesToString(PropertiesUtils.readProperties(reader, maxOutputData)));
304 }
305 else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_STATS))) {
306 int statsMaxOutputData = conf.getInt(LauncherMapper.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE,
307 Integer.MAX_VALUE);
308 is = fs.open(file);
309 reader = new BufferedReader(new InputStreamReader(is));
310 ret.put(LauncherMapper.ACTION_DATA_STATS, PropertiesUtils
311 .propertiesToString(PropertiesUtils.readProperties(reader, statsMaxOutputData)));
312 }
313 else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_ERROR_PROPS))) {
314 is = fs.open(file);
315 reader = new BufferedReader(new InputStreamReader(is));
316 ret.put(LauncherMapper.ACTION_DATA_ERROR_PROPS, IOUtils.getReaderAsString(reader, -1));
317 }
318 }
319 }
320 }
321 return ret;
322 }
323 });
324 }
325 }