001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     * 
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     * 
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.util;
019    
020    import java.io.IOException;
021    import java.net.URI;
022    import java.util.HashMap;
023    import java.util.Map;
024    
025    import org.apache.hadoop.conf.Configuration;
026    import org.apache.hadoop.filecache.DistributedCache;
027    import org.apache.hadoop.fs.FileSystem;
028    import org.apache.hadoop.fs.Path;
029    import org.apache.oozie.ErrorCode;
030    import org.apache.oozie.client.OozieClient;
031    import org.apache.oozie.client.XOozieClient;
032    import org.apache.oozie.command.CommandException;
033    import org.apache.oozie.service.HadoopAccessorException;
034    import org.apache.oozie.service.HadoopAccessorService;
035    import org.apache.oozie.service.Services;
036    
037    /**
038     * Job utilities.
039     */
040    public class JobUtils {
041        /**
042         * Normalize appPath in job conf with the provided user/group - If it's not jobs via proxy submission, after
043         * normalization appPath always points to job's Xml definition file.
044         * <p/>
045         *
046         * @param user user
047         * @param group group
048         * @param conf job configuration.
049         * @throws IOException thrown if normalization can not be done properly.
050         */
051        public static void normalizeAppPath(String user, String group, Configuration conf) throws IOException {
052            if (user == null) {
053                throw new IllegalArgumentException("user cannot be null");
054            }
055    
056            if (conf.get(XOozieClient.IS_PROXY_SUBMISSION) != null) { // do nothing for proxy submission job;
057                return;
058            }
059    
060            String wfPathStr = conf.get(OozieClient.APP_PATH);
061            String coordPathStr = conf.get(OozieClient.COORDINATOR_APP_PATH);
062            String bundlePathStr = conf.get(OozieClient.BUNDLE_APP_PATH);
063            String appPathStr = wfPathStr != null ? wfPathStr : (coordPathStr != null ? coordPathStr : bundlePathStr);
064    
065            FileSystem fs = null;
066            try {
067                URI uri = new Path(appPathStr).toUri();
068                HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
069                Configuration fsConf = has.createJobConf(uri.getAuthority());
070                fs = has.createFileSystem(user, uri, fsConf);
071            }
072            catch (HadoopAccessorException ex) {
073                throw new IOException(ex.getMessage());
074            }
075    
076            Path appPath = new Path(appPathStr);
077            String normalizedAppPathStr = appPathStr;
078            if (!fs.exists(appPath)) {
079                throw new IOException("Error: " + appPathStr + " does not exist");
080            }
081    
082            if (wfPathStr != null) {
083                conf.set(OozieClient.APP_PATH, normalizedAppPathStr);
084            }
085            else if (coordPathStr != null) {
086                conf.set(OozieClient.COORDINATOR_APP_PATH, normalizedAppPathStr);
087            }
088            else if (bundlePathStr != null) {
089                conf.set(OozieClient.BUNDLE_APP_PATH, normalizedAppPathStr);
090            }
091        }
092    
093        /**
094         * This Function will parse the value of the changed values in key value manner. the change value would be
095         * key1=value1;key2=value2
096         *
097         * @param changeValue change value.
098         * @return This returns the hash with hash<[key1,value1],[key2,value2]>
099         * @throws CommandException thrown if changeValue cannot be parsed properly.
100         */
101        public static Map<String, String> parseChangeValue(String changeValue) throws CommandException {
102            if (changeValue == null || changeValue.trim().equalsIgnoreCase("")) {
103                throw new CommandException(ErrorCode.E1015, "change value can not be empty string or null");
104            }
105    
106            Map<String, String> map = new HashMap<String, String>();
107    
108            String[] tokens = changeValue.split(";");
109            for (String token : tokens) {
110                if (!token.contains("=")) {
111                    throw new CommandException(ErrorCode.E1015, changeValue,
112                            "change value must be name=value pair or name=(empty string)");
113                }
114    
115                String[] pair = token.split("=");
116                String key = pair[0];
117    
118                if (map.containsKey(key)) {
119                    throw new CommandException(ErrorCode.E1015, changeValue, "can not specify repeated change values on "
120                            + key);
121                }
122    
123                if (pair.length == 2) {
124                    map.put(key, pair[1]);
125                }
126                else if (pair.length == 1) {
127                    map.put(key, "");
128                }
129                else {
130                    throw new CommandException(ErrorCode.E1015, changeValue, "elements on " + key
131                            + " must be name=value pair or name=(empty string)");
132                }
133            }
134    
135            return map;
136        }
137    
138        /**
139         * This method provides a wrapper around hadoop 0.20/1.x and 0.23/2.x implementations.
140         * TODO: Remove the workaround when we drop the support for hadoop 0.20.
141         * @param file Path of the file to be added
142         * @param conf Configuration that contains the classpath setting
143         * @param fs FileSystem with respect to which path should be interpreted (may be null)
144         * @throws IOException
145         */
146        public static void addFileToClassPath(Path file, Configuration conf, FileSystem fs) throws IOException {
147          Configuration defaultConf = new Configuration();
148          XConfiguration.copy(conf, defaultConf);
149          if (fs == null) {
150            // it fails with conf, therefore we pass defaultConf instead
151            fs = file.getFileSystem(defaultConf);
152          }
153          // Hadoop 0.20/1.x.
154          if (defaultConf.get("yarn.resourcemanager.address") == null) {
155              // Duplicate hadoop 1.x code to workaround MAPREDUCE-2361 in Hadoop 0.20
156              // Refer OOZIE-1806.
157              String filepath = file.toUri().getPath();
158              String classpath = conf.get("mapred.job.classpath.files");
159              conf.set("mapred.job.classpath.files", classpath == null
160                  ? filepath
161                  : classpath + System.getProperty("path.separator") + filepath);
162              URI uri = fs.makeQualified(file).toUri();
163              DistributedCache.addCacheFile(uri, conf);
164          }
165          else { // Hadoop 0.23/2.x
166              DistributedCache.addFileToClassPath(file, conf, fs);
167          }
168        }
169    }