001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.client;
019    
020    import java.io.BufferedReader;
021    import java.io.File;
022    import java.io.FileReader;
023    import java.io.IOException;
024    import java.io.InputStreamReader;
025    import java.net.HttpURLConnection;
026    import java.util.Properties;
027    
028    import org.apache.oozie.cli.OozieCLI;
029    import org.apache.oozie.client.rest.JsonTags;
030    import org.apache.oozie.client.rest.RestConstants;
031    import org.json.simple.JSONObject;
032    import org.json.simple.JSONValue;
033    
034    public class XOozieClient extends OozieClient {
035    
036        public static final String JT = "mapred.job.tracker";
037        public static final String JT_2 = "mapreduce.jobtracker.address";
038    
039        public static final String NN = "fs.default.name";
040        public static final String NN_2 = "fs.defaultFS";
041    
042        @Deprecated
043        public static final String JT_PRINCIPAL = "mapreduce.jobtracker.kerberos.principal";
044    
045        @Deprecated
046        public static final String NN_PRINCIPAL = "dfs.namenode.kerberos.principal";
047    
048        public static final String PIG_SCRIPT = "oozie.pig.script";
049    
050        public static final String PIG_OPTIONS = "oozie.pig.options";
051    
052        public static final String PIG_SCRIPT_PARAMS = "oozie.pig.script.params";
053    
054        public static final String HIVE_SCRIPT = "oozie.hive.script";
055    
056        public static final String HIVE_OPTIONS = "oozie.hive.options";
057    
058        public static final String HIVE_SCRIPT_PARAMS = "oozie.hive.script.params";
059    
060        public static final String SQOOP_COMMAND = "oozie.sqoop.command";
061    
062        public static final String SQOOP_OPTIONS = "oozie.sqoop.options";
063    
064        public static final String FILES = "oozie.files";
065    
066        public static final String ARCHIVES = "oozie.archives";
067    
068        public static final String IS_PROXY_SUBMISSION = "oozie.proxysubmission";
069    
070        protected XOozieClient() {
071        }
072    
073        /**
074         * Create an eXtended Workflow client instance.
075         *
076         * @param oozieUrl URL of the Oozie instance it will interact with.
077         */
078        public XOozieClient(String oozieUrl) {
079            super(oozieUrl);
080        }
081    
082        private String readScript(String script) throws IOException {
083            if (!new File(script).exists()) {
084                throw new IOException("Error: script file [" + script + "] does not exist");
085            }
086    
087            BufferedReader br = null;
088            try {
089                br = new BufferedReader(new FileReader(script));
090                StringBuilder sb = new StringBuilder();
091                String line;
092                while ((line = br.readLine()) != null) {
093                    sb.append(line + "\n");
094                }
095                return sb.toString();
096            }
097            finally {
098                try {
099                    br.close();
100                }
101                catch (IOException ex) {
102                    System.err.println("Error: " + ex.getMessage());
103                }
104            }
105        }
106    
107        private String serializeSqoopCommand(String[] command) {
108            StringBuilder sb = new StringBuilder();
109            for (String arg : command) {
110                sb.append(arg).append("\n");
111            }
112            return sb.toString();
113        }
114    
115        static void setStrings(Properties conf, String key, String[] values) {
116            if (values != null) {
117                conf.setProperty(key + ".size", (new Integer(values.length)).toString());
118                for (int i = 0; i < values.length; i++) {
119                    conf.setProperty(key + "." + i, values[i]);
120                }
121            }
122        }
123    
124        private void validateHttpSubmitConf(Properties conf) {
125            String JT = conf.getProperty(XOozieClient.JT);
126            String JT_2 = conf.getProperty(XOozieClient.JT_2);
127            if (JT == null) {
128                if(JT_2 == null) {
129                    throw new RuntimeException("jobtracker is not specified in conf");
130                }
131            }
132    
133            String NN = conf.getProperty(XOozieClient.NN);
134            String NN_2 = conf.getProperty(XOozieClient.NN_2);
135            if (NN == null) {
136                if(NN_2 == null) {
137                    throw new RuntimeException("namenode is not specified in conf");
138                } else {
139                    NN = NN_2;
140                }
141            }
142    
143            String libPath = conf.getProperty(LIBPATH);
144            if (libPath == null) {
145                throw new RuntimeException("libpath is not specified in conf");
146            }
147            if (!libPath.contains(":/")) {
148                String newLibPath;
149                if (libPath.startsWith("/")) {
150                    if(NN.endsWith("/")) {
151                        newLibPath = NN + libPath.substring(1);
152                    } else {
153                        newLibPath = NN + libPath;
154                    }
155                } else {
156                    throw new RuntimeException("libpath should be absolute");
157                }
158                conf.setProperty(LIBPATH, newLibPath);
159            }
160    
161            conf.setProperty(IS_PROXY_SUBMISSION, "true");
162        }
163    
164        /**
165         * Submit a Pig job via HTTP.
166         *
167         * @param conf job configuration.
168         * @param pigScriptFile pig script file.
169         * @param pigArgs pig arguments string.
170         * @return the job Id.
171         * @throws OozieClientException thrown if the job could not be submitted.
172         */
173        @Deprecated
174        public String submitPig(Properties conf, String pigScriptFile, String[] pigArgs) throws IOException, OozieClientException {
175            return submitScriptLanguage(conf, pigScriptFile, pigArgs, OozieCLI.PIG_CMD);
176        }
177    
178        /**
179         * Submit a Pig or Hive job via HTTP.
180         *
181         * @param conf job configuration.
182         * @param scriptFile  script file.
183         * @param args  arguments string.
184         * @return the job Id.
185         * @throws OozieClientException thrown if the job could not be submitted.
186         */
187        public String submitScriptLanguage(Properties conf, String scriptFile, String[] args, String jobType)
188                throws IOException, OozieClientException {
189            return submitScriptLanguage(conf, scriptFile, args, null, jobType);
190        }
191    
192        /**
193         * Submit a Pig or Hive job via HTTP.
194         *
195         * @param conf job configuration.
196         * @param scriptFile  script file.
197         * @param args  arguments string.
198         * @param params parameters string.
199         * @return the job Id.
200         * @throws OozieClientException thrown if the job could not be submitted.
201         */
202        public String submitScriptLanguage(Properties conf, String scriptFile, String[] args, String[] params, String jobType)
203                throws IOException, OozieClientException {
204            if (conf == null) {
205                throw new IllegalArgumentException("conf cannot be null");
206            }
207            if (scriptFile == null) {
208                throw new IllegalArgumentException("scriptFile cannot be null");
209            }
210    
211            validateHttpSubmitConf(conf);
212    
213            String script = "";
214            String options = "";
215            String scriptParams = "";
216    
217            if (jobType.equals(OozieCLI.HIVE_CMD)) {
218                script = XOozieClient.HIVE_SCRIPT;
219                options = XOozieClient.HIVE_OPTIONS;
220                scriptParams = XOozieClient.HIVE_SCRIPT_PARAMS;
221            }
222            else if (jobType.equals(OozieCLI.PIG_CMD)) {
223                script =  XOozieClient.PIG_SCRIPT;
224                options = XOozieClient.PIG_OPTIONS;
225                scriptParams = XOozieClient.PIG_SCRIPT_PARAMS;
226            }
227            else {
228                throw new IllegalArgumentException("jobType must be either pig or hive");
229            }
230    
231            conf.setProperty(script, readScript(scriptFile));
232            setStrings(conf, options, args);
233            setStrings(conf, scriptParams, params);
234    
235            return (new HttpJobSubmit(conf, jobType)).call();
236        }
237    
238        /**
239         * Submit a Sqoop job via HTTP.
240         *
241         * @param conf job configuration.
242         * @param command sqoop command to run.
243         * @param args  arguments string.
244         * @return the job Id.
245         * @throws OozieClientException thrown if the job could not be submitted.
246         */
247        public String submitSqoop(Properties conf, String[] command, String[] args)
248                throws OozieClientException {
249            if (conf == null) {
250                throw new IllegalArgumentException("conf cannot be null");
251            }
252            if (command == null) {
253                throw new IllegalArgumentException("command cannot be null");
254            }
255    
256            validateHttpSubmitConf(conf);
257    
258            conf.setProperty(XOozieClient.SQOOP_COMMAND, serializeSqoopCommand(command));
259            setStrings(conf, XOozieClient.SQOOP_OPTIONS, args);
260    
261            return (new HttpJobSubmit(conf, OozieCLI.SQOOP_CMD)).call();
262        }
263    
264        /**
265         * Submit a Map/Reduce job via HTTP.
266         *
267         * @param conf job configuration.
268         * @return the job Id.
269         * @throws OozieClientException thrown if the job could not be submitted.
270         */
271        public String submitMapReduce(Properties conf) throws OozieClientException {
272            if (conf == null) {
273                throw new IllegalArgumentException("conf cannot be null");
274            }
275    
276            validateHttpSubmitConf(conf);
277    
278            return (new HttpJobSubmit(conf, "mapreduce")).call();
279        }
280    
281        private class HttpJobSubmit extends ClientCallable<String> {
282            private Properties conf;
283    
284            HttpJobSubmit(Properties conf, String jobType) {
285                super("POST", RestConstants.JOBS, "", prepareParams(RestConstants.JOBTYPE_PARAM, jobType));
286                this.conf = notNull(conf, "conf");
287            }
288    
289            @Override
290            protected String call(HttpURLConnection conn) throws IOException, OozieClientException {
291                conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE);
292                writeToXml(conf, conn.getOutputStream());
293                if (conn.getResponseCode() == HttpURLConnection.HTTP_CREATED) {
294                    JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream()));
295                    return (String) json.get(JsonTags.JOB_ID);
296                }
297                if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
298                    handleError(conn);
299                }
300                return null;
301            }
302        }
303    
304        /**
305         * set LIBPATH for HTTP submission job.
306         *
307         * @param conf Configuration object.
308         * @param pathStr lib HDFS path.
309         */
310        public void setLib(Properties conf, String pathStr) {
311            conf.setProperty(LIBPATH, pathStr);
312        }
313    
314        /**
315         * The equivalent to <file> tag in oozie's workflow xml.
316         *
317         * @param conf Configuration object.
318         * @param file file HDFS path. A "#..." symbolic string can be appended to the path to specify symbolic link name.
319         *             For example, "/user/oozie/parameter_file#myparams". If no "#..." is specified, file name will be used as
320         *             symbolic link name.
321         */
322        public void addFile(Properties conf, String file) {
323            if (file == null || file.length() == 0) {
324                throw new IllegalArgumentException("file cannot be null or empty");
325            }
326            String files = conf.getProperty(FILES);
327            conf.setProperty(FILES, files == null ? file : files + "," + file);
328        }
329    
330        /**
331         * The equivalent to <archive> tag in oozie's workflow xml.
332         *
333         * @param conf Configuration object.
334         * @param file file HDFS path. A "#..." symbolic string can be appended to the path to specify symbolic link name.
335         *             For example, "/user/oozie/udf1.jar#my.jar". If no "#..." is specified, file name will be used as
336         *             symbolic link name.
337         */
338        public void addArchive(Properties conf, String file) {
339            if (file == null || file.length() == 0) {
340                throw new IllegalArgumentException("file cannot be null or empty");
341            }
342            String files = conf.getProperty(ARCHIVES);
343            conf.setProperty(ARCHIVES, files == null ? file : files + "," + file);
344        }
345    }