001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *     http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.action.hadoop;
019    
020    import java.io.IOException;
021    import java.io.StringReader;
022    import java.net.URISyntaxException;
023    import java.util.ArrayList;
024    import java.util.List;
025    import java.util.Properties;
026    import java.util.StringTokenizer;
027    
028    import org.apache.hadoop.conf.Configuration;
029    import org.apache.hadoop.fs.FileSystem;
030    import org.apache.hadoop.fs.Path;
031    import org.apache.hadoop.mapred.Counters;
032    import org.apache.hadoop.mapred.JobClient;
033    import org.apache.hadoop.mapred.JobConf;
034    import org.apache.hadoop.mapred.JobID;
035    import org.apache.hadoop.mapred.RunningJob;
036    import org.apache.oozie.action.ActionExecutorException;
037    import org.apache.oozie.client.WorkflowAction;
038    import org.apache.oozie.service.HadoopAccessorException;
039    import org.apache.oozie.util.XConfiguration;
040    import org.apache.oozie.util.XmlUtils;
041    import org.apache.oozie.util.XLog;
042    import org.jdom.Element;
043    import org.jdom.JDOMException;
044    import org.jdom.Namespace;
045    
046    public class SqoopActionExecutor extends JavaActionExecutor {
047    
048      public static final String OOZIE_ACTION_EXTERNAL_STATS_WRITE = "oozie.action.external.stats.write";
049      private static final String SQOOP_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.SqoopMain";
050      static final String SQOOP_ARGS = "oozie.sqoop.args";
051    
052        public SqoopActionExecutor() {
053            super("sqoop");
054        }
055    
056        @Override
057        public List<Class> getLauncherClasses() {
058            List<Class> classes = new ArrayList<Class>();
059            try {
060                classes.add(Class.forName(SQOOP_MAIN_CLASS_NAME));
061            }
062            catch (ClassNotFoundException e) {
063                throw new RuntimeException("Class not found", e);
064            }
065            return classes;
066        }
067    
068        @Override
069        protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
070            return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SQOOP_MAIN_CLASS_NAME);
071        }
072    
073        @Override
074        @SuppressWarnings("unchecked")
075        Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
076                throws ActionExecutorException {
077            super.setupActionConf(actionConf, context, actionXml, appPath);
078            Namespace ns = actionXml.getNamespace();
079    
080            try {
081                Element e = actionXml.getChild("configuration", ns);
082                if (e != null) {
083                    String strConf = XmlUtils.prettyPrint(e).toString();
084                    XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
085                    checkForDisallowedProps(inlineConf, "inline configuration");
086                    XConfiguration.copy(inlineConf, actionConf);
087                }
088            } catch (IOException ex) {
089                throw convertException(ex);
090            }
091    
092            String[] args;
093            if (actionXml.getChild("command", ns) != null) {
094                String command = actionXml.getChild("command", ns).getTextTrim();
095                StringTokenizer st = new StringTokenizer(command, " ");
096                List<String> l = new ArrayList<String>();
097                while (st.hasMoreTokens()) {
098                    l.add(st.nextToken());
099                }
100                args = l.toArray(new String[l.size()]);
101            }
102            else {
103                List<Element> eArgs = (List<Element>) actionXml.getChildren("arg", ns);
104                args = new String[eArgs.size()];
105                for (int i = 0; i < eArgs.size(); i++) {
106                    args[i] = eArgs.get(i).getTextTrim();
107                }
108            }
109    
110            setSqoopCommand(actionConf, args);
111            return actionConf;
112        }
113    
114        private void setSqoopCommand(Configuration conf, String[] args) {
115            MapReduceMain.setStrings(conf, SQOOP_ARGS, args);
116        }
117    
118        /**
119         * We will gather counters from all executed action Hadoop jobs (e.g. jobs
120         * that moved data, not the launcher itself) and merge them together. There
121         * will be only one job most of the time. The only exception is
122         * import-all-table option that will execute one job per one exported table.
123         *
124         * @param context Action context
125         * @param action Workflow action
126         * @throws ActionExecutorException
127         */
128        @Override
129        public void end(Context context, WorkflowAction action) throws ActionExecutorException {
130            super.end(context, action);
131            JobClient jobClient = null;
132    
133            boolean exception = false;
134            try {
135                if (action.getStatus() == WorkflowAction.Status.OK) {
136                    Element actionXml = XmlUtils.parseXml(action.getConf());
137                    JobConf jobConf = createBaseHadoopConf(context, actionXml);
138                    jobClient = createJobClient(context, jobConf);
139    
140                    // Cumulative counters for all Sqoop mapreduce jobs
141                    Counters counters = null;
142    
143                    // Sqoop do not have to create mapreduce job each time
144                    String externalIds = action.getExternalChildIDs();
145                    if (externalIds != null && !externalIds.trim().isEmpty()) {
146                        String []jobIds = externalIds.split(",");
147    
148                        for(String jobId : jobIds) {
149                            RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
150                            if (runningJob == null) {
151                              throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "SQOOP001",
152                                "Unknown hadoop job [{0}] associated with action [{1}].  Failing this action!", action
153                                .getExternalId(), action.getId());
154                            }
155    
156                            Counters taskCounters = runningJob.getCounters();
157                            if(taskCounters != null) {
158                                if(counters == null) {
159                                  counters = taskCounters;
160                                } else {
161                                  counters.incrAllCounters(taskCounters);
162                                }
163                            } else {
164                              XLog.getLog(getClass()).warn("Could not find Hadoop Counters for job: [{0}]", jobId);
165                            }
166                        }
167                    }
168    
169                    if (counters != null) {
170                        ActionStats stats = new MRStats(counters);
171                        String statsJsonString = stats.toJSON();
172                        context.setVar(MapReduceActionExecutor.HADOOP_COUNTERS, statsJsonString);
173    
174                        // If action stats write property is set to false by user or
175                        // size of stats is greater than the maximum allowed size,
176                        // do not store the action stats
177                        if (Boolean.parseBoolean(evaluateConfigurationProperty(actionXml,
178                                OOZIE_ACTION_EXTERNAL_STATS_WRITE, "true"))
179                                && (statsJsonString.getBytes().length <= getMaxExternalStatsSize())) {
180                            context.setExecutionStats(statsJsonString);
181                            LOG.debug(
182                              "Printing stats for sqoop action as a JSON string : [{0}]", statsJsonString);
183                        }
184                    } else {
185                        context.setVar(MapReduceActionExecutor.HADOOP_COUNTERS, "");
186                        XLog.getLog(getClass()).warn("Can't find any associated Hadoop job counters");
187                    }
188                }
189            }
190            catch (Exception ex) {
191                exception = true;
192                throw convertException(ex);
193            }
194            finally {
195                if (jobClient != null) {
196                    try {
197                        jobClient.close();
198                    }
199                    catch (Exception e) {
200                        if (exception) {
201                            LOG.error("JobClient error: ", e);
202                        }
203                        else {
204                            throw convertException(e);
205                        }
206                    }
207                }
208            }
209        }
210    
211        // Return the value of the specified configuration property
212        private String evaluateConfigurationProperty(Element actionConf, String key, String defaultValue)
213                throws ActionExecutorException {
214            try {
215                if (actionConf != null) {
216                    Namespace ns = actionConf.getNamespace();
217                    Element e = actionConf.getChild("configuration", ns);
218    
219                    if(e != null) {
220                      String strConf = XmlUtils.prettyPrint(e).toString();
221                      XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
222                      return inlineConf.get(key, defaultValue);
223                    }
224                }
225                return defaultValue;
226            }
227            catch (IOException ex) {
228                throw convertException(ex);
229            }
230        }
231    
232        /**
233         * Get the stats and external child IDs
234         *
235         * @param actionFs the FileSystem object
236         * @param runningJob the runningJob
237         * @param action the Workflow action
238         * @param context executor context
239         *
240         */
241        @Override
242        protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context)
243                throws HadoopAccessorException, JDOMException, IOException, URISyntaxException{
244            super.getActionData(actionFs, runningJob, action, context);
245    
246            if (action.getData() != null) {
247                // Load stored Hadoop jobs ids and promote them as external child
248                // ids
249                Properties props = new Properties();
250                props.load(new StringReader(action.getData()));
251                context.setExternalChildIDs((String) props.get(LauncherMain.HADOOP_JOBS));
252            }
253        }
254    
255        @Override
256        protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
257            return true;
258        }
259    
260    
261        /**
262         * Return the sharelib name for the action.
263         *
264         * @return returns <code>sqoop</code>.
265         * @param actionXml
266         */
267        @Override
268        protected String getDefaultShareLibName(Element actionXml) {
269            return "sqoop";
270        }
271    
272    }