001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.action.hadoop;
019
020 import java.io.IOException;
021 import java.io.StringReader;
022 import java.net.URISyntaxException;
023 import java.util.ArrayList;
024 import java.util.List;
025 import java.util.Properties;
026 import java.util.StringTokenizer;
027
028 import org.apache.hadoop.conf.Configuration;
029 import org.apache.hadoop.fs.FileSystem;
030 import org.apache.hadoop.fs.Path;
031 import org.apache.hadoop.mapred.Counters;
032 import org.apache.hadoop.mapred.JobClient;
033 import org.apache.hadoop.mapred.JobConf;
034 import org.apache.hadoop.mapred.JobID;
035 import org.apache.hadoop.mapred.RunningJob;
036 import org.apache.oozie.action.ActionExecutorException;
037 import org.apache.oozie.client.WorkflowAction;
038 import org.apache.oozie.service.HadoopAccessorException;
039 import org.apache.oozie.util.XConfiguration;
040 import org.apache.oozie.util.XmlUtils;
041 import org.apache.oozie.util.XLog;
042 import org.jdom.Element;
043 import org.jdom.JDOMException;
044 import org.jdom.Namespace;
045
046 public class SqoopActionExecutor extends JavaActionExecutor {
047
048 public static final String OOZIE_ACTION_EXTERNAL_STATS_WRITE = "oozie.action.external.stats.write";
049 private static final String SQOOP_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.SqoopMain";
050 static final String SQOOP_ARGS = "oozie.sqoop.args";
051
052 public SqoopActionExecutor() {
053 super("sqoop");
054 }
055
056 @Override
057 public List<Class> getLauncherClasses() {
058 List<Class> classes = new ArrayList<Class>();
059 try {
060 classes.add(Class.forName(SQOOP_MAIN_CLASS_NAME));
061 }
062 catch (ClassNotFoundException e) {
063 throw new RuntimeException("Class not found", e);
064 }
065 return classes;
066 }
067
068 @Override
069 protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
070 return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SQOOP_MAIN_CLASS_NAME);
071 }
072
073 @Override
074 @SuppressWarnings("unchecked")
075 Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
076 throws ActionExecutorException {
077 super.setupActionConf(actionConf, context, actionXml, appPath);
078 Namespace ns = actionXml.getNamespace();
079
080 try {
081 Element e = actionXml.getChild("configuration", ns);
082 if (e != null) {
083 String strConf = XmlUtils.prettyPrint(e).toString();
084 XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
085 checkForDisallowedProps(inlineConf, "inline configuration");
086 XConfiguration.copy(inlineConf, actionConf);
087 }
088 } catch (IOException ex) {
089 throw convertException(ex);
090 }
091
092 String[] args;
093 if (actionXml.getChild("command", ns) != null) {
094 String command = actionXml.getChild("command", ns).getTextTrim();
095 StringTokenizer st = new StringTokenizer(command, " ");
096 List<String> l = new ArrayList<String>();
097 while (st.hasMoreTokens()) {
098 l.add(st.nextToken());
099 }
100 args = l.toArray(new String[l.size()]);
101 }
102 else {
103 List<Element> eArgs = (List<Element>) actionXml.getChildren("arg", ns);
104 args = new String[eArgs.size()];
105 for (int i = 0; i < eArgs.size(); i++) {
106 args[i] = eArgs.get(i).getTextTrim();
107 }
108 }
109
110 setSqoopCommand(actionConf, args);
111 return actionConf;
112 }
113
114 private void setSqoopCommand(Configuration conf, String[] args) {
115 MapReduceMain.setStrings(conf, SQOOP_ARGS, args);
116 }
117
118 /**
119 * We will gather counters from all executed action Hadoop jobs (e.g. jobs
120 * that moved data, not the launcher itself) and merge them together. There
121 * will be only one job most of the time. The only exception is
122 * import-all-table option that will execute one job per one exported table.
123 *
124 * @param context Action context
125 * @param action Workflow action
126 * @throws ActionExecutorException
127 */
128 @Override
129 public void end(Context context, WorkflowAction action) throws ActionExecutorException {
130 super.end(context, action);
131 JobClient jobClient = null;
132
133 boolean exception = false;
134 try {
135 if (action.getStatus() == WorkflowAction.Status.OK) {
136 Element actionXml = XmlUtils.parseXml(action.getConf());
137 JobConf jobConf = createBaseHadoopConf(context, actionXml);
138 jobClient = createJobClient(context, jobConf);
139
140 // Cumulative counters for all Sqoop mapreduce jobs
141 Counters counters = null;
142
143 // Sqoop do not have to create mapreduce job each time
144 String externalIds = action.getExternalChildIDs();
145 if (externalIds != null && !externalIds.trim().isEmpty()) {
146 String []jobIds = externalIds.split(",");
147
148 for(String jobId : jobIds) {
149 RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
150 if (runningJob == null) {
151 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "SQOOP001",
152 "Unknown hadoop job [{0}] associated with action [{1}]. Failing this action!", action
153 .getExternalId(), action.getId());
154 }
155
156 Counters taskCounters = runningJob.getCounters();
157 if(taskCounters != null) {
158 if(counters == null) {
159 counters = taskCounters;
160 } else {
161 counters.incrAllCounters(taskCounters);
162 }
163 } else {
164 XLog.getLog(getClass()).warn("Could not find Hadoop Counters for job: [{0}]", jobId);
165 }
166 }
167 }
168
169 if (counters != null) {
170 ActionStats stats = new MRStats(counters);
171 String statsJsonString = stats.toJSON();
172 context.setVar(MapReduceActionExecutor.HADOOP_COUNTERS, statsJsonString);
173
174 // If action stats write property is set to false by user or
175 // size of stats is greater than the maximum allowed size,
176 // do not store the action stats
177 if (Boolean.parseBoolean(evaluateConfigurationProperty(actionXml,
178 OOZIE_ACTION_EXTERNAL_STATS_WRITE, "true"))
179 && (statsJsonString.getBytes().length <= getMaxExternalStatsSize())) {
180 context.setExecutionStats(statsJsonString);
181 LOG.debug(
182 "Printing stats for sqoop action as a JSON string : [{0}]", statsJsonString);
183 }
184 } else {
185 context.setVar(MapReduceActionExecutor.HADOOP_COUNTERS, "");
186 XLog.getLog(getClass()).warn("Can't find any associated Hadoop job counters");
187 }
188 }
189 }
190 catch (Exception ex) {
191 exception = true;
192 throw convertException(ex);
193 }
194 finally {
195 if (jobClient != null) {
196 try {
197 jobClient.close();
198 }
199 catch (Exception e) {
200 if (exception) {
201 LOG.error("JobClient error: ", e);
202 }
203 else {
204 throw convertException(e);
205 }
206 }
207 }
208 }
209 }
210
211 // Return the value of the specified configuration property
212 private String evaluateConfigurationProperty(Element actionConf, String key, String defaultValue)
213 throws ActionExecutorException {
214 try {
215 if (actionConf != null) {
216 Namespace ns = actionConf.getNamespace();
217 Element e = actionConf.getChild("configuration", ns);
218
219 if(e != null) {
220 String strConf = XmlUtils.prettyPrint(e).toString();
221 XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
222 return inlineConf.get(key, defaultValue);
223 }
224 }
225 return defaultValue;
226 }
227 catch (IOException ex) {
228 throw convertException(ex);
229 }
230 }
231
232 /**
233 * Get the stats and external child IDs
234 *
235 * @param actionFs the FileSystem object
236 * @param runningJob the runningJob
237 * @param action the Workflow action
238 * @param context executor context
239 *
240 */
241 @Override
242 protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context)
243 throws HadoopAccessorException, JDOMException, IOException, URISyntaxException{
244 super.getActionData(actionFs, runningJob, action, context);
245
246 if (action.getData() != null) {
247 // Load stored Hadoop jobs ids and promote them as external child
248 // ids
249 Properties props = new Properties();
250 props.load(new StringReader(action.getData()));
251 context.setExternalChildIDs((String) props.get(LauncherMain.HADOOP_JOBS));
252 }
253 }
254
255 @Override
256 protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
257 return true;
258 }
259
260
261 /**
262 * Return the sharelib name for the action.
263 *
264 * @return returns <code>sqoop</code>.
265 * @param actionXml
266 */
267 @Override
268 protected String getDefaultShareLibName(Element actionXml) {
269 return "sqoop";
270 }
271
272 }