001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import org.apache.hadoop.conf.Configuration;
021    import org.apache.hadoop.fs.FileStatus;
022    import org.apache.hadoop.fs.FileSystem;
023    import org.apache.hadoop.fs.Path;
024    import org.apache.hadoop.fs.PathFilter;
025    import org.apache.hadoop.mapred.JobConf;
026    import org.apache.oozie.client.OozieClient;
027    import org.apache.oozie.workflow.WorkflowApp;
028    import org.apache.oozie.workflow.WorkflowException;
029    import org.apache.oozie.util.IOUtils;
030    import org.apache.oozie.util.XConfiguration;
031    import org.apache.oozie.util.XLog;
032    import org.apache.oozie.ErrorCode;
033    
034    import java.io.IOException;
035    import java.io.InputStreamReader;
036    import java.io.Reader;
037    import java.io.StringWriter;
038    import java.net.URI;
039    import java.net.URISyntaxException;
040    import java.util.ArrayList;
041    import java.util.Arrays;
042    import java.util.Collection;
043    import java.util.LinkedHashSet;
044    import java.util.List;
045    import java.util.Map;
046    import java.util.Set;
047    
048    /**
049     * Service that provides application workflow definition reading from the path and creation of the proto configuration.
050     */
051    public abstract class WorkflowAppService implements Service {
052    
053        public static final String CONF_PREFIX = Service.CONF_PREFIX + "WorkflowAppService.";
054    
055        public static final String SYSTEM_LIB_PATH = CONF_PREFIX + "system.libpath";
056    
057        public static final String APP_LIB_PATH_LIST = "oozie.wf.application.lib";
058    
059        public static final String HADOOP_USER = "user.name";
060    
061        public static final String CONFG_MAX_WF_LENGTH = CONF_PREFIX + "WorkflowDefinitionMaxLength";
062    
063        public static final String OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE = "oozie.subworkflow.classpath.inheritance";
064    
065        public static final String OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE = "oozie.wf.subworkflow.classpath.inheritance";
066    
067        private Path systemLibPath;
068        private long maxWFLength;
069        private boolean oozieSubWfCPInheritance;
070    
071        /**
072         * Initialize the workflow application service.
073         *
074         * @param services services instance.
075         */
076        public void init(Services services) {
077            Configuration conf = services.getConf();
078    
079            String path = conf.get(SYSTEM_LIB_PATH, " ");
080            if (path.trim().length() > 0) {
081                systemLibPath = new Path(path.trim());
082            }
083    
084            maxWFLength = conf.getInt(CONFG_MAX_WF_LENGTH, 100000);
085    
086            oozieSubWfCPInheritance = conf.getBoolean(OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE, false);
087        }
088    
089        /**
090         * Destroy the workflow application service.
091         */
092        public void destroy() {
093        }
094    
095        /**
096         * Return the public interface for workflow application service.
097         *
098         * @return {@link WorkflowAppService}.
099         */
100        public Class<? extends Service> getInterface() {
101            return WorkflowAppService.class;
102        }
103    
104        /**
105         * Read workflow definition.
106         *
107         *
108         * @param appPath application path.
109         * @param user user name.
110         * @return workflow definition.
111         * @throws WorkflowException thrown if the definition could not be read.
112         */
113        protected String readDefinition(String appPath, String user, Configuration conf)
114                throws WorkflowException {
115            try {
116                URI uri = new URI(appPath);
117                HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
118                JobConf jobConf = has.createJobConf(uri.getAuthority());
119                FileSystem fs = has.createFileSystem(user, uri, jobConf);
120    
121                // app path could be a directory
122                Path path = new Path(uri.getPath());
123                if (!fs.isFile(path)) {
124                    path = new Path(path, "workflow.xml");
125                }
126    
127                FileStatus fsStatus = fs.getFileStatus(path);
128                if (fsStatus.getLen() > this.maxWFLength) {
129                    throw new WorkflowException(ErrorCode.E0736, fsStatus.getLen(), this.maxWFLength);
130                }
131    
132                Reader reader = new InputStreamReader(fs.open(path));
133                StringWriter writer = new StringWriter();
134                IOUtils.copyCharStream(reader, writer);
135                return writer.toString();
136    
137            }
138            catch (WorkflowException wfe) {
139                throw wfe;
140            }
141            catch (IOException ex) {
142                throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex);
143            }
144            catch (URISyntaxException ex) {
145                throw new WorkflowException(ErrorCode.E0711, appPath, ex.getMessage(), ex);
146            }
147            catch (HadoopAccessorException ex) {
148                throw new WorkflowException(ex);
149            }
150            catch (Exception ex) {
151                throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex);
152            }
153        }
154        /**
155         * Create proto configuration. <p/> The proto configuration includes the user,group and the paths which need to be
156         * added to distributed cache. These paths include .jar,.so and the resource file paths.
157         *
158         * @param jobConf job configuration.
159         * @param isWorkflowJob indicates if the job is a workflow job or not.
160         * @return proto configuration.
161         * @throws WorkflowException thrown if the proto action configuration could not be created.
162         */
163        public XConfiguration createProtoActionConf(Configuration jobConf, boolean isWorkflowJob)
164                throws WorkflowException {
165            try {
166                HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
167                URI uri = new URI(jobConf.get(OozieClient.APP_PATH));
168    
169                Configuration conf = has.createJobConf(uri.getAuthority());
170                XConfiguration protoConf = new XConfiguration();
171    
172    
173                String user = jobConf.get(OozieClient.USER_NAME);
174                conf.set(OozieClient.USER_NAME, user);
175                protoConf.set(OozieClient.USER_NAME, user);
176    
177                FileSystem fs = has.createFileSystem(user, uri, conf);
178    
179                Path appPath = new Path(uri);
180                XLog.getLog(getClass()).debug("jobConf.libPath = " + jobConf.get(OozieClient.LIBPATH));
181                XLog.getLog(getClass()).debug("jobConf.appPath = " + appPath);
182    
183                Collection<String> filePaths;
184                if (isWorkflowJob) {
185                    // app path could be a directory
186                    Path path = new Path(uri.getPath());
187                    if (!fs.isFile(path)) {
188                        filePaths = getLibFiles(fs, new Path(appPath + "/lib"));
189                    } else {
190                        filePaths = getLibFiles(fs, new Path(appPath.getParent(), "lib"));
191                    }
192                }
193                else {
194                    filePaths = new LinkedHashSet<String>();
195                }
196    
197                String[] libPaths = jobConf.getStrings(OozieClient.LIBPATH);
198                if (libPaths != null && libPaths.length > 0) {
199                    for (int i = 0; i < libPaths.length; i++) {
200                        if (libPaths[i].trim().length() > 0) {
201                            Path libPath = new Path(libPaths[i].trim());
202                            Collection<String> libFilePaths = getLibFiles(fs, libPath);
203                            filePaths.addAll(libFilePaths);
204                        }
205                    }
206                }
207    
208                // Check if a subworkflow should inherit the libs from the parent WF
209                // OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE has priority over OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE from oozie-site
210                // If OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE isn't specified, we use OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE
211                if (jobConf.getBoolean(OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE, oozieSubWfCPInheritance)) {
212                    // Keep any libs from a parent workflow that might already be in APP_LIB_PATH_LIST and also remove duplicates
213                    String[] parentFilePaths = jobConf.getStrings(APP_LIB_PATH_LIST);
214                    if (parentFilePaths != null && parentFilePaths.length > 0) {
215                        String[] filePathsNames = filePaths.toArray(new String[filePaths.size()]);
216                        for (int i = 0; i < filePathsNames.length; i++) {
217                            Path p = new Path(filePathsNames[i]);
218                            filePathsNames[i] = p.getName();
219                        }
220                        Arrays.sort(filePathsNames);
221                        List<String> nonDuplicateParentFilePaths = new ArrayList<String>();
222                        for (String parentFilePath : parentFilePaths) {
223                            Path p = new Path(parentFilePath);
224                            if (Arrays.binarySearch(filePathsNames, p.getName()) < 0) {
225                                nonDuplicateParentFilePaths.add(parentFilePath);
226                            }
227                        }
228                        filePaths.addAll(nonDuplicateParentFilePaths);
229                    }
230                }
231    
232                protoConf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()]));
233    
234                //Add all properties start with 'oozie.'
235                for (Map.Entry<String, String> entry : jobConf) {
236                    if (entry.getKey().startsWith("oozie.")) {
237                        String name = entry.getKey();
238                        String value = entry.getValue();
239                        // if property already exists, should not overwrite
240                        if(protoConf.get(name) == null) {
241                            protoConf.set(name, value);
242                        }
243                    }
244                }
245                return protoConf;
246            }
247            catch (IOException ex) {
248                throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex);
249            }
250            catch (URISyntaxException ex) {
251                throw new WorkflowException(ErrorCode.E0711, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex);
252            }
253            catch (HadoopAccessorException ex) {
254                throw new WorkflowException(ex);
255            }
256            catch (Exception ex) {
257                throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH),
258                                            ex.getMessage(), ex);
259            }
260        }
261    
262        /**
263         * Parse workflow definition.
264         *
265         * @param jobConf
266         * @return
267         * @throws WorkflowException
268         */
269        public abstract WorkflowApp parseDef(Configuration jobConf) throws WorkflowException;
270    
271        /**
272         * Parse workflow definition along with config-default.xml config
273         *
274         * @param jobConf job configuration
275         * @param configDefault config from config-default.xml
276         * @return workflow application thrown if the workflow application could not
277         *         be parsed
278         * @throws WorkflowException
279         */
280        public abstract WorkflowApp parseDef(Configuration jobConf, Configuration configDefault) throws WorkflowException;
281    
282        /**
283         * Parse workflow definition.
284         * @param wfXml workflow.
285         * @param jobConf job configuration
286         * @return workflow application.
287         * @throws WorkflowException thrown if the workflow application could not be parsed.
288         */
289        public abstract WorkflowApp parseDef(String wfXml, Configuration jobConf) throws WorkflowException;
290    
291        /**
292         * Get all library paths.
293         *
294         * @param fs file system object.
295         * @param libPath hdfs library path.
296         * @return list of paths.
297         * @throws IOException thrown if the lib paths could not be obtained.
298         */
299        private Collection<String> getLibFiles(FileSystem fs, Path libPath) throws IOException {
300            Set<String> libPaths = new LinkedHashSet<String>();
301            if (fs.exists(libPath)) {
302                FileStatus[] files = fs.listStatus(libPath, new NoPathFilter());
303    
304                for (FileStatus file : files) {
305                    libPaths.add(file.getPath().toUri().toString());
306                }
307            }
308            else {
309                XLog.getLog(getClass()).warn("libpath [{0}] does not exist", libPath);
310            }
311            return libPaths;
312        }
313    
314        /*
315         * Filter class doing no filtering.
316         * We dont need define this class, but seems fs.listStatus() is not working properly without this.
317         * So providing this dummy no filtering Filter class.
318         */
319        private class NoPathFilter implements PathFilter {
320            @Override
321            public boolean accept(Path path) {
322                return true;
323            }
324        }
325    
326        /**
327         * Returns Oozie system libpath.
328         *
329         * @return Oozie system libpath (sharelib) in HDFS if present, otherwise it returns <code>NULL</code>.
330         */
331        public Path getSystemLibPath() {
332            return systemLibPath;
333        }
334    }