001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     * 
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     * 
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.command.wf;
019    
020    import org.apache.hadoop.conf.Configuration;
021    import org.apache.oozie.WorkflowJobBean;
022    import org.apache.oozie.ErrorCode;
023    import org.apache.oozie.service.JPAService;
024    import org.apache.oozie.service.WorkflowStoreService;
025    import org.apache.oozie.service.WorkflowAppService;
026    import org.apache.oozie.service.Services;
027    import org.apache.oozie.service.DagXLogInfoService;
028    import org.apache.oozie.util.InstrumentUtils;
029    import org.apache.oozie.util.LogUtils;
030    import org.apache.oozie.util.XLog;
031    import org.apache.oozie.util.ParamChecker;
032    import org.apache.oozie.util.XConfiguration;
033    import org.apache.oozie.util.XmlUtils;
034    import org.apache.oozie.command.CommandException;
035    import org.apache.oozie.executor.jpa.WorkflowJobInsertJPAExecutor;
036    import org.apache.oozie.store.StoreException;
037    import org.apache.oozie.workflow.WorkflowApp;
038    import org.apache.oozie.workflow.WorkflowException;
039    import org.apache.oozie.workflow.WorkflowInstance;
040    import org.apache.oozie.workflow.WorkflowLib;
041    import org.apache.oozie.util.PropertiesUtils;
042    import org.apache.oozie.client.OozieClient;
043    import org.apache.oozie.client.WorkflowJob;
044    import org.apache.oozie.client.XOozieClient;
045    import org.jdom.Element;
046    import org.jdom.Namespace;
047    
048    import java.util.Date;
049    import java.util.List;
050    import java.util.Map;
051    import java.util.Set;
052    import java.util.HashSet;
053    
054    public abstract class SubmitHttpXCommand extends WorkflowXCommand<String> {
055    
056        protected static final Set<String> MANDATORY_OOZIE_CONFS = new HashSet<String>();
057        protected static final Set<String> OPTIONAL_OOZIE_CONFS = new HashSet<String>();
058    
059        static {
060            MANDATORY_OOZIE_CONFS.add(XOozieClient.JT);
061            MANDATORY_OOZIE_CONFS.add(XOozieClient.NN);
062            MANDATORY_OOZIE_CONFS.add(OozieClient.LIBPATH);
063    
064            OPTIONAL_OOZIE_CONFS.add(XOozieClient.FILES);
065            OPTIONAL_OOZIE_CONFS.add(XOozieClient.ARCHIVES);
066        }
067    
068        private Configuration conf;
069    
070        public SubmitHttpXCommand(String name, String type, Configuration conf) {
071            super(name, type, 1);
072            this.conf = ParamChecker.notNull(conf, "conf");
073        }
074    
075        private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
076        private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>();
077    
078        static {
079            String[] badUserProps = { PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES,
080                    PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB,
081                    PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN,
082                    PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS };
083            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES);
084    
085            String[] badDefaultProps = { PropertiesUtils.HADOOP_USER};
086            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
087            PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
088        }
089    
090        abstract protected Element generateSection(Configuration conf, Namespace ns);
091    
092        abstract protected Namespace getSectionNamespace();
093    
094        abstract protected String getWorkflowName();
095    
096        protected void checkMandatoryConf(Configuration conf) {
097            for (String key : MANDATORY_OOZIE_CONFS) {
098                String value = conf.get(key);
099                if (value == null) {
100                    throw new RuntimeException(key + " is not specified");
101                }
102            }
103        }
104    
105        protected Namespace getWorkflowNamespace() {
106            return Namespace.getNamespace("uri:oozie:workflow:0.2");
107        }
108        /**
109         * Generate workflow xml from conf object
110         *
111         * @param conf the configuration object
112         * @return workflow xml def string representation
113         */
114        protected String getWorkflowXml(Configuration conf) {
115            checkMandatoryConf(conf);
116    
117            Namespace ns = getWorkflowNamespace();
118            Element root = new Element("workflow-app", ns);
119            String name = getWorkflowName();
120            root.setAttribute("name", "oozie-" + name);
121    
122            Element start = new Element("start", ns);
123            String nodeName = name + "1";
124            start.setAttribute("to", nodeName);
125            root.addContent(start);
126    
127            Element action = new Element("action", ns);
128            action.setAttribute("name", nodeName);
129    
130            Element ele = generateSection(conf, getSectionNamespace());
131            action.addContent(ele);
132    
133            Element ok = new Element("ok", ns);
134            ok.setAttribute("to", "end");
135            action.addContent(ok);
136    
137            Element error = new Element("error", ns);
138            error.setAttribute("to", "fail");
139            action.addContent(error);
140    
141            root.addContent(action);
142    
143            Element kill = new Element("kill", ns);
144            kill.setAttribute("name", "fail");
145            Element message = new Element("message", ns);
146            message.addContent(name + " failed, error message[${wf:errorMessage(wf:lastErrorNode())}]");
147            kill.addContent(message);
148            root.addContent(kill);
149    
150            Element end = new Element("end", ns);
151            end.setAttribute("name", "end");
152            root.addContent(end);
153    
154            return XmlUtils.prettyPrint(root).toString();
155        };
156    
157        protected Element generateConfigurationSection(List<String> Dargs, Namespace ns) {
158            Element configuration = new Element("configuration", ns);
159            for (String arg : Dargs) {
160                String name = null, value = null;
161                int pos = arg.indexOf("=");
162                if (pos == -1) { // "-D<name>" or "-D" only
163                    name = arg.substring(2, arg.length());
164                    value = "";
165                }
166                else { // "-D<name>=<value>"
167                    name = arg.substring(2, pos);
168                    value = arg.substring(pos + 1, arg.length());
169                }
170    
171                Element property = new Element("property", ns);
172                Element nameElement = new Element("name", ns);
173                nameElement.addContent(name);
174                property.addContent(nameElement);
175                Element valueElement = new Element("value", ns);
176                valueElement.addContent(value);
177                property.addContent(valueElement);
178                configuration.addContent(property);
179            }
180    
181            return configuration;
182        }
183    
184        /* (non-Javadoc)
185         * @see org.apache.oozie.command.XCommand#execute()
186         */
187        @Override
188        protected String execute() throws CommandException {
189            InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
190            WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
191            try {
192                XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
193                String wfXml = getWorkflowXml(conf);
194                LOG.debug("workflow xml created on the server side is :\n");
195                LOG.debug(wfXml);
196                WorkflowApp app = wps.parseDef(wfXml, conf);
197                XConfiguration protoActionConf = wps.createProtoActionConf(conf, false);
198                WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
199    
200                PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
201    
202                // Resolving all variables in the job properties.
203                // This ensures the Hadoop Configuration semantics is preserved.
204                XConfiguration resolvedVarsConf = new XConfiguration();
205                for (Map.Entry<String, String> entry : conf) {
206                    resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
207                }
208                conf = resolvedVarsConf;
209    
210                WorkflowInstance wfInstance;
211                try {
212                    wfInstance = workflowLib.createInstance(app, conf);
213                }
214                catch (WorkflowException e) {
215                    throw new StoreException(e);
216                }
217    
218                Configuration conf = wfInstance.getConf();
219    
220                WorkflowJobBean workflow = new WorkflowJobBean();
221                workflow.setId(wfInstance.getId());
222                workflow.setAppName(app.getName());
223                workflow.setAppPath(conf.get(OozieClient.APP_PATH));
224                workflow.setConf(XmlUtils.prettyPrint(conf).toString());
225                workflow.setProtoActionConf(protoActionConf.toXmlString());
226                workflow.setCreatedTime(new Date());
227                workflow.setLastModifiedTime(new Date());
228                workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
229                workflow.setStatus(WorkflowJob.Status.PREP);
230                workflow.setRun(0);
231                workflow.setUser(conf.get(OozieClient.USER_NAME));
232                workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
233                workflow.setWorkflowInstance(wfInstance);
234                workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
235    
236                LogUtils.setLogInfo(workflow, logInfo);
237                JPAService jpaService = Services.get().get(JPAService.class);
238                if (jpaService != null) {
239                    jpaService.execute(new WorkflowJobInsertJPAExecutor(workflow));
240                }
241                else {
242                    LOG.error(ErrorCode.E0610);
243                    return null;
244                }
245    
246                return workflow.getId();
247            }
248            catch (WorkflowException ex) {
249                throw new CommandException(ex);
250            }
251            catch (Exception ex) {
252                throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
253            }
254        }
255    
256        static private void addSection(Element X, Namespace ns, String filesStr, String tagName) {
257            if (filesStr != null) {
258                String[] files = filesStr.split(",");
259                for (String f : files) {
260                    Element tagElement = new Element(tagName, ns);
261                    if (f.contains("#")) {
262                        tagElement.addContent(f);
263                    }
264                    else {
265                        String filename = f.substring(f.lastIndexOf("/") + 1, f.length());
266                        if (filename == null || filename.isEmpty()) {
267                            tagElement.addContent(f);
268                        }
269                        else {
270                            tagElement.addContent(f + "#" + filename);
271                        }
272                    }
273                    X.addContent(tagElement);
274                }
275            }
276        }
277    
278        /**
279         * Add file section in X.
280         *
281         * @param parent XML element to be appended
282         * @param conf Configuration object
283         * @param ns XML element namespace
284         */
285        static void addFileSection(Element X, Configuration conf, Namespace ns) {
286            String filesStr = conf.get(XOozieClient.FILES);
287            addSection(X, ns, filesStr, "file");
288        }
289    
290        /**
291         * Add archive section in X.
292         *
293         * @param parent XML element to be appended
294         * @param conf Configuration object
295         * @param ns XML element namespace
296         */
297        static void addArchiveSection(Element X, Configuration conf, Namespace ns) {
298            String archivesStr = conf.get(XOozieClient.ARCHIVES);
299            addSection(X, ns, archivesStr, "archive");
300        }
301    }