001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.coord;
019    
020    import java.net.URI;
021    import java.net.URISyntaxException;
022    
023    import org.apache.hadoop.conf.Configuration;
024    import org.apache.oozie.DagELFunctions;
025    import org.apache.oozie.client.WorkflowJob;
026    import org.apache.oozie.dependency.URIHandler;
027    import org.apache.oozie.service.Services;
028    import org.apache.oozie.service.URIHandlerService;
029    import org.apache.oozie.util.ELEvaluator;
030    import org.apache.oozie.util.HCatURI;
031    import org.apache.oozie.util.XLog;
032    
033    /**
034     * This class implements the EL function for HCat datasets in coordinator
035     */
036    
037    public class HCatELFunctions {
038        private static final Configuration EMPTY_CONF = new Configuration(true);
039    
040        enum EventType {
041            input, output
042        }
043    
044        /* Workflow Parameterization EL functions */
045    
046        /**
047         * Return true if partitions exists or false if not.
048         *
049         * @param uri hcatalog partition uri.
050         * @return <code>true</code> if the uri exists, <code>false</code> if it does not.
051         * @throws Exception
052         */
053        public static boolean hcat_exists(String uri) throws Exception {
054            URI hcatURI = new URI(uri);
055            URIHandlerService uriService = Services.get().get(URIHandlerService.class);
056            URIHandler handler = uriService.getURIHandler(hcatURI);
057            WorkflowJob workflow = DagELFunctions.getWorkflow();
058            String user = workflow.getUser();
059            return handler.exists(hcatURI, EMPTY_CONF, user);
060        }
061    
062        /* Coord EL functions */
063    
064        /**
065         * Echo the same EL function without evaluating anything
066         *
067         * @param dataInName
068         * @return the same EL function
069         */
070        public static String ph1_coord_databaseIn_echo(String dataName) {
071            // Checking if the dataIn is correct?
072            isValidDataEvent(dataName);
073            return echoUnResolved("databaseIn", "'" + dataName + "'");
074        }
075    
076        public static String ph1_coord_databaseOut_echo(String dataName) {
077            // Checking if the dataOut is correct?
078            isValidDataEvent(dataName);
079            return echoUnResolved("databaseOut", "'" + dataName + "'");
080        }
081    
082        public static String ph1_coord_tableIn_echo(String dataName) {
083            // Checking if the dataIn is correct?
084            isValidDataEvent(dataName);
085            return echoUnResolved("tableIn", "'" + dataName + "'");
086        }
087    
088        public static String ph1_coord_tableOut_echo(String dataName) {
089            // Checking if the dataOut is correct?
090            isValidDataEvent(dataName);
091            return echoUnResolved("tableOut", "'" + dataName + "'");
092        }
093    
094        public static String ph1_coord_dataInPartitionFilter_echo(String dataInName, String type) {
095            // Checking if the dataIn/dataOut is correct?
096            isValidDataEvent(dataInName);
097            return echoUnResolved("dataInPartitionFilter", "'" + dataInName + "', '" + type + "'");
098        }
099    
100        public static String ph1_coord_dataInPartitionMin_echo(String dataInName, String partition) {
101            // Checking if the dataIn/dataOut is correct?
102            isValidDataEvent(dataInName);
103            return echoUnResolved("dataInPartitionMin", "'" + dataInName + "', '" + partition + "'");
104        }
105    
106        public static String ph1_coord_dataInPartitionMax_echo(String dataInName, String partition) {
107            // Checking if the dataIn/dataOut is correct?
108            isValidDataEvent(dataInName);
109            return echoUnResolved("dataInPartitionMax", "'" + dataInName + "', '" + partition + "'");
110        }
111    
112        public static String ph1_coord_dataOutPartitions_echo(String dataOutName) {
113            // Checking if the dataIn/dataOut is correct?
114            isValidDataEvent(dataOutName);
115            return echoUnResolved("dataOutPartitions", "'" + dataOutName + "'");
116        }
117    
118        public static String ph1_coord_dataInPartitions_echo(String dataInName, String type) {
119            // Checking if the dataIn/dataOut is correct?
120            isValidDataEvent(dataInName);
121            return echoUnResolved("dataInPartitions", "'" + dataInName + "', '" + type + "'");
122        }
123    
124        public static String ph1_coord_dataOutPartitionValue_echo(String dataOutName, String partition) {
125            // Checking if the dataIn/dataOut is correct?
126            isValidDataEvent(dataOutName);
127            return echoUnResolved("dataOutPartitionValue", "'" + dataOutName + "', '" + partition + "'");
128        }
129    
130        /**
131         * Extract the hcat DB name from the URI-template associate with
132         * 'dataInName'. Caller needs to specify the EL-evaluator level variable
133         * 'oozie.coord.el.dataset.bean' with synchronous dataset object
134         * (SyncCoordDataset)
135         *
136         * @param dataInName
137         * @return DB name
138         */
139        public static String ph3_coord_databaseIn(String dataName) {
140            HCatURI hcatURI = getURIFromResolved(dataName, EventType.input);
141            if (hcatURI != null) {
142                return hcatURI.getDb();
143            }
144            else {
145                return "";
146            }
147        }
148    
149        /**
150         * Extract the hcat DB name from the URI-template associate with
151         * 'dataOutName'. Caller needs to specify the EL-evaluator level variable
152         * 'oozie.coord.el.dataset.bean' with synchronous dataset object
153         * (SyncCoordDataset)
154         *
155         * @param dataOutName
156         * @return DB name
157         */
158        public static String ph3_coord_databaseOut(String dataName) {
159            HCatURI hcatURI = getURIFromResolved(dataName, EventType.output);
160            if (hcatURI != null) {
161                return hcatURI.getDb();
162            }
163            else {
164                return "";
165            }
166        }
167    
168        /**
169         * Extract the hcat Table name from the URI-template associate with
170         * 'dataInName'. Caller needs to specify the EL-evaluator level variable
171         * 'oozie.coord.el.dataset.bean' with synchronous dataset object
172         * (SyncCoordDataset)
173         *
174         * @param dataInName
175         * @return Table name
176         */
177        public static String ph3_coord_tableIn(String dataName) {
178            HCatURI hcatURI = getURIFromResolved(dataName, EventType.input);
179            if (hcatURI != null) {
180                return hcatURI.getTable();
181            }
182            else {
183                return "";
184            }
185        }
186    
187        /**
188         * Extract the hcat Table name from the URI-template associate with
189         * 'dataOutName'. Caller needs to specify the EL-evaluator level variable
190         * 'oozie.coord.el.dataset.bean' with synchronous dataset object
191         * (SyncCoordDataset)
192         *
193         * @param dataOutName
194         * @return Table name
195         */
196        public static String ph3_coord_tableOut(String dataName) {
197            HCatURI hcatURI = getURIFromResolved(dataName, EventType.output);
198            if (hcatURI != null) {
199                return hcatURI.getTable();
200            }
201            else {
202                return "";
203            }
204        }
205    
206        /**
207         * Used to specify the HCat partition filter which is input dependency for workflow job.<p/> Look for two evaluator-level
208         * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the current list of
209         * HCat URIs. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something
210         * unresolved, this function will echo back the original function <p/> otherwise it sends the partition filter.
211         *
212         * @param dataInName : Datain name
213         * @param type : for action type - pig, MR or hive
214         */
215        public static String ph3_coord_dataInPartitionFilter(String dataInName, String type) {
216            ELEvaluator eval = ELEvaluator.getCurrent();
217            String uris = (String) eval.getVariable(".datain." + dataInName);
218            Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
219            if (unresolved != null && unresolved.booleanValue() == true) {
220                return "${coord:dataInPartitionFilter('" + dataInName + "', '" + type + "')}";
221            }
222            return createPartitionFilter(uris, type);
223        }
224    
225        /**
226         * Used to specify the HCat partition's value defining output for workflow job.<p/> Look for two evaluator-level
227         * variables <p/> A) .dataout.<DATAOUT_NAME> B) .dataout.<DATAOUT_NAME>.unresolved <p/> A defines the current list of
228         * HCat URIs. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something
229         * unresolved, this function will echo back the original function <p/> otherwise it sends the partition value.
230         *
231         * @param dataOutName : Dataout name
232         * @param partitionName : Specific partition name whose value is wanted
233         */
234        public static String ph3_coord_dataOutPartitionValue(String dataOutName, String partitionName) {
235            ELEvaluator eval = ELEvaluator.getCurrent();
236            String uri = (String) eval.getVariable(".dataout." + dataOutName);
237            Boolean unresolved = (Boolean) eval.getVariable(".dataout." + dataOutName + ".unresolved");
238            if (unresolved != null && unresolved.booleanValue() == true) {
239                return "${coord:dataOutPartitionValue('" + dataOutName + "', '" + partitionName + "')}";
240            }
241            try {
242                HCatURI hcatUri = new HCatURI(uri);
243                return hcatUri.getPartitionValue(partitionName);
244            }
245            catch(URISyntaxException urie) {
246                XLog.getLog(HCatELFunctions.class).warn("Exception with uriTemplate [{0}]. Reason [{1}]: ", uri, urie);
247                throw new RuntimeException("HCat URI can't be parsed " + urie);
248            }
249        }
250    
251        /**
252         * Used to specify the entire HCat partition defining output for workflow job.<p/> Look for two evaluator-level
253         * variables <p/> A) .dataout.<DATAOUT_NAME> B) .dataout.<DATAOUT_NAME>.unresolved <p/> A defines the data-out
254         * HCat URI. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something
255         * unresolved, this function will echo back the original function <p/> otherwise it sends the partition.
256         *
257         * @param dataOutName : DataOut name
258         */
259        public static String ph3_coord_dataOutPartitions(String dataOutName) {
260            ELEvaluator eval = ELEvaluator.getCurrent();
261            String uri = (String) eval.getVariable(".dataout." + dataOutName);
262            Boolean unresolved = (Boolean) eval.getVariable(".dataout." + dataOutName + ".unresolved");
263            if (unresolved != null && unresolved.booleanValue() == true) {
264                return "${coord:dataOutPartitions('" + dataOutName + "')}";
265            }
266            try {
267                return new HCatURI(uri).toPartitionString();
268            }
269            catch (URISyntaxException e) {
270                throw new RuntimeException("Parsing exception for HCatURI " + uri + ". details: " + e);
271            }
272        }
273    
274        /**
275         * Used to specify the entire HCat partition defining input for workflow job. <p/> Look for two evaluator-level
276         * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the data-in HCat URI.
277         * <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something unresolved,
278         * this function will echo back the original function <p/> otherwise it sends the partition.
279         *
280         * @param dataInName : DataIn name
281         * @param type : for action type: hive-export
282         */
283        public static String ph3_coord_dataInPartitions(String dataInName, String type) {
284            ELEvaluator eval = ELEvaluator.getCurrent();
285            String uri = (String) eval.getVariable(".datain." + dataInName);
286            Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
287            if (unresolved != null && unresolved.booleanValue() == true) {
288                return "${coord:dataInPartitions('" + dataInName + "', '" + type + "')}";
289            }
290            String partitionValue = null;
291            if (uri != null) {
292                if (type.equals("hive-export")) {
293                    String[] uriList = uri.split(CoordELFunctions.DIR_SEPARATOR);
294                    if (uriList.length > 1) {
295                        throw new RuntimeException("Multiple partitions not supported for hive-export type. Dataset name: "
296                            + dataInName + " URI: " + uri);
297                    }
298                    try {
299                        partitionValue = new HCatURI(uri).toPartitionValueString(type);
300                    }
301                    catch (URISyntaxException e) {
302                        throw new RuntimeException("Parsing exception for HCatURI " + uri, e);
303                    }
304                } else {
305                      throw new RuntimeException("Unsupported type: " + type + " dataset name: " + dataInName);
306                }
307            }
308            else {
309                XLog.getLog(HCatELFunctions.class).warn("URI is null");
310                return null;
311            }
312            return partitionValue;
313        }
314    
315        /**
316         * Used to specify the MAXIMUM value of an HCat partition which is input dependency for workflow job.<p/> Look for two evaluator-level
317         * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the current list of
318         * HCat URIs. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something
319         * unresolved, this function will echo back the original function <p/> otherwise it sends the max partition value.
320         *
321         * @param dataInName : Datain name
322         * @param partitionName : Specific partition name whose MAX value is wanted
323         */
324        public static String ph3_coord_dataInPartitionMin(String dataInName, String partitionName) {
325            ELEvaluator eval = ELEvaluator.getCurrent();
326            String uris = (String) eval.getVariable(".datain." + dataInName);
327            Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
328            if (unresolved != null && unresolved.booleanValue() == true) {
329                return "${coord:dataInPartitionMin('" + dataInName + "', '" + partitionName + "')}";
330            }
331            String minPartition = null;
332            if (uris != null) {
333                String[] uriList = uris.split(CoordELFunctions.DIR_SEPARATOR);
334                // get the partition values list and find minimum
335                try {
336                    // initialize minValue with first partition value
337                    minPartition = new HCatURI(uriList[0]).getPartitionValue(partitionName);
338                    if (minPartition == null || minPartition.isEmpty()) {
339                        throw new RuntimeException("No value in data-in uri for partition key: " + partitionName);
340                    }
341                    for (int i = 1; i < uriList.length; i++) {
342                            String value = new HCatURI(uriList[i]).getPartitionValue(partitionName);
343                            if(value.compareTo(minPartition) < 0) { //sticking to string comparison since some numerical date
344                                                                    //values can also contain letters e.g. 20120101T0300Z (UTC)
345                                minPartition = value;
346                            }
347                    }
348                }
349                catch(URISyntaxException urie) {
350                    throw new RuntimeException("HCat URI can't be parsed " + urie);
351                }
352            }
353            else {
354                XLog.getLog(HCatELFunctions.class).warn("URI is null");
355                return null;
356            }
357            return minPartition;
358        }
359    
360        /**
361         * Used to specify the MINIMUM value of an HCat partition which is input dependency for workflow job.<p/> Look for two evaluator-level
362         * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the current list of
363         * HCat URIs. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something
364         * unresolved, this function will echo back the original function <p/> otherwise it sends the min partition value.
365         *
366         * @param dataInName : Datain name
367         * @param partitionName : Specific partition name whose MIN value is wanted
368         */
369        public static String ph3_coord_dataInPartitionMax(String dataInName, String partitionName) {
370            ELEvaluator eval = ELEvaluator.getCurrent();
371            String uris = (String) eval.getVariable(".datain." + dataInName);
372            Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
373            if (unresolved != null && unresolved.booleanValue() == true) {
374                return "${coord:dataInPartitionMin('" + dataInName + "', '" + partitionName + "')}";
375            }
376            String maxPartition = null;
377            if (uris != null) {
378                String[] uriList = uris.split(CoordELFunctions.DIR_SEPARATOR);
379                // get the partition values list and find minimum
380                try {
381                    // initialize minValue with first partition value
382                    maxPartition = new HCatURI(uriList[0]).getPartitionValue(partitionName);
383                    if (maxPartition == null || maxPartition.isEmpty()) {
384                        throw new RuntimeException("No value in data-in uri for partition key: " + partitionName);
385                    }
386                    for(int i = 1; i < uriList.length; i++) {
387                            String value = new HCatURI(uriList[i]).getPartitionValue(partitionName);
388                            if(value.compareTo(maxPartition) > 0) {
389                                maxPartition = value;
390                            }
391                    }
392                }
393                catch(URISyntaxException urie) {
394                    throw new RuntimeException("HCat URI can't be parsed " + urie);
395                }
396            }
397            else {
398                XLog.getLog(HCatELFunctions.class).warn("URI is null");
399                return null;
400            }
401            return maxPartition;
402        }
403    
404        private static String createPartitionFilter(String uris, String type) {
405            String[] uriList = uris.split(CoordELFunctions.DIR_SEPARATOR);
406            StringBuilder filter = new StringBuilder("");
407            if (uriList.length > 0) {
408                for (String uri : uriList) {
409                    if (filter.length() > 0) {
410                        filter.append(" OR ");
411                    }
412                    try {
413                        filter.append(new HCatURI(uri).toPartitionFilter(type));
414                    }
415                    catch (URISyntaxException e) {
416                        throw new RuntimeException("Parsing exception for HCatURI " + uri + ". details: " + e);
417                    }
418                }
419            }
420            return filter.toString();
421        }
422    
423        private static HCatURI getURIFromResolved(String dataInName, EventType type) {
424            final XLog LOG = XLog.getLog(HCatELFunctions.class);
425            StringBuilder uriTemplate = new StringBuilder();
426            ELEvaluator eval = ELEvaluator.getCurrent();
427            String uris;
428            if(type == EventType.input) {
429                uris = (String) eval.getVariable(".datain." + dataInName);
430            }
431            else { //type=output
432                uris = (String) eval.getVariable(".dataout." + dataInName);
433            }
434            if (uris != null) {
435                String[] uri = uris.split(CoordELFunctions.DIR_SEPARATOR, -1);
436                uriTemplate.append(uri[0]);
437            }
438            else {
439                LOG.warn("URI is NULL");
440                return null;
441            }
442            LOG.info("uriTemplate [{0}] ", uriTemplate);
443            HCatURI hcatURI;
444            try {
445                hcatURI = new HCatURI(uriTemplate.toString());
446            }
447            catch (URISyntaxException e) {
448                LOG.info("uriTemplate [{0}]. Reason [{1}]: ", uriTemplate, e);
449                throw new RuntimeException("HCat URI can't be parsed " + e);
450            }
451            return hcatURI;
452        }
453    
454        private static boolean isValidDataEvent(String dataInName) {
455            ELEvaluator eval = ELEvaluator.getCurrent();
456            String val = (String) eval.getVariable("oozie.dataname." + dataInName);
457            if (val == null || (val.equals("data-in") == false && val.equals("data-out") == false)) {
458                XLog.getLog(HCatELFunctions.class).error("dataset name " + dataInName + " is not valid. val :" + val);
459                throw new RuntimeException("data set name " + dataInName + " is not valid");
460            }
461            return true;
462        }
463    
464        private static String echoUnResolved(String functionName, String n) {
465            return echoUnResolvedPre(functionName, n, "coord:");
466        }
467    
468        private static String echoUnResolvedPre(String functionName, String n, String prefix) {
469            ELEvaluator eval = ELEvaluator.getCurrent();
470            eval.setVariable(".wrap", "true");
471            return prefix + functionName + "(" + n + ")"; // Unresolved
472        }
473    
474    }