001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.dependency;
019    
020    import java.io.IOException;
021    import java.net.URI;
022    import java.net.URISyntaxException;
023    import java.util.Arrays;
024    import java.util.HashMap;
025    import java.util.HashSet;
026    import java.util.List;
027    import java.util.Map;
028    import java.util.Set;
029    
030    import org.apache.hadoop.conf.Configuration;
031    import org.apache.hadoop.hive.conf.HiveConf;
032    import org.apache.hadoop.security.UserGroupInformation;
033    import org.apache.hive.hcatalog.api.ConnectionFailureException;
034    import org.apache.hive.hcatalog.api.HCatClient;
035    import org.apache.hive.hcatalog.api.HCatPartition;
036    import org.apache.hive.hcatalog.common.HCatException;
037    import org.apache.oozie.ErrorCode;
038    import org.apache.oozie.action.hadoop.HCatLauncherURIHandler;
039    import org.apache.oozie.action.hadoop.LauncherURIHandler;
040    import org.apache.oozie.dependency.hcat.HCatMessageHandler;
041    import org.apache.oozie.service.HCatAccessorException;
042    import org.apache.oozie.service.HCatAccessorService;
043    import org.apache.oozie.service.PartitionDependencyManagerService;
044    import org.apache.oozie.service.Services;
045    import org.apache.oozie.service.URIHandlerService;
046    import org.apache.oozie.util.HCatURI;
047    import org.apache.oozie.util.XLog;
048    
049    public class HCatURIHandler implements URIHandler {
050    
051        private Set<String> supportedSchemes;
052        private Map<String, DependencyType> dependencyTypes;
053        private List<Class<?>> classesToShip;
054    
055        @Override
056        public void init(Configuration conf) {
057            dependencyTypes = new HashMap<String, DependencyType>();
058            supportedSchemes = new HashSet<String>();
059            String[] schemes = conf.getStrings(URIHandlerService.URI_HANDLER_SUPPORTED_SCHEMES_PREFIX
060                    + this.getClass().getSimpleName() + URIHandlerService.URI_HANDLER_SUPPORTED_SCHEMES_SUFFIX, "hcat");
061            supportedSchemes.addAll(Arrays.asList(schemes));
062            classesToShip = new HCatLauncherURIHandler().getClassesForLauncher();
063        }
064    
065        @Override
066        public Set<String> getSupportedSchemes() {
067            return supportedSchemes;
068        }
069    
070        @Override
071        public Class<? extends LauncherURIHandler> getLauncherURIHandlerClass() {
072            return HCatLauncherURIHandler.class;
073        }
074    
075        @Override
076        public List<Class<?>> getClassesForLauncher() {
077            return classesToShip;
078        }
079    
080        @Override
081        public DependencyType getDependencyType(URI uri) throws URIHandlerException {
082            DependencyType depType = DependencyType.PULL;
083            // Not initializing in constructor as this will be part of oozie.services.ext
084            // and will be initialized after URIHandlerService
085            HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
086            if (hcatService != null) {
087                depType = dependencyTypes.get(uri.getAuthority());
088                if (depType == null) {
089                    depType = hcatService.isKnownPublisher(uri) ? DependencyType.PUSH : DependencyType.PULL;
090                    dependencyTypes.put(uri.getAuthority(), depType);
091                }
092            }
093            return depType;
094        }
095    
096        @Override
097        public void registerForNotification(URI uri, Configuration conf, String user, String actionID)
098                throws URIHandlerException {
099            HCatURI hcatURI;
100            try {
101                hcatURI = new HCatURI(uri);
102            }
103            catch (URISyntaxException e) {
104                throw new URIHandlerException(ErrorCode.E0906, uri, e);
105            }
106            HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
107            if (!hcatService.isRegisteredForNotification(hcatURI)) {
108                HCatClient client = getHCatClient(uri, conf, user);
109                try {
110                    String topic = client.getMessageBusTopicName(hcatURI.getDb(), hcatURI.getTable());
111                    if (topic == null) {
112                        return;
113                    }
114                    hcatService.registerForNotification(hcatURI, topic, new HCatMessageHandler(uri.getAuthority()));
115                }
116                catch (HCatException e) {
117                    throw new HCatAccessorException(ErrorCode.E1501, e);
118                }
119                finally {
120                    closeQuietly(client, true);
121                }
122            }
123            PartitionDependencyManagerService pdmService = Services.get().get(PartitionDependencyManagerService.class);
124            pdmService.addMissingDependency(hcatURI, actionID);
125        }
126    
127        @Override
128        public boolean unregisterFromNotification(URI uri, String actionID) {
129            HCatURI hcatURI;
130            try {
131                hcatURI = new HCatURI(uri);
132            }
133            catch (URISyntaxException e) {
134                throw new RuntimeException(e); // Unexpected at this point
135            }
136            PartitionDependencyManagerService pdmService = Services.get().get(PartitionDependencyManagerService.class);
137            return pdmService.removeMissingDependency(hcatURI, actionID);
138        }
139    
140        @Override
141        public Context getContext(URI uri, Configuration conf, String user) throws URIHandlerException {
142            HCatClient client = getHCatClient(uri, conf, user);
143            return new HCatContext(conf, user, client);
144        }
145    
146        @Override
147        public boolean exists(URI uri, Context context) throws URIHandlerException {
148            HCatClient client = ((HCatContext) context).getHCatClient();
149            return exists(uri, client, false);
150        }
151    
152        @Override
153        public boolean exists(URI uri, Configuration conf, String user) throws URIHandlerException {
154            HCatClient client = getHCatClient(uri, conf, user);
155            return exists(uri, client, true);
156        }
157    
158        @Override
159        public String getURIWithDoneFlag(String uri, String doneFlag) throws URIHandlerException {
160            return uri;
161        }
162    
163        @Override
164        public void validate(String uri) throws URIHandlerException {
165            try {
166                new HCatURI(uri); // will fail if uri syntax is incorrect
167            }
168            catch (URISyntaxException e) {
169                throw new URIHandlerException(ErrorCode.E0906, uri, e);
170            }
171    
172        }
173    
174        @Override
175        public void destroy() {
176    
177        }
178    
179        private HCatClient getHCatClient(URI uri, Configuration conf, String user) throws HCatAccessorException {
180            HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
181            if (hcatService.getHCatConf() != null) {
182                conf = hcatService.getHCatConf();
183            }
184            final HiveConf hiveConf = new HiveConf(conf, this.getClass());
185            String serverURI = getMetastoreConnectURI(uri);
186            if (!serverURI.equals("")) {
187                hiveConf.set("hive.metastore.local", "false");
188            }
189            hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, serverURI);
190            try {
191                XLog.getLog(HCatURIHandler.class).info(
192                        "Creating HCatClient for user [{0}] login_user [{1}] and server [{2}] ", user,
193                        UserGroupInformation.getLoginUser(), serverURI);
194    
195                // HiveMetastoreClient (hive 0.9) currently does not work if UGI has doAs
196                // We are good to connect as the oozie user since listPartitions does not require
197                // authorization
198                /*
199                UserGroupInformation ugi = ugiService.getProxyUser(user);
200                return ugi.doAs(new PrivilegedExceptionAction<HCatClient>() {
201                    public HCatClient run() throws Exception {
202                        return HCatClient.create(hiveConf);
203                    }
204                });
205                */
206    
207                return HCatClient.create(hiveConf);
208            }
209            catch (HCatException e) {
210                throw new HCatAccessorException(ErrorCode.E1501, e);
211            }
212            catch (IOException e) {
213                throw new HCatAccessorException(ErrorCode.E1501, e);
214            }
215    
216        }
217    
218        private String getMetastoreConnectURI(URI uri) {
219            String metastoreURI;
220            // For unit tests
221            if (uri.getAuthority().equals("unittest-local")) {
222                metastoreURI = "";
223            }
224            else {
225                // Hardcoding hcat to thrift mapping till support for webhcat(templeton)
226                // is added
227                metastoreURI = "thrift://" + uri.getAuthority();
228            }
229            return metastoreURI;
230        }
231    
232        private boolean exists(URI uri, HCatClient client, boolean closeClient) throws HCatAccessorException {
233            try {
234                HCatURI hcatURI = new HCatURI(uri.toString());
235                List<HCatPartition> partitions = client.getPartitions(hcatURI.getDb(), hcatURI.getTable(),
236                        hcatURI.getPartitionMap());
237                return (partitions != null && !partitions.isEmpty());
238            }
239            catch (ConnectionFailureException e) {
240                throw new HCatAccessorException(ErrorCode.E1501, e);
241            }
242            catch (HCatException e) {
243                throw new HCatAccessorException(ErrorCode.E0902, e);
244            }
245            catch (URISyntaxException e) {
246                throw new HCatAccessorException(ErrorCode.E0902, e);
247            }
248            finally {
249                closeQuietly(client, closeClient);
250            }
251        }
252    
253        private void closeQuietly(HCatClient client, boolean close) {
254            if (close && client != null) {
255                try {
256                    client.close();
257                }
258                catch (Exception ignore) {
259                    XLog.getLog(HCatURIHandler.class).warn("Error closing hcat client", ignore);
260                }
261            }
262        }
263    
264        static class HCatContext extends Context {
265    
266            private HCatClient hcatClient;
267    
268            /**
269             * Create a HCatContext that can be used to access a hcat URI
270             *
271             * @param conf Configuration to access the URI
272             * @param user name of the user the URI should be accessed as
273             * @param hcatClient HCatClient to talk to hcatalog server
274             */
275            public HCatContext(Configuration conf, String user, HCatClient hcatClient) {
276                super(conf, user);
277                this.hcatClient = hcatClient;
278            }
279    
280            /**
281             * Get the HCatClient to talk to hcatalog server
282             *
283             * @return HCatClient to talk to hcatalog server
284             */
285            public HCatClient getHCatClient() {
286                return hcatClient;
287            }
288    
289            @Override
290            public void destroy() {
291                try {
292                    hcatClient.close();
293                }
294                catch (Exception ignore) {
295                    XLog.getLog(HCatContext.class).warn("Error closing hcat client", ignore);
296                }
297            }
298    
299        }
300    
301    }