001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import org.apache.hadoop.io.Text;
021    import org.apache.hadoop.mapred.JobClient;
022    import org.apache.hadoop.mapred.JobConf;
023    import org.apache.hadoop.fs.FileSystem;
024    import org.apache.hadoop.fs.Path;
025    import org.apache.hadoop.conf.Configuration;
026    import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
027    import org.apache.hadoop.net.NetUtils;
028    import org.apache.hadoop.security.SecurityUtil;
029    import org.apache.hadoop.security.UserGroupInformation;
030    import org.apache.hadoop.filecache.DistributedCache;
031    import org.apache.hadoop.security.token.Token;
032    import org.apache.oozie.ErrorCode;
033    import org.apache.oozie.action.hadoop.JavaActionExecutor;
034    import org.apache.oozie.util.ParamChecker;
035    import org.apache.oozie.util.XConfiguration;
036    import org.apache.oozie.util.XLog;
037    import org.apache.oozie.util.JobUtils;
038    
039    import java.io.File;
040    import java.io.FileInputStream;
041    import java.io.FilenameFilter;
042    import java.io.IOException;
043    import java.io.InputStream;
044    import java.net.InetAddress;
045    import java.net.URI;
046    import java.net.URISyntaxException;
047    import java.security.PrivilegedExceptionAction;
048    import java.util.HashMap;
049    import java.util.Map;
050    import java.util.Set;
051    import java.util.HashSet;
052    import java.util.concurrent.ConcurrentHashMap;
053    
054    /**
055     * The HadoopAccessorService returns HadoopAccessor instances configured to work on behalf of a user-group. <p/> The
056     * default accessor used is the base accessor which just injects the UGI into the configuration instance used to
057     * create/obtain JobClient and FileSystem instances.
058     */
059    public class HadoopAccessorService implements Service {
060    
061        private static XLog LOG = XLog.getLog(HadoopAccessorService.class);
062    
063        public static final String CONF_PREFIX = Service.CONF_PREFIX + "HadoopAccessorService.";
064        public static final String JOB_TRACKER_WHITELIST = CONF_PREFIX + "jobTracker.whitelist";
065        public static final String NAME_NODE_WHITELIST = CONF_PREFIX + "nameNode.whitelist";
066        public static final String HADOOP_CONFS = CONF_PREFIX + "hadoop.configurations";
067        public static final String ACTION_CONFS = CONF_PREFIX + "action.configurations";
068        public static final String KERBEROS_AUTH_ENABLED = CONF_PREFIX + "kerberos.enabled";
069        public static final String KERBEROS_KEYTAB = CONF_PREFIX + "keytab.file";
070        public static final String KERBEROS_PRINCIPAL = CONF_PREFIX + "kerberos.principal";
071        public static final Text MR_TOKEN_ALIAS = new Text("oozie mr token");
072    
073        protected static final String OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED = "oozie.HadoopAccessorService.created";
074        /** The Kerberos principal for the job tracker.*/
075        protected static final String JT_PRINCIPAL = "mapreduce.jobtracker.kerberos.principal";
076        /** The Kerberos principal for the resource manager.*/
077        protected static final String RM_PRINCIPAL = "yarn.resourcemanager.principal";
078        protected static final String HADOOP_JOB_TRACKER = "mapred.job.tracker";
079        protected static final String HADOOP_JOB_TRACKER_2 = "mapreduce.jobtracker.address";
080        protected static final String HADOOP_YARN_RM = "yarn.resourcemanager.address";
081        private static final Map<String, Text> mrTokenRenewers = new HashMap<String, Text>();
082    
083        private Set<String> jobTrackerWhitelist = new HashSet<String>();
084        private Set<String> nameNodeWhitelist = new HashSet<String>();
085        private Map<String, Configuration> hadoopConfigs = new HashMap<String, Configuration>();
086        private Map<String, File> actionConfigDirs = new HashMap<String, File>();
087        private Map<String, Map<String, XConfiguration>> actionConfigs = new HashMap<String, Map<String, XConfiguration>>();
088    
089        private UserGroupInformationService ugiService;
090    
091        /**
092         * Supported filesystem schemes for namespace federation
093         */
094        public static final String SUPPORTED_FILESYSTEMS = CONF_PREFIX + "supported.filesystems";
095        public static final String[] DEFAULT_SUPPORTED_SCHEMES = new String[]{"hdfs","hftp","webhdfs"};
096        private Set<String> supportedSchemes;
097        private boolean allSchemesSupported;
098    
099        public void init(Services services) throws ServiceException {
100            this.ugiService = services.get(UserGroupInformationService.class);
101            init(services.getConf());
102        }
103    
104        //for testing purposes, see XFsTestCase
105        public void init(Configuration conf) throws ServiceException {
106            for (String name : conf.getStringCollection(JOB_TRACKER_WHITELIST)) {
107                String tmp = name.toLowerCase().trim();
108                if (tmp.length() == 0) {
109                    continue;
110                }
111                jobTrackerWhitelist.add(tmp);
112            }
113            XLog.getLog(getClass()).info(
114                    "JOB_TRACKER_WHITELIST :" + conf.getStringCollection(JOB_TRACKER_WHITELIST)
115                            + ", Total entries :" + jobTrackerWhitelist.size());
116            for (String name : conf.getStringCollection(NAME_NODE_WHITELIST)) {
117                String tmp = name.toLowerCase().trim();
118                if (tmp.length() == 0) {
119                    continue;
120                }
121                nameNodeWhitelist.add(tmp);
122            }
123            XLog.getLog(getClass()).info(
124                    "NAME_NODE_WHITELIST :" + conf.getStringCollection(NAME_NODE_WHITELIST)
125                            + ", Total entries :" + nameNodeWhitelist.size());
126    
127            boolean kerberosAuthOn = conf.getBoolean(KERBEROS_AUTH_ENABLED, true);
128            XLog.getLog(getClass()).info("Oozie Kerberos Authentication [{0}]", (kerberosAuthOn) ? "enabled" : "disabled");
129            if (kerberosAuthOn) {
130                kerberosInit(conf);
131            }
132            else {
133                Configuration ugiConf = new Configuration();
134                ugiConf.set("hadoop.security.authentication", "simple");
135                UserGroupInformation.setConfiguration(ugiConf);
136            }
137    
138            if (ugiService == null) { //for testing purposes, see XFsTestCase
139                this.ugiService = new UserGroupInformationService();
140            }
141    
142            loadHadoopConfigs(conf);
143            preLoadActionConfigs(conf);
144    
145            supportedSchemes = new HashSet<String>();
146            String[] schemesFromConf = conf.getStrings(SUPPORTED_FILESYSTEMS, DEFAULT_SUPPORTED_SCHEMES);
147            if(schemesFromConf != null) {
148                for (String scheme: schemesFromConf) {
149                    scheme = scheme.trim();
150                    // If user gives "*", supportedSchemes will be empty, so that checking is not done i.e. all schemes allowed
151                    if(scheme.equals("*")) {
152                        if(schemesFromConf.length > 1) {
153                            throw new ServiceException(ErrorCode.E0100, getClass().getName(),
154                                SUPPORTED_FILESYSTEMS + " should contain either only wildcard or explicit list, not both");
155                        }
156                        allSchemesSupported = true;
157                    }
158                    supportedSchemes.add(scheme);
159                }
160            }
161        }
162    
163        private void kerberosInit(Configuration serviceConf) throws ServiceException {
164                try {
165                    String keytabFile = serviceConf.get(KERBEROS_KEYTAB,
166                                                        System.getProperty("user.home") + "/oozie.keytab").trim();
167                    if (keytabFile.length() == 0) {
168                        throw new ServiceException(ErrorCode.E0026, KERBEROS_KEYTAB);
169                    }
170                    String principal = SecurityUtil.getServerPrincipal(
171                             serviceConf.get(KERBEROS_PRINCIPAL, "oozie/localhost@LOCALHOST"),
172                             InetAddress.getLocalHost().getCanonicalHostName());
173                    if (principal.length() == 0) {
174                        throw new ServiceException(ErrorCode.E0026, KERBEROS_PRINCIPAL);
175                    }
176                    Configuration conf = new Configuration();
177                    conf.set("hadoop.security.authentication", "kerberos");
178                    UserGroupInformation.setConfiguration(conf);
179                    UserGroupInformation.loginUserFromKeytab(principal, keytabFile);
180                    XLog.getLog(getClass()).info("Got Kerberos ticket, keytab [{0}], Oozie principal principal [{1}]",
181                                                 keytabFile, principal);
182                }
183                catch (ServiceException ex) {
184                    throw ex;
185                }
186                catch (Exception ex) {
187                    throw new ServiceException(ErrorCode.E0100, getClass().getName(), ex.getMessage(), ex);
188                }
189        }
190    
191        private static final String[] HADOOP_CONF_FILES =
192            {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml", "hadoop-site.xml", "ssl-client.xml"};
193    
194    
195        private Configuration loadHadoopConf(File dir) throws IOException {
196            Configuration hadoopConf = new XConfiguration();
197            for (String file : HADOOP_CONF_FILES) {
198                File f = new File(dir, file);
199                if (f.exists()) {
200                    InputStream is = new FileInputStream(f);
201                    Configuration conf = new XConfiguration(is);
202                    is.close();
203                    XConfiguration.copy(conf, hadoopConf);
204                }
205            }
206            return hadoopConf;
207        }
208    
209        private Map<String, File> parseConfigDirs(String[] confDefs, String type) throws ServiceException, IOException {
210            Map<String, File> map = new HashMap<String, File>();
211            File configDir = new File(ConfigurationService.getConfigurationDirectory());
212            for (String confDef : confDefs) {
213                if (confDef.trim().length() > 0) {
214                    String[] parts = confDef.split("=");
215                    if (parts.length == 2) {
216                        String hostPort = parts[0];
217                        String confDir = parts[1];
218                        File dir = new File(confDir);
219                        if (!dir.isAbsolute()) {
220                            dir = new File(configDir, confDir);
221                        }
222                        if (dir.exists()) {
223                            map.put(hostPort.toLowerCase(), dir);
224                        }
225                        else {
226                            throw new ServiceException(ErrorCode.E0100, getClass().getName(),
227                                                       "could not find " + type + " configuration directory: " +
228                                                       dir.getAbsolutePath());
229                        }
230                    }
231                    else {
232                        throw new ServiceException(ErrorCode.E0100, getClass().getName(),
233                                                   "Incorrect " + type + " configuration definition: " + confDef);
234                    }
235                }
236            }
237            return map;
238        }
239    
240        private void loadHadoopConfigs(Configuration serviceConf) throws ServiceException {
241            try {
242                Map<String, File> map = parseConfigDirs(serviceConf.getStrings(HADOOP_CONFS, "*=hadoop-conf"), "hadoop");
243                for (Map.Entry<String, File> entry : map.entrySet()) {
244                    hadoopConfigs.put(entry.getKey(), loadHadoopConf(entry.getValue()));
245                }
246            }
247            catch (ServiceException ex) {
248                throw ex;
249            }
250            catch (Exception ex) {
251                throw new ServiceException(ErrorCode.E0100, getClass().getName(), ex.getMessage(), ex);
252            }
253        }
254    
255        private void preLoadActionConfigs(Configuration serviceConf) throws ServiceException {
256            try {
257                actionConfigDirs = parseConfigDirs(serviceConf.getStrings(ACTION_CONFS, "*=hadoop-conf"), "action");
258                for (String hostport : actionConfigDirs.keySet()) {
259                    actionConfigs.put(hostport, new ConcurrentHashMap<String, XConfiguration>());
260                }
261            }
262            catch (ServiceException ex) {
263                throw ex;
264            }
265            catch (Exception ex) {
266                throw new ServiceException(ErrorCode.E0100, getClass().getName(), ex.getMessage(), ex);
267            }
268        }
269    
270        public void destroy() {
271        }
272    
273        public Class<? extends Service> getInterface() {
274            return HadoopAccessorService.class;
275        }
276    
277        private UserGroupInformation getUGI(String user) throws IOException {
278            return ugiService.getProxyUser(user);
279        }
280    
281        /**
282         * Creates a JobConf using the site configuration for the specified hostname:port.
283         * <p/>
284         * If the specified hostname:port is not defined it falls back to the '*' site
285         * configuration if available. If the '*' site configuration is not available,
286         * the JobConf has all Hadoop defaults.
287         *
288         * @param hostPort hostname:port to lookup Hadoop site configuration.
289         * @return a JobConf with the corresponding site configuration for hostPort.
290         */
291        public JobConf createJobConf(String hostPort) {
292            JobConf jobConf = new JobConf();
293            XConfiguration.copy(getConfiguration(hostPort), jobConf);
294            jobConf.setBoolean(OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, true);
295            return jobConf;
296        }
297    
298        private XConfiguration loadActionConf(String hostPort, String action) {
299            File dir = actionConfigDirs.get(hostPort);
300            XConfiguration actionConf = new XConfiguration();
301            if (dir != null) {
302                File actionConfFile = new File(dir, action + ".xml");
303                if (actionConfFile.exists()) {
304                    try {
305                        actionConf = new XConfiguration(new FileInputStream(actionConfFile));
306                    }
307                    catch (IOException ex) {
308                        XLog.getLog(getClass()).warn("Could not read file [{0}] for action [{1}] configuration for hostPort [{2}]",
309                                                     actionConfFile.getAbsolutePath(), action, hostPort);
310                    }
311                }
312    
313                // See if a dir with the action name exists.   If so, load all the xml files in the dir
314                File actionConfDir = new File(dir, action);
315    
316                if (actionConfDir.exists() && actionConfDir.isDirectory()) {
317                    XLog.getLog(getClass()).info("Processing configuration files under [{0}]"
318                            + " for action [{1}] and hostPort [{2}]",
319                            actionConfDir.getAbsolutePath(), action, hostPort);
320                    File[] xmlFiles = actionConfDir.listFiles(
321                            new FilenameFilter() {
322                                @Override
323                                public boolean accept(File dir, String name) {
324                                    return name.endsWith(".xml");
325                                }});
326                    for (File f : xmlFiles) {
327                        if (f.isFile() && f.canRead()) {
328                            XLog.getLog(getClass()).info("Processing configuration file [{0}]", f.getName());
329                            FileInputStream fis = null;
330                            try {
331                                fis = new FileInputStream(f);
332                                XConfiguration conf = new XConfiguration(fis);
333                                XConfiguration.copy(conf, actionConf);
334                            }
335                            catch (IOException ex) {
336                                XLog.getLog(getClass())
337                                    .warn("Could not read file [{0}] for action [{1}] configuration and hostPort [{2}]",
338                                        f.getAbsolutePath(), action, hostPort);
339                            }
340                            finally {
341                                if (fis != null) {
342                                    try { fis.close(); } catch(IOException ioe) { }
343                                }
344                            }
345                        }
346                    }
347                }
348            }
349            return actionConf;
350        }
351    
352        /**
353         * Returns a Configuration containing any defaults for an action for a particular cluster.
354         * <p/>
355         * This configuration is used as default for the action configuration and enables cluster
356         * level default values per action.
357         *
358         * @param hostPort hostname"port to lookup the action default confiugration.
359         * @param action action name.
360         * @return the default configuration for the action for the specified cluster.
361         */
362        public XConfiguration createActionDefaultConf(String hostPort, String action) {
363            hostPort = (hostPort != null) ? hostPort.toLowerCase() : null;
364            Map<String, XConfiguration> hostPortActionConfigs = actionConfigs.get(hostPort);
365            if (hostPortActionConfigs == null) {
366                hostPortActionConfigs = actionConfigs.get("*");
367                hostPort = "*";
368            }
369            XConfiguration actionConf = hostPortActionConfigs.get(action);
370            if (actionConf == null) {
371                // doing lazy loading as we don't know upfront all actions, no need to synchronize
372                // as it is a read operation an in case of a race condition loading and inserting
373                // into the Map is idempotent and the action-config Map is a ConcurrentHashMap
374                actionConf = loadActionConf(hostPort, action);
375                hostPortActionConfigs.put(action, actionConf);
376            }
377            return new XConfiguration(actionConf.toProperties());
378        }
379    
380        private Configuration getConfiguration(String hostPort) {
381            hostPort = (hostPort != null) ? hostPort.toLowerCase() : null;
382            Configuration conf = hadoopConfigs.get(hostPort);
383            if (conf == null) {
384                conf = hadoopConfigs.get("*");
385                if (conf == null) {
386                    conf = new XConfiguration();
387                }
388            }
389            return conf;
390        }
391    
392        public String getPasswordFromHadoopConf(Configuration conf, String propName) {
393            LOG = XLog.getLog(HadoopAccessorService.class);
394            Configuration hadoopConf = getConfiguration("*");
395            String password = "";
396            LOG.info("password key is " + propName);
397            char[] passChar = null;
398            try{
399                passChar = hadoopConf.getPassword(propName);
400                LOG.info("password is " + passChar);
401            }
402            catch (IOException ex) {
403    
404            }
405            if (passChar == null || passChar.length == 0) {
406                password = conf.get(propName, "").trim();
407            }
408            else {
409                password = new String(passChar);
410            }
411    
412            LOG.info("final password is " + password);
413            return password;
414        }
415    
416        /**
417         * Return a JobClient created with the provided user/group.
418         *
419         *
420         * @param conf JobConf with all necessary information to create the
421         *        JobClient.
422         * @return JobClient created with the provided user/group.
423         * @throws HadoopAccessorException if the client could not be created.
424         */
425        public JobClient createJobClient(String user, final JobConf conf) throws HadoopAccessorException {
426            ParamChecker.notEmpty(user, "user");
427            if (!conf.getBoolean(OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, false)) {
428                throw new HadoopAccessorException(ErrorCode.E0903);
429            }
430            String jobTracker = conf.get(JavaActionExecutor.HADOOP_JOB_TRACKER);
431            validateJobTracker(jobTracker);
432            try {
433                UserGroupInformation ugi = getUGI(user);
434                JobClient jobClient = ugi.doAs(new PrivilegedExceptionAction<JobClient>() {
435                    public JobClient run() throws Exception {
436                        return new JobClient(conf);
437                    }
438                });
439                Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(getMRDelegationTokenRenewer(conf));
440                conf.getCredentials().addToken(MR_TOKEN_ALIAS, mrdt);
441                return jobClient;
442            }
443            catch (InterruptedException ex) {
444                throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
445            }
446            catch (IOException ex) {
447                throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
448            }
449        }
450    
451        /**
452         * Return a FileSystem created with the provided user for the specified URI.
453         *
454         *
455         * @param uri file system URI.
456         * @param conf Configuration with all necessary information to create the FileSystem.
457         * @return FileSystem created with the provided user/group.
458         * @throws HadoopAccessorException if the filesystem could not be created.
459         */
460        public FileSystem createFileSystem(String user, final URI uri, final Configuration conf)
461                throws HadoopAccessorException {
462            ParamChecker.notEmpty(user, "user");
463            if (!conf.getBoolean(OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, false)) {
464                throw new HadoopAccessorException(ErrorCode.E0903);
465            }
466    
467            checkSupportedFilesystem(uri);
468    
469            String nameNode = uri.getAuthority();
470            if (nameNode == null) {
471                nameNode = conf.get("fs.default.name");
472                if (nameNode != null) {
473                    try {
474                        nameNode = new URI(nameNode).getAuthority();
475                    }
476                    catch (URISyntaxException ex) {
477                        throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
478                    }
479                }
480            }
481            validateNameNode(nameNode);
482    
483            try {
484                UserGroupInformation ugi = getUGI(user);
485                return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
486                    public FileSystem run() throws Exception {
487                        return FileSystem.get(uri, conf);
488                    }
489                });
490            }
491            catch (InterruptedException ex) {
492                throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
493            }
494            catch (IOException ex) {
495                throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
496            }
497        }
498    
499        /**
500         * Validate Job tracker
501         * @param jobTrackerUri
502         * @throws HadoopAccessorException
503         */
504        protected void validateJobTracker(String jobTrackerUri) throws HadoopAccessorException {
505            validate(jobTrackerUri, jobTrackerWhitelist, ErrorCode.E0900);
506        }
507    
508        /**
509         * Validate Namenode list
510         * @param nameNodeUri
511         * @throws HadoopAccessorException
512         */
513        protected void validateNameNode(String nameNodeUri) throws HadoopAccessorException {
514            validate(nameNodeUri, nameNodeWhitelist, ErrorCode.E0901);
515        }
516    
517        private void validate(String uri, Set<String> whitelist, ErrorCode error) throws HadoopAccessorException {
518            if (uri != null) {
519                uri = uri.toLowerCase().trim();
520                if (whitelist.size() > 0 && !whitelist.contains(uri)) {
521                    throw new HadoopAccessorException(error, uri);
522                }
523            }
524        }
525    
526        public Text getMRDelegationTokenRenewer(JobConf jobConf) throws IOException {
527            if (UserGroupInformation.isSecurityEnabled()) { // secure cluster
528                return getMRTokenRenewerInternal(jobConf);
529            }
530            else {
531                return MR_TOKEN_ALIAS; //Doesn't matter what we pass as renewer
532            }
533        }
534    
535        // Package private for unit test purposes
536        Text getMRTokenRenewerInternal(JobConf jobConf) throws IOException {
537            // Getting renewer correctly for JT principal also though JT in hadoop 1.x does not have
538            // support for renewing/cancelling tokens
539            String servicePrincipal = jobConf.get(RM_PRINCIPAL, jobConf.get(JT_PRINCIPAL));
540            Text renewer;
541            if (servicePrincipal != null) { // secure cluster
542                renewer = mrTokenRenewers.get(servicePrincipal);
543                if (renewer == null) {
544                    // Mimic org.apache.hadoop.mapred.Master.getMasterPrincipal()
545                    String target = jobConf.get(HADOOP_YARN_RM, jobConf.get(HADOOP_JOB_TRACKER_2));
546                    if (target == null) {
547                        target = jobConf.get(HADOOP_JOB_TRACKER);
548                    }
549                    try {
550                        String addr = NetUtils.createSocketAddr(target).getHostName();
551                        renewer = new Text(SecurityUtil.getServerPrincipal(servicePrincipal, addr));
552                        LOG.info("Delegation Token Renewer details: Principal=" + servicePrincipal + ",Target=" + target
553                                + ",Renewer=" + renewer);
554                    }
555                    catch (IllegalArgumentException iae) {
556                        renewer = new Text(servicePrincipal.split("[/@]")[0]);
557                        LOG.info("Delegation Token Renewer for " + servicePrincipal + " is " + renewer);
558                    }
559                    mrTokenRenewers.put(servicePrincipal, renewer);
560                }
561            }
562            else {
563                renewer = MR_TOKEN_ALIAS; //Doesn't matter what we pass as renewer
564            }
565            return renewer;
566        }
567    
568        public void addFileToClassPath(String user, final Path file, final Configuration conf)
569                throws IOException {
570            ParamChecker.notEmpty(user, "user");
571            try {
572                UserGroupInformation ugi = getUGI(user);
573                ugi.doAs(new PrivilegedExceptionAction<Void>() {
574                    @Override
575                    public Void run() throws Exception {
576                        JobUtils.addFileToClassPath(file, conf, null);
577                        return null;
578                    }
579                });
580    
581            }
582            catch (InterruptedException ex) {
583                throw new IOException(ex);
584            }
585    
586        }
587    
588        /**
589         * checks configuration parameter if filesystem scheme is among the list of supported ones
590         * this makes system robust to filesystems other than HDFS also
591         */
592    
593        public void checkSupportedFilesystem(URI uri) throws HadoopAccessorException {
594            if (allSchemesSupported)
595                return;
596            String uriScheme = uri.getScheme();
597            if (uriScheme != null) {    // skip the check if no scheme is given
598                if(!supportedSchemes.isEmpty()) {
599                    XLog.getLog(this.getClass()).debug("Checking if filesystem " + uriScheme + " is supported");
600                    if (!supportedSchemes.contains(uriScheme)) {
601                        throw new HadoopAccessorException(ErrorCode.E0904, uriScheme, uri.toString());
602                    }
603                 }
604             }
605        }
606    
607        public Set<String> getSupportedSchemes() {
608            return supportedSchemes;
609        }
610    
611    }