001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.action.hadoop;
019    
020    import java.io.IOException;
021    import java.net.URISyntaxException;
022    import java.util.ArrayList;
023    import java.util.HashMap;
024    import java.util.List;
025    import java.util.Map;
026    
027    import org.apache.hadoop.fs.FSDataOutputStream;
028    import org.apache.hadoop.fs.FileStatus;
029    import org.apache.hadoop.fs.FileSystem;
030    import org.apache.hadoop.fs.FileUtil;
031    import org.apache.hadoop.fs.Path;
032    import org.apache.hadoop.fs.permission.FsPermission;
033    import org.apache.hadoop.mapred.JobConf;
034    import org.apache.oozie.action.ActionExecutor;
035    import org.apache.oozie.action.ActionExecutorException;
036    import org.apache.oozie.client.WorkflowAction;
037    import org.apache.oozie.service.HadoopAccessorException;
038    import org.apache.oozie.service.HadoopAccessorService;
039    import org.apache.oozie.service.Services;
040    import org.apache.oozie.util.XConfiguration;
041    import org.apache.oozie.util.XmlUtils;
042    import org.jdom.Element;
043    
044    /**
045     * File system action executor. <p/> This executes the file system mkdir, move and delete commands
046     */
047    public class FsActionExecutor extends ActionExecutor {
048    
049        private final int maxGlobCount;
050    
051        public FsActionExecutor() {
052            super("fs");
053            maxGlobCount = getOozieConf().getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX,
054                    LauncherMapper.GLOB_MAX_DEFAULT);
055        }
056    
057        Path getPath(Element element, String attribute) {
058            String str = element.getAttributeValue(attribute).trim();
059            return new Path(str);
060        }
061    
062        void validatePath(Path path, boolean withScheme) throws ActionExecutorException {
063            try {
064                String scheme = path.toUri().getScheme();
065                if (withScheme) {
066                    if (scheme == null) {
067                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS001",
068                                                          "Missing scheme in path [{0}]", path);
069                    }
070                    else {
071                        Services.get().get(HadoopAccessorService.class).checkSupportedFilesystem(path.toUri());
072                    }
073                }
074                else {
075                    if (scheme != null) {
076                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS002",
077                                                          "Scheme [{0}] not allowed in path [{1}]", scheme, path);
078                    }
079                }
080            }
081            catch (HadoopAccessorException hex) {
082                throw convertException(hex);
083            }
084        }
085    
086        Path resolveToFullPath(Path nameNode, Path path, boolean withScheme) throws ActionExecutorException {
087            Path fullPath;
088    
089            // If no nameNode is given, validate the path as-is and return it as-is
090            if (nameNode == null) {
091                validatePath(path, withScheme);
092                fullPath = path;
093            } else {
094                // If the path doesn't have a scheme or authority, use the nameNode which should have already been verified earlier
095                String pathScheme = path.toUri().getScheme();
096                String pathAuthority = path.toUri().getAuthority();
097                if (pathScheme == null || pathAuthority == null) {
098                    if (path.isAbsolute()) {
099                        String nameNodeSchemeAuthority = nameNode.toUri().getScheme() + "://" + nameNode.toUri().getAuthority();
100                        fullPath = new Path(nameNodeSchemeAuthority + path.toString());
101                    } else {
102                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS011",
103                                "Path [{0}] cannot be relative", path);
104                    }
105                } else {
106                    // If the path has a scheme and authority, but its not the nameNode then validate the path as-is and return it as-is
107                    // If it is the nameNode, then it should have already been verified earlier so return it as-is
108                    if (!nameNode.toUri().getScheme().equals(pathScheme) || !nameNode.toUri().getAuthority().equals(pathAuthority)) {
109                        validatePath(path, withScheme);
110                    }
111                    fullPath = path;
112                }
113            }
114            return fullPath;
115        }
116    
117        void validateSameNN(Path source, Path dest) throws ActionExecutorException {
118            Path destPath = new Path(source, dest);
119            String t = destPath.toUri().getScheme() + destPath.toUri().getAuthority();
120            String s = source.toUri().getScheme() + source.toUri().getAuthority();
121    
122            //checking whether NN prefix of source and target is same. can modify this to adjust for a set of multiple whitelisted NN
123            if(!t.equals(s)) {
124                throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS007",
125                        "move, target NN URI different from that of source", dest);
126            }
127        }
128    
129        @SuppressWarnings("unchecked")
130        void doOperations(Context context, Element element) throws ActionExecutorException {
131            try {
132                FileSystem fs = context.getAppFileSystem();
133                boolean recovery = fs.exists(getRecoveryPath(context));
134                if (!recovery) {
135                    fs.mkdirs(getRecoveryPath(context));
136                }
137    
138                Path nameNodePath = null;
139                Element nameNodeElement = element.getChild("name-node", element.getNamespace());
140                if (nameNodeElement != null) {
141                    String nameNode = nameNodeElement.getTextTrim();
142                    if (nameNode != null) {
143                        nameNodePath = new Path(nameNode);
144                        // Verify the name node now
145                        validatePath(nameNodePath, true);
146                    }
147                }
148    
149                XConfiguration fsConf = new XConfiguration();
150                Path appPath = new Path(context.getWorkflow().getAppPath());
151                // app path could be a file
152                if (fs.isFile(appPath)) {
153                    appPath = appPath.getParent();
154                }
155                JavaActionExecutor.parseJobXmlAndConfiguration(context, element, appPath, fsConf);
156    
157                for (Element commandElement : (List<Element>) element.getChildren()) {
158                    String command = commandElement.getName();
159                    if (command.equals("mkdir")) {
160                        Path path = getPath(commandElement, "path");
161                        mkdir(context, fsConf, nameNodePath, path);
162                    }
163                    else {
164                        if (command.equals("delete")) {
165                            Path path = getPath(commandElement, "path");
166                            delete(context, fsConf, nameNodePath, path);
167                        }
168                        else {
169                            if (command.equals("move")) {
170                                Path source = getPath(commandElement, "source");
171                                Path target = getPath(commandElement, "target");
172                                move(context, fsConf, nameNodePath, source, target, recovery);
173                            }
174                            else {
175                                if (command.equals("chmod")) {
176                                    Path path = getPath(commandElement, "path");
177                                    boolean recursive = commandElement.getChild("recursive", commandElement.getNamespace()) != null;
178                                    String str = commandElement.getAttributeValue("dir-files");
179                                    boolean dirFiles = (str == null) || Boolean.parseBoolean(str);
180                                    String permissionsMask = commandElement.getAttributeValue("permissions").trim();
181                                    chmod(context, fsConf, nameNodePath, path, permissionsMask, dirFiles, recursive);
182                                }
183                                else {
184                                    if (command.equals("touchz")) {
185                                        Path path = getPath(commandElement, "path");
186                                        touchz(context, fsConf, nameNodePath, path);
187                                    }
188                                    else {
189                                        if (command.equals("chgrp")) {
190                                            Path path = getPath(commandElement, "path");
191                                            boolean recursive = commandElement.getChild("recursive",
192                                                    commandElement.getNamespace()) != null;
193                                            String group = commandElement.getAttributeValue("group");
194                                            String str = commandElement.getAttributeValue("dir-files");
195                                            boolean dirFiles = (str == null) || Boolean.parseBoolean(str);
196                                            chgrp(context, fsConf, nameNodePath, path, context.getWorkflow().getUser(),
197                                                    group, dirFiles, recursive);
198                                        }
199                                    }
200                                }
201                            }
202                        }
203                    }
204                }
205            }
206            catch (Exception ex) {
207                throw convertException(ex);
208            }
209        }
210    
211        void chgrp(Context context, XConfiguration fsConf, Path nameNodePath, Path path, String user, String group,
212                boolean dirFiles, boolean recursive) throws ActionExecutorException {
213    
214            HashMap<String, String> argsMap = new HashMap<String, String>();
215            argsMap.put("user", user);
216            argsMap.put("group", group);
217            try {
218                FileSystem fs = getFileSystemFor(path, context, fsConf);
219                path = resolveToFullPath(nameNodePath, path, true);
220                Path[] pathArr = FileUtil.stat2Paths(fs.globStatus(path));
221                if (pathArr == null || pathArr.length == 0) {
222                    throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS009", "chgrp"
223                            + ", path(s) that matches [{0}] does not exist", path);
224                }
225                checkGlobMax(pathArr);
226                for (Path p : pathArr) {
227                    recursiveFsOperation("chgrp", fs, nameNodePath, p, argsMap, dirFiles, recursive, true);
228                }
229            }
230            catch (Exception ex) {
231                throw convertException(ex);
232            }
233        }
234    
235        private void recursiveFsOperation(String op, FileSystem fs, Path nameNodePath, Path path,
236                Map<String, String> argsMap, boolean dirFiles, boolean recursive, boolean isRoot)
237                throws ActionExecutorException {
238    
239            try {
240                FileStatus pathStatus = fs.getFileStatus(path);
241                List<Path> paths = new ArrayList<Path>();
242    
243                if (dirFiles && pathStatus.isDir()) {
244                    if (isRoot) {
245                        paths.add(path);
246                    }
247                    FileStatus[] filesStatus = fs.listStatus(path);
248                    for (int i = 0; i < filesStatus.length; i++) {
249                        Path p = filesStatus[i].getPath();
250                        paths.add(p);
251                        if (recursive && filesStatus[i].isDir()) {
252                            recursiveFsOperation(op, fs, null, p, argsMap, dirFiles, recursive, false);
253                        }
254                    }
255                }
256                else {
257                    paths.add(path);
258                }
259                for (Path p : paths) {
260                    doFsOperation(op, fs, p, argsMap);
261                }
262            }
263            catch (Exception ex) {
264                throw convertException(ex);
265            }
266        }
267    
268        private void doFsOperation(String op, FileSystem fs, Path p, Map<String, String> argsMap)
269                throws ActionExecutorException, IOException {
270            if (op.equals("chmod")) {
271                String permissions = argsMap.get("permissions");
272                FsPermission newFsPermission = createShortPermission(permissions, p);
273                fs.setPermission(p, newFsPermission);
274            }
275            else if (op.equals("chgrp")) {
276                String user = argsMap.get("user");
277                String group = argsMap.get("group");
278                fs.setOwner(p, user, group);
279            }
280        }
281    
282        /**
283         * @param path
284         * @param context
285         * @param fsConf
286         * @return FileSystem
287         * @throws HadoopAccessorException
288         */
289        private FileSystem getFileSystemFor(Path path, Context context, XConfiguration fsConf) throws HadoopAccessorException {
290            String user = context.getWorkflow().getUser();
291            HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
292            JobConf conf = has.createJobConf(path.toUri().getAuthority());
293            XConfiguration.copy(context.getProtoActionConf(), conf);
294            if (fsConf != null) {
295                XConfiguration.copy(fsConf, conf);
296            }
297            return has.createFileSystem(user, path.toUri(), conf);
298        }
299    
300        /**
301         * @param path
302         * @param user
303         * @param group
304         * @return FileSystem
305         * @throws HadoopAccessorException
306         */
307        private FileSystem getFileSystemFor(Path path, String user) throws HadoopAccessorException {
308            HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
309            JobConf jobConf = has.createJobConf(path.toUri().getAuthority());
310            return has.createFileSystem(user, path.toUri(), jobConf);
311        }
312    
313        void mkdir(Context context, Path path) throws ActionExecutorException {
314            mkdir(context, null, null, path);
315        }
316    
317        void mkdir(Context context, XConfiguration fsConf, Path nameNodePath, Path path) throws ActionExecutorException {
318            try {
319                path = resolveToFullPath(nameNodePath, path, true);
320                FileSystem fs = getFileSystemFor(path, context, fsConf);
321    
322                if (!fs.exists(path)) {
323                    if (!fs.mkdirs(path)) {
324                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS004",
325                                                          "mkdir, path [{0}] could not create directory", path);
326                    }
327                }
328            }
329            catch (Exception ex) {
330                throw convertException(ex);
331            }
332        }
333    
334        /**
335         * Delete path
336         *
337         * @param context
338         * @param path
339         * @throws ActionExecutorException
340         */
341        public void delete(Context context, Path path) throws ActionExecutorException {
342            delete(context, null, null, path);
343        }
344    
345        /**
346         * Delete path
347         *
348         * @param context
349         * @param fsConf
350         * @param nameNodePath
351         * @param path
352         * @throws ActionExecutorException
353         */
354        public void delete(Context context, XConfiguration fsConf, Path nameNodePath, Path path) throws ActionExecutorException {
355            try {
356                path = resolveToFullPath(nameNodePath, path, true);
357                FileSystem fs = getFileSystemFor(path, context, fsConf);
358                Path[] pathArr = FileUtil.stat2Paths(fs.globStatus(path));
359                if (pathArr != null && pathArr.length > 0) {
360                    checkGlobMax(pathArr);
361                    for (Path p : pathArr) {
362                        if (fs.exists(p)) {
363                            if (!fs.delete(p, true)) {
364                                throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS005",
365                                        "delete, path [{0}] could not delete path", p);
366                            }
367                        }
368                    }
369                }
370            }
371            catch (Exception ex) {
372                throw convertException(ex);
373            }
374        }
375    
376        /**
377         * Delete path
378         *
379         * @param user
380         * @param group
381         * @param path
382         * @throws ActionExecutorException
383         */
384        public void delete(String user, String group, Path path) throws ActionExecutorException {
385            try {
386                validatePath(path, true);
387                FileSystem fs = getFileSystemFor(path, user);
388    
389                if (fs.exists(path)) {
390                    if (!fs.delete(path, true)) {
391                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS005",
392                                "delete, path [{0}] could not delete path", path);
393                    }
394                }
395            }
396            catch (Exception ex) {
397                throw convertException(ex);
398            }
399        }
400    
401        /**
402         * Move source to target
403         *
404         * @param context
405         * @param source
406         * @param target
407         * @param recovery
408         * @throws ActionExecutorException
409         */
410        public void move(Context context, Path source, Path target, boolean recovery) throws ActionExecutorException {
411            move(context, null, null, source, target, recovery);
412        }
413    
414        /**
415         * Move source to target
416         *
417         * @param context
418         * @param fsConf
419         * @param nameNodePath
420         * @param source
421         * @param target
422         * @param recovery
423         * @throws ActionExecutorException
424         */
425        public void move(Context context, XConfiguration fsConf, Path nameNodePath, Path source, Path target, boolean recovery)
426                throws ActionExecutorException {
427            try {
428                source = resolveToFullPath(nameNodePath, source, true);
429                validateSameNN(source, target);
430                FileSystem fs = getFileSystemFor(source, context, fsConf);
431                Path[] pathArr = FileUtil.stat2Paths(fs.globStatus(source));
432                if (( pathArr == null || pathArr.length == 0 ) ){
433                    if (!recovery) {
434                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS006",
435                            "move, source path [{0}] does not exist", source);
436                    } else {
437                        return;
438                    }
439                }
440                if (pathArr.length > 1 && (!fs.exists(target) || fs.isFile(target))) {
441                    if(!recovery) {
442                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS012",
443                                "move, could not rename multiple sources to the same target name");
444                    } else {
445                        return;
446                    }
447                }
448                checkGlobMax(pathArr);
449                for (Path p : pathArr) {
450                    if (!fs.rename(p, target) && !recovery) {
451                        throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS008",
452                                "move, could not move [{0}] to [{1}]", p, target);
453                    }
454                }
455            }
456            catch (Exception ex) {
457                throw convertException(ex);
458            }
459        }
460    
461        void chmod(Context context, Path path, String permissions, boolean dirFiles, boolean recursive) throws ActionExecutorException {
462            chmod(context, null, null, path, permissions, dirFiles, recursive);
463        }
464    
465        void chmod(Context context, XConfiguration fsConf, Path nameNodePath, Path path, String permissions,
466                boolean dirFiles, boolean recursive) throws ActionExecutorException {
467    
468            HashMap<String, String> argsMap = new HashMap<String, String>();
469            argsMap.put("permissions", permissions);
470            try {
471                FileSystem fs = getFileSystemFor(path, context, fsConf);
472                path = resolveToFullPath(nameNodePath, path, true);
473                Path[] pathArr = FileUtil.stat2Paths(fs.globStatus(path));
474                if (pathArr == null || pathArr.length == 0) {
475                    throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS009", "chmod"
476                            + ", path(s) that matches [{0}] does not exist", path);
477                }
478                checkGlobMax(pathArr);
479                for (Path p : pathArr) {
480                    recursiveFsOperation("chmod", fs, nameNodePath, p, argsMap, dirFiles, recursive, true);
481                }
482    
483            }
484            catch (Exception ex) {
485                throw convertException(ex);
486            }
487        }
488    
489        void touchz(Context context, Path path) throws ActionExecutorException {
490            touchz(context, null, null, path);
491        }
492    
493        void touchz(Context context, XConfiguration fsConf, Path nameNodePath, Path path) throws ActionExecutorException {
494            try {
495                path = resolveToFullPath(nameNodePath, path, true);
496                FileSystem fs = getFileSystemFor(path, context, fsConf);
497    
498                FileStatus st;
499                if (fs.exists(path)) {
500                    st = fs.getFileStatus(path);
501                    if (st.isDir()) {
502                        throw new Exception(path.toString() + " is a directory");
503                    } else if (st.getLen() != 0)
504                        throw new Exception(path.toString() + " must be a zero-length file");
505                }
506                FSDataOutputStream out = fs.create(path);
507                out.close();
508            }
509            catch (Exception ex) {
510                throw convertException(ex);
511            }
512        }
513    
514        FsPermission createShortPermission(String permissions, Path path) throws ActionExecutorException {
515            if (permissions.length() == 3) {
516                char user = permissions.charAt(0);
517                char group = permissions.charAt(1);
518                char other = permissions.charAt(2);
519                int useri = user - '0';
520                int groupi = group - '0';
521                int otheri = other - '0';
522                int mask = useri * 100 + groupi * 10 + otheri;
523                short omask = Short.parseShort(Integer.toString(mask), 8);
524                return new FsPermission(omask);
525            }
526            else {
527                if (permissions.length() == 10) {
528                    return FsPermission.valueOf(permissions);
529                }
530                else {
531                    throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS010",
532                                                      "chmod, path [{0}] invalid permissions mask [{1}]", path, permissions);
533                }
534            }
535        }
536    
537        @Override
538        public void check(Context context, WorkflowAction action) throws ActionExecutorException {
539        }
540    
541        @Override
542        public void kill(Context context, WorkflowAction action) throws ActionExecutorException {
543        }
544    
545        @Override
546        public void start(Context context, WorkflowAction action) throws ActionExecutorException {
547            try {
548                context.setStartData("-", "-", "-");
549                Element actionXml = XmlUtils.parseXml(action.getConf());
550                doOperations(context, actionXml);
551                context.setExecutionData("OK", null);
552            }
553            catch (Exception ex) {
554                throw convertException(ex);
555            }
556        }
557    
558        @Override
559        public void end(Context context, WorkflowAction action) throws ActionExecutorException {
560            String externalStatus = action.getExternalStatus();
561            WorkflowAction.Status status = externalStatus.equals("OK") ? WorkflowAction.Status.OK :
562                                           WorkflowAction.Status.ERROR;
563            context.setEndData(status, getActionSignal(status));
564            if (!context.getProtoActionConf().getBoolean("oozie.action.keep.action.dir", false)) {
565                try {
566                    FileSystem fs = context.getAppFileSystem();
567                    fs.delete(context.getActionDir(), true);
568                }
569                catch (Exception ex) {
570                    throw convertException(ex);
571                }
572            }
573        }
574    
575        @Override
576        public boolean isCompleted(String externalStatus) {
577            return true;
578        }
579    
580        /**
581         * @param context
582         * @return
583         * @throws HadoopAccessorException
584         * @throws IOException
585         * @throws URISyntaxException
586         */
587        public Path getRecoveryPath(Context context) throws HadoopAccessorException, IOException, URISyntaxException {
588            return new Path(context.getActionDir(), "fs-" + context.getRecoveryId());
589        }
590    
591        private void checkGlobMax(Path[] pathArr) throws ActionExecutorException {
592            if(pathArr.length > maxGlobCount) {
593                throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS013",
594                        "too many globbed files/dirs to do FS operation");
595            }
596        }
597    
598    }