From bf11465847a6e176640c4d62b482f894d1c373b2 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Tue, 25 Jun 2013 00:22:12 +0000 Subject: [PATCH] HADOOP-9355. Abstract symlink tests to use either FileContext or FileSystem. (Andrew Wang via Colin Patrick McCabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1496305 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/fs/FSTestWrapper.java | 140 +++++++ .../java/org/apache/hadoop/fs/FSWrapper.java | 112 +++++ .../hadoop/fs/FileContextTestWrapper.java | 335 +++++++++++++++ .../hadoop/fs/FileSystemTestWrapper.java | 393 ++++++++++++++++++ 5 files changed, 983 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSWrapper.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestWrapper.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestWrapper.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7cae36cb931..70de952e42d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -297,6 +297,9 @@ Release 2.2.0 - UNRELEASED HADOOP-9661. Allow metrics sources to be extended. (sandyr via tucu) + HADOOP-9370. Write FSWrapper class to wrap FileSystem and FileContext for + better test coverage. (Andrew Wang via Colin Patrick McCabe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java new file mode 100644 index 00000000000..3adb8f5d7f3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.io.IOException; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.fs.Options.CreateOpts; + +/** + * Abstraction of filesystem functionality with additional helper methods + * commonly used in tests. This allows generic tests to be written which apply + * to the two filesystem abstractions in Hadoop: {@link FileSystem} and + * {@link FileContext}. + */ +public abstract class FSTestWrapper implements FSWrapper { + + // + // Test helper methods taken from FileContextTestHelper + // + + protected static final int DEFAULT_BLOCK_SIZE = 1024; + protected static final int DEFAULT_NUM_BLOCKS = 2; + + protected String testRootDir = null; + protected String absTestRootDir = null; + + public FSTestWrapper(String testRootDir) { + // Use default test dir if not provided + if (testRootDir == null || testRootDir.isEmpty()) { + testRootDir = System.getProperty("test.build.data", "build/test/data"); + } + // salt test dir with some random digits for safe parallel runs + this.testRootDir = testRootDir + "/" + + RandomStringUtils.randomAlphanumeric(10); + } + + public static byte[] getFileData(int numOfBlocks, long blockSize) { + byte[] data = new byte[(int) (numOfBlocks * blockSize)]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (i % 10); + } + return data; + } + + public Path getTestRootPath() { + return makeQualified(new Path(testRootDir)); + } + + public Path getTestRootPath(String pathString) { + return makeQualified(new Path(testRootDir, pathString)); + } + + // the getAbsolutexxx method is needed because the root test dir + // can be messed up by changing the working dir. + + public String getAbsoluteTestRootDir() throws IOException { + if (absTestRootDir == null) { + if (testRootDir.startsWith("/")) { + absTestRootDir = testRootDir; + } else { + absTestRootDir = getWorkingDirectory().toString() + "/" + + testRootDir; + } + } + return absTestRootDir; + } + + public Path getAbsoluteTestRootPath() throws IOException { + return makeQualified(new Path(getAbsoluteTestRootDir())); + } + + abstract public FSTestWrapper getLocalFSWrapper() + throws UnsupportedFileSystemException, IOException; + + abstract public Path getDefaultWorkingDirectory() throws IOException; + + /* + * Create files with numBlocks blocks each with block size blockSize. + */ + abstract public long createFile(Path path, int numBlocks, + CreateOpts... options) throws IOException; + + abstract public long createFile(Path path, int numBlocks, int blockSize) + throws IOException; + + abstract public long createFile(Path path) throws IOException; + + abstract public long createFile(String name) throws IOException; + + abstract public long createFileNonRecursive(String name) throws IOException; + + abstract public long createFileNonRecursive(Path path) throws IOException; + + abstract public void appendToFile(Path path, int numBlocks, + CreateOpts... options) throws IOException; + + abstract public boolean exists(Path p) throws IOException; + + abstract public boolean isFile(Path p) throws IOException; + + abstract public boolean isDir(Path p) throws IOException; + + abstract public boolean isSymlink(Path p) throws IOException; + + abstract public void writeFile(Path path, byte b[]) throws IOException; + + abstract public byte[] readFile(Path path, int len) throws IOException; + + abstract public FileStatus containsPath(Path path, FileStatus[] dirList) + throws IOException; + + abstract public FileStatus containsPath(String path, FileStatus[] dirList) + throws IOException; + + enum fileType { + isDir, isFile, isSymlink + }; + + abstract public void checkFileStatus(String path, fileType expectedType) + throws IOException; + + abstract public void checkFileLinkStatus(String path, fileType expectedType) + throws IOException; +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSWrapper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSWrapper.java new file mode 100644 index 00000000000..e8875bf08da --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSWrapper.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.EnumSet; + +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.AccessControlException; + +/** + * Abstraction of filesystem operations that is essentially an interface + * extracted from {@link FileContext}. + */ +public interface FSWrapper { + + abstract public void setWorkingDirectory(final Path newWDir) + throws IOException; + + abstract public Path getWorkingDirectory(); + + abstract public Path makeQualified(final Path path); + + abstract public FSDataOutputStream create(final Path f, + final EnumSet createFlag, Options.CreateOpts... opts) + throws AccessControlException, FileAlreadyExistsException, + FileNotFoundException, ParentNotDirectoryException, + UnsupportedFileSystemException, IOException; + + abstract public void mkdir(final Path dir, final FsPermission permission, + final boolean createParent) throws AccessControlException, + FileAlreadyExistsException, FileNotFoundException, + ParentNotDirectoryException, UnsupportedFileSystemException, IOException; + + abstract public boolean delete(final Path f, final boolean recursive) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException; + + abstract public FSDataInputStream open(final Path f) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException; + + abstract public boolean setReplication(final Path f, final short replication) + throws AccessControlException, FileNotFoundException, + IOException; + + abstract public void rename(final Path src, final Path dst, + final Options.Rename... options) throws AccessControlException, + FileAlreadyExistsException, FileNotFoundException, + ParentNotDirectoryException, UnsupportedFileSystemException, IOException; + + abstract public void setPermission(final Path f, final FsPermission permission) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException; + + abstract public void setOwner(final Path f, final String username, + final String groupname) throws AccessControlException, + UnsupportedFileSystemException, FileNotFoundException, + IOException; + + abstract public void setTimes(final Path f, final long mtime, final long atime) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException; + + abstract public FileChecksum getFileChecksum(final Path f) + throws AccessControlException, FileNotFoundException, IOException; + + abstract public FileStatus getFileStatus(final Path f) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException; + + abstract public FileStatus getFileLinkStatus(final Path f) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException; + + abstract public Path getLinkTarget(final Path f) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException; + + abstract public BlockLocation[] getFileBlockLocations(final Path f, + final long start, final long len) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException; + + abstract public void createSymlink(final Path target, final Path link, + final boolean createParent) throws AccessControlException, + FileAlreadyExistsException, FileNotFoundException, + ParentNotDirectoryException, UnsupportedFileSystemException, IOException; + + abstract public RemoteIterator listStatusIterator(final Path f) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException; + + abstract public FileStatus[] listStatus(final Path f) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException; +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestWrapper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestWrapper.java new file mode 100644 index 00000000000..56736da90e0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestWrapper.java @@ -0,0 +1,335 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.FileNotFoundException; +import java.util.EnumSet; + +import org.apache.hadoop.fs.Options.CreateOpts; +import org.apache.hadoop.fs.Options.CreateOpts.BlockSize; +import org.apache.hadoop.fs.Options.Rename; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.security.AccessControlException; +import org.junit.Assert; + +/** + * Helper class for unit tests. + */ +public final class FileContextTestWrapper extends FSTestWrapper { + + private final FileContext fc; + + public FileContextTestWrapper(FileContext context) { + this(context, null); + } + + public FileContextTestWrapper(FileContext context, String rootDir) { + super(rootDir); + this.fc = context; + } + + public FSTestWrapper getLocalFSWrapper() + throws UnsupportedFileSystemException { + return new FileContextTestWrapper(FileContext.getLocalFSFileContext()); + } + + public Path getDefaultWorkingDirectory() throws IOException { + return getTestRootPath("/user/" + System.getProperty("user.name")) + .makeQualified(fc.getDefaultFileSystem().getUri(), + fc.getWorkingDirectory()); + } + + /* + * Create files with numBlocks blocks each with block size blockSize. + */ + public long createFile(Path path, int numBlocks, CreateOpts... options) + throws IOException { + BlockSize blockSizeOpt = + (BlockSize) CreateOpts.getOpt(CreateOpts.BlockSize.class, options); + long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue() + : DEFAULT_BLOCK_SIZE; + FSDataOutputStream out = + fc.create(path, EnumSet.of(CreateFlag.CREATE), options); + byte[] data = getFileData(numBlocks, blockSize); + out.write(data, 0, data.length); + out.close(); + return data.length; + } + + public long createFile(Path path, int numBlocks, int blockSize) + throws IOException { + return createFile(path, numBlocks, CreateOpts.blockSize(blockSize), + CreateOpts.createParent()); + } + + public long createFile(Path path) throws IOException { + return createFile(path, DEFAULT_NUM_BLOCKS, CreateOpts.createParent()); + } + + public long createFile(String name) throws IOException { + Path path = getTestRootPath(name); + return createFile(path); + } + + public long createFileNonRecursive(String name) throws IOException { + Path path = getTestRootPath(name); + return createFileNonRecursive(path); + } + + public long createFileNonRecursive(Path path) throws IOException { + return createFile(path, DEFAULT_NUM_BLOCKS, CreateOpts.donotCreateParent()); + } + + public void appendToFile(Path path, int numBlocks, CreateOpts... options) + throws IOException { + BlockSize blockSizeOpt = + (BlockSize) CreateOpts.getOpt(CreateOpts.BlockSize.class, options); + long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue() + : DEFAULT_BLOCK_SIZE; + FSDataOutputStream out; + out = fc.create(path, EnumSet.of(CreateFlag.APPEND)); + byte[] data = getFileData(numBlocks, blockSize); + out.write(data, 0, data.length); + out.close(); + } + + public boolean exists(Path p) throws IOException { + return fc.util().exists(p); + } + + public boolean isFile(Path p) throws IOException { + try { + return fc.getFileStatus(p).isFile(); + } catch (FileNotFoundException e) { + return false; + } + } + + public boolean isDir(Path p) throws IOException { + try { + return fc.getFileStatus(p).isDirectory(); + } catch (FileNotFoundException e) { + return false; + } + } + + public boolean isSymlink(Path p) throws IOException { + try { + return fc.getFileLinkStatus(p).isSymlink(); + } catch (FileNotFoundException e) { + return false; + } + } + + public void writeFile(Path path, byte b[]) throws IOException { + FSDataOutputStream out = + fc.create(path,EnumSet.of(CreateFlag.CREATE), CreateOpts.createParent()); + out.write(b); + out.close(); + } + + public byte[] readFile(Path path, int len) throws IOException { + DataInputStream dis = fc.open(path); + byte[] buffer = new byte[len]; + IOUtils.readFully(dis, buffer, 0, len); + dis.close(); + return buffer; + } + + public FileStatus containsPath(Path path, FileStatus[] dirList) + throws IOException { + for(int i = 0; i < dirList.length; i ++) { + if (path.equals(dirList[i].getPath())) + return dirList[i]; + } + return null; + } + + public FileStatus containsPath(String path, FileStatus[] dirList) + throws IOException { + return containsPath(new Path(path), dirList); + } + + public void checkFileStatus(String path, fileType expectedType) + throws IOException { + FileStatus s = fc.getFileStatus(new Path(path)); + Assert.assertNotNull(s); + if (expectedType == fileType.isDir) { + Assert.assertTrue(s.isDirectory()); + } else if (expectedType == fileType.isFile) { + Assert.assertTrue(s.isFile()); + } else if (expectedType == fileType.isSymlink) { + Assert.assertTrue(s.isSymlink()); + } + Assert.assertEquals(fc.makeQualified(new Path(path)), s.getPath()); + } + + public void checkFileLinkStatus(String path, fileType expectedType) + throws IOException { + FileStatus s = fc.getFileLinkStatus(new Path(path)); + Assert.assertNotNull(s); + if (expectedType == fileType.isDir) { + Assert.assertTrue(s.isDirectory()); + } else if (expectedType == fileType.isFile) { + Assert.assertTrue(s.isFile()); + } else if (expectedType == fileType.isSymlink) { + Assert.assertTrue(s.isSymlink()); + } + Assert.assertEquals(fc.makeQualified(new Path(path)), s.getPath()); + } + + // + // FileContext wrappers + // + + @Override + public Path makeQualified(Path path) { + return fc.makeQualified(path); + } + + @Override + public void mkdir(Path dir, FsPermission permission, boolean createParent) + throws AccessControlException, FileAlreadyExistsException, + FileNotFoundException, ParentNotDirectoryException, + UnsupportedFileSystemException, IOException { + fc.mkdir(dir, permission, createParent); + } + + @Override + public boolean delete(Path f, boolean recursive) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + return fc.delete(f, recursive); + } + + @Override + public FileStatus getFileLinkStatus(Path f) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException { + return fc.getFileLinkStatus(f); + } + + @Override + public void createSymlink(Path target, Path link, boolean createParent) + throws AccessControlException, FileAlreadyExistsException, + FileNotFoundException, ParentNotDirectoryException, + UnsupportedFileSystemException, IOException { + fc.createSymlink(target, link, createParent); + } + + @Override + public void setWorkingDirectory(Path newWDir) throws IOException { + fc.setWorkingDirectory(newWDir); + } + + @Override + public Path getWorkingDirectory() { + return fc.getWorkingDirectory(); + } + + @Override + public FileStatus getFileStatus(Path f) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException { + return fc.getFileStatus(f); + } + + @Override + public FSDataOutputStream create(Path f, EnumSet createFlag, + CreateOpts... opts) throws AccessControlException, + FileAlreadyExistsException, FileNotFoundException, + ParentNotDirectoryException, UnsupportedFileSystemException, IOException { + return fc.create(f, createFlag, opts); + } + + @Override + public FSDataInputStream open(Path f) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException { + return fc.open(f); + } + + @Override + public boolean setReplication(final Path f, final short replication) + throws AccessControlException, FileNotFoundException, + IOException { + return fc.setReplication(f, replication); + } + + @Override + public Path getLinkTarget(Path f) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException { + return fc.getLinkTarget(f); + } + + @Override + public void rename(Path src, Path dst, Rename... options) + throws AccessControlException, FileAlreadyExistsException, + FileNotFoundException, ParentNotDirectoryException, + UnsupportedFileSystemException, IOException { + fc.rename(src, dst, options); + } + + @Override + public BlockLocation[] getFileBlockLocations(Path f, long start, long len) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + return fc.getFileBlockLocations(f, start, len); + } + + @Override + public FileChecksum getFileChecksum(Path f) throws AccessControlException, + FileNotFoundException, IOException { + return fc.getFileChecksum(f); + } + + @Override + public RemoteIterator listStatusIterator(Path f) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + return fc.listStatus(f); + } + + @Override + public void setPermission(final Path f, final FsPermission permission) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + fc.setPermission(f, permission); + } + + @Override + public void setOwner(final Path f, final String username, + final String groupname) throws AccessControlException, + UnsupportedFileSystemException, FileNotFoundException, + IOException { + fc.setOwner(f, username, groupname); + } + + @Override + public void setTimes(Path f, long mtime, long atime) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + fc.setTimes(f, mtime, atime); + } + + @Override + public FileStatus[] listStatus(Path f) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException { + return fc.util().listStatus(f); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestWrapper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestWrapper.java new file mode 100644 index 00000000000..a6bdd38756f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestWrapper.java @@ -0,0 +1,393 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.io.DataInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.EnumSet; + +import org.apache.hadoop.fs.Options.CreateOpts; +import org.apache.hadoop.fs.Options.CreateOpts.BlockSize; +import org.apache.hadoop.fs.Options.Rename; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.util.Progressable; +import org.junit.Assert; + +/** + * Helper class for unit tests. + */ +public final class FileSystemTestWrapper extends FSTestWrapper { + + private final FileSystem fs; + + public FileSystemTestWrapper(FileSystem fs) { + this(fs, null); + } + + public FileSystemTestWrapper(FileSystem fs, String rootDir) { + super(rootDir); + this.fs = fs; + } + + public FSTestWrapper getLocalFSWrapper() + throws IOException { + return new FileSystemTestWrapper(FileSystem.getLocal(fs.getConf())); + } + + public Path getDefaultWorkingDirectory() throws IOException { + return getTestRootPath("/user/" + System.getProperty("user.name")) + .makeQualified(fs.getUri(), + fs.getWorkingDirectory()); + } + + /* + * Create files with numBlocks blocks each with block size blockSize. + */ + public long createFile(Path path, int numBlocks, CreateOpts... options) + throws IOException { + BlockSize blockSizeOpt = + (BlockSize) CreateOpts.getOpt(CreateOpts.BlockSize.class, options); + long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue() + : DEFAULT_BLOCK_SIZE; + FSDataOutputStream out = + create(path, EnumSet.of(CreateFlag.CREATE), options); + byte[] data = getFileData(numBlocks, blockSize); + out.write(data, 0, data.length); + out.close(); + return data.length; + } + + public long createFile(Path path, int numBlocks, int blockSize) + throws IOException { + return createFile(path, numBlocks, CreateOpts.blockSize(blockSize), + CreateOpts.createParent()); + } + + public long createFile(Path path) throws IOException { + return createFile(path, DEFAULT_NUM_BLOCKS, CreateOpts.createParent()); + } + + public long createFile(String name) throws IOException { + Path path = getTestRootPath(name); + return createFile(path); + } + + public long createFileNonRecursive(String name) throws IOException { + Path path = getTestRootPath(name); + return createFileNonRecursive(path); + } + + public long createFileNonRecursive(Path path) throws IOException { + return createFile(path, DEFAULT_NUM_BLOCKS, CreateOpts.donotCreateParent()); + } + + public void appendToFile(Path path, int numBlocks, CreateOpts... options) + throws IOException { + BlockSize blockSizeOpt = + (BlockSize) CreateOpts.getOpt(CreateOpts.BlockSize.class, options); + long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue() + : DEFAULT_BLOCK_SIZE; + FSDataOutputStream out; + out = fs.append(path); + byte[] data = getFileData(numBlocks, blockSize); + out.write(data, 0, data.length); + out.close(); + } + + public boolean exists(Path p) throws IOException { + return fs.exists(p); + } + + public boolean isFile(Path p) throws IOException { + try { + return fs.getFileStatus(p).isFile(); + } catch (FileNotFoundException e) { + return false; + } + } + + public boolean isDir(Path p) throws IOException { + try { + return fs.getFileStatus(p).isDirectory(); + } catch (FileNotFoundException e) { + return false; + } + } + + public boolean isSymlink(Path p) throws IOException { + throw new UnsupportedFileSystemException( + "FileSystem does not support symlinks"); + } + + public void writeFile(Path path, byte b[]) throws IOException { + FSDataOutputStream out = + create(path,EnumSet.of(CreateFlag.CREATE), CreateOpts.createParent()); + out.write(b); + out.close(); + } + + public byte[] readFile(Path path, int len) throws IOException { + DataInputStream dis = fs.open(path); + byte[] buffer = new byte[len]; + IOUtils.readFully(dis, buffer, 0, len); + dis.close(); + return buffer; + } + + public FileStatus containsPath(Path path, FileStatus[] dirList) + throws IOException { + for(int i = 0; i < dirList.length; i ++) { + if (path.equals(dirList[i].getPath())) + return dirList[i]; + } + return null; + } + + public FileStatus containsPath(String path, FileStatus[] dirList) + throws IOException { + return containsPath(new Path(path), dirList); + } + + public void checkFileStatus(String path, fileType expectedType) + throws IOException { + FileStatus s = fs.getFileStatus(new Path(path)); + Assert.assertNotNull(s); + if (expectedType == fileType.isDir) { + Assert.assertTrue(s.isDirectory()); + } else if (expectedType == fileType.isFile) { + Assert.assertTrue(s.isFile()); + } else if (expectedType == fileType.isSymlink) { + Assert.assertTrue(s.isSymlink()); + } + Assert.assertEquals(fs.makeQualified(new Path(path)), s.getPath()); + } + + public void checkFileLinkStatus(String path, fileType expectedType) + throws IOException { + throw new UnsupportedFileSystemException( + "FileSystem does not support symlinks"); + } + + // + // FileContext wrappers + // + + @Override + public Path makeQualified(Path path) { + return fs.makeQualified(path); + } + + @Override + public void mkdir(Path dir, FsPermission permission, boolean createParent) + throws AccessControlException, FileAlreadyExistsException, + FileNotFoundException, ParentNotDirectoryException, + UnsupportedFileSystemException, IOException { + // Note that there is no "mkdir" in FileSystem, it always does + // "mkdir -p" (creating parent directories). + fs.mkdirs(dir, permission); + } + + @Override + public boolean delete(Path f, boolean recursive) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + return fs.delete(f, recursive); + } + + @Override + public FileStatus getFileLinkStatus(Path f) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException { + throw new UnsupportedFileSystemException( + "FileSystem does not support symlinks"); + } + + @Override + public void createSymlink(Path target, Path link, boolean createParent) + throws AccessControlException, FileAlreadyExistsException, + FileNotFoundException, ParentNotDirectoryException, + UnsupportedFileSystemException, IOException { + throw new UnsupportedFileSystemException( + "FileSystem does not support symlinks"); + } + + @Override + public void setWorkingDirectory(Path newWDir) throws IOException { + fs.setWorkingDirectory(newWDir); + } + + @Override + public Path getWorkingDirectory() { + return fs.getWorkingDirectory(); + } + + @Override + public FileStatus getFileStatus(Path f) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException { + return fs.getFileStatus(f); + } + + @Override + public FSDataOutputStream create(Path f, EnumSet createFlag, + CreateOpts... opts) throws AccessControlException, + FileAlreadyExistsException, FileNotFoundException, + ParentNotDirectoryException, UnsupportedFileSystemException, IOException { + + // Need to translate the FileContext-style options into FileSystem-style + + // Permissions with umask + CreateOpts.Perms permOpt = (CreateOpts.Perms) CreateOpts.getOpt( + CreateOpts.Perms.class, opts); + FsPermission umask = FsPermission.getUMask(fs.getConf()); + FsPermission permission = (permOpt != null) ? permOpt.getValue() + : FsPermission.getFileDefault().applyUMask(umask); + permission = permission.applyUMask(umask); + // Overwrite + boolean overwrite = createFlag.contains(CreateFlag.OVERWRITE); + // bufferSize + int bufferSize = fs.getConf().getInt( + CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, + CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT); + CreateOpts.BufferSize bufOpt = (CreateOpts.BufferSize) CreateOpts.getOpt( + CreateOpts.BufferSize.class, opts); + bufferSize = (bufOpt != null) ? bufOpt.getValue() : bufferSize; + // replication + short replication = fs.getDefaultReplication(f); + CreateOpts.ReplicationFactor repOpt = + (CreateOpts.ReplicationFactor) CreateOpts.getOpt( + CreateOpts.ReplicationFactor.class, opts); + replication = (repOpt != null) ? repOpt.getValue() : replication; + // blockSize + long blockSize = fs.getDefaultBlockSize(f); + CreateOpts.BlockSize blockOpt = (CreateOpts.BlockSize) CreateOpts.getOpt( + CreateOpts.BlockSize.class, opts); + blockSize = (blockOpt != null) ? blockOpt.getValue() : blockSize; + // Progressable + Progressable progress = null; + CreateOpts.Progress progressOpt = (CreateOpts.Progress) CreateOpts.getOpt( + CreateOpts.Progress.class, opts); + progress = (progressOpt != null) ? progressOpt.getValue() : progress; + return fs.create(f, permission, overwrite, bufferSize, replication, + blockSize, progress); + } + + @Override + public FSDataInputStream open(Path f) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException { + return fs.open(f); + } + + @Override + public Path getLinkTarget(Path f) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException { + throw new UnsupportedFileSystemException( + "FileSystem does not support symlinks"); + } + + @Override + public boolean setReplication(final Path f, final short replication) + throws AccessControlException, FileNotFoundException, + IOException { + return fs.setReplication(f, replication); + } + + @SuppressWarnings("deprecation") + @Override + public void rename(Path src, Path dst, Rename... options) + throws AccessControlException, FileAlreadyExistsException, + FileNotFoundException, ParentNotDirectoryException, + UnsupportedFileSystemException, IOException { + fs.rename(src, dst, options); + } + + @Override + public BlockLocation[] getFileBlockLocations(Path f, long start, long len) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + return fs.getFileBlockLocations(f, start, len); + } + + @Override + public FileChecksum getFileChecksum(Path f) throws AccessControlException, + FileNotFoundException, IOException { + return fs.getFileChecksum(f); + } + + private class FakeRemoteIterator implements RemoteIterator { + + private E[] elements; + private int count; + + FakeRemoteIterator(E[] elements) { + this.elements = elements; + count = 0; + } + + @Override + public boolean hasNext() throws IOException { + return count < elements.length; + } + + @Override + public E next() throws IOException { + if (hasNext()) { + return elements[count++]; + } + return null; + } + } + + @Override + public RemoteIterator listStatusIterator(Path f) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + // Fake the RemoteIterator, because FileSystem has no such thing + FileStatus[] statuses = fs.listStatus(f); + return new FakeRemoteIterator(statuses); + } + + @Override + public void setPermission(final Path f, final FsPermission permission) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + fs.setPermission(f, permission); + } + + @Override + public void setOwner(final Path f, final String username, + final String groupname) throws AccessControlException, + UnsupportedFileSystemException, FileNotFoundException, + IOException { + fs.setOwner(f, username, groupname); + } + + @Override + public void setTimes(Path f, long mtime, long atime) + throws AccessControlException, FileNotFoundException, + UnsupportedFileSystemException, IOException { + fs.setTimes(f, mtime, atime); + } + + @Override + public FileStatus[] listStatus(Path f) throws AccessControlException, + FileNotFoundException, UnsupportedFileSystemException, IOException { + return fs.listStatus(f); + } +}