From c03b3291feb34d2a37f7bcddb3323e47a6349247 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Fri, 3 Feb 2012 23:03:39 +0000 Subject: [PATCH] svn merge -c 1240383 and svn merge -c 1240385 FIXES HADOOP-8013 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1240386 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/fs/FilterFileSystem.java | 22 +++- .../org/apache/hadoop/fs/LocalFileSystem.java | 12 ++- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 39 ++----- .../hadoop/fs/viewfs/ViewFileSystem.java | 10 +- .../hadoop/fs/FileSystemTestHelper.java | 5 +- .../fs/viewfs/TestChRootedFileSystem.java | 2 +- .../viewfs/TestViewFileSystemDelegation.java | 102 ++++++++++++++++++ 8 files changed, 152 insertions(+), 43 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 848e8706619..b9534c88e27 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -98,6 +98,9 @@ Release 0.23.1 - Unreleased OPTIMIZATIONS BUG FIXES + HADOOP-8013. ViewFileSystem does not honor setVerifyChecksum + (Daryn Sharp via bobby) + HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc (Jon Eagles via bobby) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 8df51b3ac82..dde2520041e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs; import java.io.*; import java.net.URI; +import java.net.URISyntaxException; import java.util.EnumSet; import java.util.List; @@ -51,6 +52,7 @@ import org.apache.hadoop.util.Progressable; public class FilterFileSystem extends FileSystem { protected FileSystem fs; + private String swapScheme; /* * so that extending classes can define it @@ -77,7 +79,11 @@ public class FilterFileSystem extends FileSystem { * @param conf the configuration */ public void initialize(URI name, Configuration conf) throws IOException { - fs.initialize(name, conf); + super.initialize(name, conf); + String scheme = name.getScheme(); + if (!scheme.equals(fs.getUri().getScheme())) { + swapScheme = scheme; + } } /** Returns a URI whose scheme and authority identify this FileSystem.*/ @@ -96,7 +102,19 @@ public class FilterFileSystem extends FileSystem { /** Make sure that a path specifies a FileSystem. */ public Path makeQualified(Path path) { - return fs.makeQualified(path); + Path fqPath = fs.makeQualified(path); + // swap in our scheme if the filtered fs is using a different scheme + if (swapScheme != null) { + try { + // NOTE: should deal with authority, but too much other stuff is broken + fqPath = new Path( + new URI(swapScheme, fqPath.toUri().getSchemeSpecificPart(), null) + ); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + return fqPath; } /////////////////////////////////////////////////////////////// diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java index 9e6c5d665cf..88ee7b82242 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystem.java @@ -24,6 +24,7 @@ import java.util.*; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; /**************************************************************** * Implement the FileSystem API for the checksumed local filesystem. @@ -34,21 +35,26 @@ import org.apache.hadoop.classification.InterfaceStability; public class LocalFileSystem extends ChecksumFileSystem { static final URI NAME = URI.create("file:///"); static private Random rand = new Random(); - FileSystem rfs; public LocalFileSystem() { this(new RawLocalFileSystem()); } public FileSystem getRaw() { - return rfs; + return getRawFileSystem(); } public LocalFileSystem(FileSystem rawLocalFileSystem) { super(rawLocalFileSystem); - rfs = rawLocalFileSystem; } + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + super.initialize(uri, conf); + // ctor didn't initialize the filtered fs + getRawFileSystem().initialize(uri, conf); + } + /** Convert a path to a File. */ public File pathToFile(Path path) { return ((RawLocalFileSystem)fs).pathToFile(path); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java index 84eb55ce13b..18ec724b7aa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java @@ -19,8 +19,6 @@ package org.apache.hadoop.fs.viewfs; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -82,37 +80,16 @@ class ChRootedFileSystem extends FilterFileSystem { /** * Constructor - * @param fs base file system - * @param theRoot chRoot for this file system - * @throws URISyntaxException + * @param uri base file system + * @param conf configuration + * @throws IOException */ - public ChRootedFileSystem(final FileSystem fs, final Path theRoot) - throws URISyntaxException { - super(fs); - makeQualified(theRoot); //check that root is a valid path for fs - // Would like to call myFs.checkPath(theRoot); - // but not public - chRootPathPart = new Path(theRoot.toUri().getPath()); + public ChRootedFileSystem(final URI uri, Configuration conf) + throws IOException { + super(FileSystem.get(uri, conf)); + chRootPathPart = new Path(uri.getPath()); chRootPathPartString = chRootPathPart.toUri().getPath(); - try { - initialize(fs.getUri(), fs.getConf()); - } catch (IOException e) { // This exception should not be thrown - throw new RuntimeException("This should not occur"); - } - - /* - * We are making URI include the chrootedPath: e.g. file:///chrootedPath. - * This is questionable since Path#makeQualified(uri, path) ignores - * the pathPart of a uri. Since this class is internal we can ignore - * this issue but if we were to make it external then this needs - * to be resolved. - */ - // Handle the two cases: - // scheme:/// and scheme://authority/ - myUri = new URI(fs.getUri().toString() + - (fs.getUri().getAuthority() == null ? "" : Path.SEPARATOR) + - chRootPathPart.toString().substring(1)); - + myUri = uri; workingDir = getHomeDirectory(); // We don't use the wd of the myFs } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 0778e7f9eff..1addf2fd88b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -168,8 +168,7 @@ public class ViewFileSystem extends FileSystem { protected FileSystem getTargetFileSystem(final URI uri) throws URISyntaxException, IOException { - return new ChRootedFileSystem(FileSystem.get(uri, config), - new Path(uri.getPath())); + return new ChRootedFileSystem(uri, config); } @Override @@ -464,8 +463,11 @@ public class ViewFileSystem extends FileSystem { @Override public void setVerifyChecksum(final boolean verifyChecksum) { - // This is a file system level operations, however ViewFileSystem - // points to many file systems. Noop for ViewFileSystem. + List> mountPoints = + fsState.getMountPoints(); + for (InodeTree.MountPoint mount : mountPoints) { + mount.target.targetFileSystem.setVerifyChecksum(verifyChecksum); + } } public MountPoint[] getMountPoints() { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java index b2a9e16038d..05fec95631d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java @@ -72,14 +72,15 @@ public final class FileSystemTestHelper { public static String getAbsoluteTestRootDir(FileSystem fSys) throws IOException { - if (absTestRootDir == null) { + // NOTE: can't cache because of different filesystems! + //if (absTestRootDir == null) if (TEST_ROOT_DIR.startsWith("/")) { absTestRootDir = TEST_ROOT_DIR; } else { absTestRootDir = fSys.getWorkingDirectory().toString() + "/" + TEST_ROOT_DIR; } - } + //} return absTestRootDir; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java index d1e9f598e66..c46ab96f375 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java @@ -51,7 +51,7 @@ public class TestChRootedFileSystem { // ChRoot to the root of the testDirectory - fSys = new ChRootedFileSystem(fSysTarget, chrootedTo); + fSys = new ChRootedFileSystem(chrootedTo.toUri(), conf); } @After diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java new file mode 100644 index 00000000000..b9c0a087b31 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegation.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.viewfs; + +import java.io.IOException; +import java.net.URI; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemTestHelper; +import org.apache.hadoop.fs.FsConstants; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.*; +import static org.junit.Assert.*; + +/** + * Verify that viewfs propagates certain methods to the underlying fs + */ +public class TestViewFileSystemDelegation { //extends ViewFileSystemTestSetup { + static Configuration conf; + static FileSystem viewFs; + static FakeFileSystem fs1; + static FakeFileSystem fs2; + + @BeforeClass + public static void setup() throws Exception { + conf = ViewFileSystemTestSetup.configWithViewfsScheme(); + fs1 = setupFileSystem(new URI("fs1:/"), FakeFileSystem.class); + fs2 = setupFileSystem(new URI("fs2:/"), FakeFileSystem.class); + viewFs = FileSystem.get(FsConstants.VIEWFS_URI, conf); + } + + static FakeFileSystem setupFileSystem(URI uri, Class clazz) + throws Exception { + String scheme = uri.getScheme(); + conf.set("fs."+scheme+".impl", clazz.getName()); + FakeFileSystem fs = (FakeFileSystem)FileSystem.get(uri, conf); + assertEquals(uri, fs.getUri()); + Path targetPath = FileSystemTestHelper.getAbsoluteTestRootPath(fs); + ConfigUtil.addLink(conf, "/mounts/"+scheme, targetPath.toUri()); + return fs; + } + + @Test + public void testSanity() { + assertEquals("fs1:/", fs1.getUri().toString()); + assertEquals("fs2:/", fs2.getUri().toString()); + } + + @Test + public void testVerifyChecksum() throws Exception { + checkVerifyChecksum(false); + checkVerifyChecksum(true); + } + + void checkVerifyChecksum(boolean flag) { + viewFs.setVerifyChecksum(flag); + assertEquals(flag, fs1.getVerifyChecksum()); + assertEquals(flag, fs2.getVerifyChecksum()); + } + + static class FakeFileSystem extends LocalFileSystem { + boolean verifyChecksum = true; + URI uri; + + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + super.initialize(uri, conf); + this.uri = uri; + } + + @Override + public URI getUri() { + return uri; + } + + @Override + public void setVerifyChecksum(boolean verifyChecksum) { + this.verifyChecksum = verifyChecksum; + } + + public boolean getVerifyChecksum(){ + return verifyChecksum; + } + } +}