svn merge -c 1240383 and svn merge -c 1240385 FIXES HADOOP-8013

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1240386 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-02-03 23:03:39 +00:00
parent 220317c188
commit c03b3291fe
8 changed files with 152 additions and 43 deletions

View File

@ -98,6 +98,9 @@ Release 0.23.1 - Unreleased
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES
HADOOP-8013. ViewFileSystem does not honor setVerifyChecksum
(Daryn Sharp via bobby)
HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc
(Jon Eagles via bobby) (Jon Eagles via bobby)

View File

@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
import java.io.*; import java.io.*;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.List; import java.util.List;
@ -51,6 +52,7 @@ import org.apache.hadoop.util.Progressable;
public class FilterFileSystem extends FileSystem { public class FilterFileSystem extends FileSystem {
protected FileSystem fs; protected FileSystem fs;
private String swapScheme;
/* /*
* so that extending classes can define it * so that extending classes can define it
@ -77,7 +79,11 @@ public class FilterFileSystem extends FileSystem {
* @param conf the configuration * @param conf the configuration
*/ */
public void initialize(URI name, Configuration conf) throws IOException { public void initialize(URI name, Configuration conf) throws IOException {
fs.initialize(name, conf); super.initialize(name, conf);
String scheme = name.getScheme();
if (!scheme.equals(fs.getUri().getScheme())) {
swapScheme = scheme;
}
} }
/** Returns a URI whose scheme and authority identify this FileSystem.*/ /** Returns a URI whose scheme and authority identify this FileSystem.*/
@ -96,7 +102,19 @@ public class FilterFileSystem extends FileSystem {
/** Make sure that a path specifies a FileSystem. */ /** Make sure that a path specifies a FileSystem. */
public Path makeQualified(Path path) { public Path makeQualified(Path path) {
return fs.makeQualified(path); Path fqPath = fs.makeQualified(path);
// swap in our scheme if the filtered fs is using a different scheme
if (swapScheme != null) {
try {
// NOTE: should deal with authority, but too much other stuff is broken
fqPath = new Path(
new URI(swapScheme, fqPath.toUri().getSchemeSpecificPart(), null)
);
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
}
}
return fqPath;
} }
/////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////

View File

@ -24,6 +24,7 @@ import java.util.*;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
/**************************************************************** /****************************************************************
* Implement the FileSystem API for the checksumed local filesystem. * Implement the FileSystem API for the checksumed local filesystem.
@ -34,21 +35,26 @@ import org.apache.hadoop.classification.InterfaceStability;
public class LocalFileSystem extends ChecksumFileSystem { public class LocalFileSystem extends ChecksumFileSystem {
static final URI NAME = URI.create("file:///"); static final URI NAME = URI.create("file:///");
static private Random rand = new Random(); static private Random rand = new Random();
FileSystem rfs;
public LocalFileSystem() { public LocalFileSystem() {
this(new RawLocalFileSystem()); this(new RawLocalFileSystem());
} }
public FileSystem getRaw() { public FileSystem getRaw() {
return rfs; return getRawFileSystem();
} }
public LocalFileSystem(FileSystem rawLocalFileSystem) { public LocalFileSystem(FileSystem rawLocalFileSystem) {
super(rawLocalFileSystem); super(rawLocalFileSystem);
rfs = rawLocalFileSystem;
} }
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
super.initialize(uri, conf);
// ctor didn't initialize the filtered fs
getRawFileSystem().initialize(uri, conf);
}
/** Convert a path to a File. */ /** Convert a path to a File. */
public File pathToFile(Path path) { public File pathToFile(Path path) {
return ((RawLocalFileSystem)fs).pathToFile(path); return ((RawLocalFileSystem)fs).pathToFile(path);

View File

@ -19,8 +19,6 @@ package org.apache.hadoop.fs.viewfs;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -82,37 +80,16 @@ class ChRootedFileSystem extends FilterFileSystem {
/** /**
* Constructor * Constructor
* @param fs base file system * @param uri base file system
* @param theRoot chRoot for this file system * @param conf configuration
* @throws URISyntaxException * @throws IOException
*/ */
public ChRootedFileSystem(final FileSystem fs, final Path theRoot) public ChRootedFileSystem(final URI uri, Configuration conf)
throws URISyntaxException { throws IOException {
super(fs); super(FileSystem.get(uri, conf));
makeQualified(theRoot); //check that root is a valid path for fs chRootPathPart = new Path(uri.getPath());
// Would like to call myFs.checkPath(theRoot);
// but not public
chRootPathPart = new Path(theRoot.toUri().getPath());
chRootPathPartString = chRootPathPart.toUri().getPath(); chRootPathPartString = chRootPathPart.toUri().getPath();
try { myUri = uri;
initialize(fs.getUri(), fs.getConf());
} catch (IOException e) { // This exception should not be thrown
throw new RuntimeException("This should not occur");
}
/*
* We are making URI include the chrootedPath: e.g. file:///chrootedPath.
* This is questionable since Path#makeQualified(uri, path) ignores
* the pathPart of a uri. Since this class is internal we can ignore
* this issue but if we were to make it external then this needs
* to be resolved.
*/
// Handle the two cases:
// scheme:/// and scheme://authority/
myUri = new URI(fs.getUri().toString() +
(fs.getUri().getAuthority() == null ? "" : Path.SEPARATOR) +
chRootPathPart.toString().substring(1));
workingDir = getHomeDirectory(); workingDir = getHomeDirectory();
// We don't use the wd of the myFs // We don't use the wd of the myFs
} }

View File

@ -168,8 +168,7 @@ public class ViewFileSystem extends FileSystem {
protected protected
FileSystem getTargetFileSystem(final URI uri) FileSystem getTargetFileSystem(final URI uri)
throws URISyntaxException, IOException { throws URISyntaxException, IOException {
return new ChRootedFileSystem(FileSystem.get(uri, config), return new ChRootedFileSystem(uri, config);
new Path(uri.getPath()));
} }
@Override @Override
@ -464,8 +463,11 @@ public class ViewFileSystem extends FileSystem {
@Override @Override
public void setVerifyChecksum(final boolean verifyChecksum) { public void setVerifyChecksum(final boolean verifyChecksum) {
// This is a file system level operations, however ViewFileSystem List<InodeTree.MountPoint<FileSystem>> mountPoints =
// points to many file systems. Noop for ViewFileSystem. fsState.getMountPoints();
for (InodeTree.MountPoint<FileSystem> mount : mountPoints) {
mount.target.targetFileSystem.setVerifyChecksum(verifyChecksum);
}
} }
public MountPoint[] getMountPoints() { public MountPoint[] getMountPoints() {

View File

@ -72,14 +72,15 @@ public final class FileSystemTestHelper {
public static String getAbsoluteTestRootDir(FileSystem fSys) public static String getAbsoluteTestRootDir(FileSystem fSys)
throws IOException { throws IOException {
if (absTestRootDir == null) { // NOTE: can't cache because of different filesystems!
//if (absTestRootDir == null)
if (TEST_ROOT_DIR.startsWith("/")) { if (TEST_ROOT_DIR.startsWith("/")) {
absTestRootDir = TEST_ROOT_DIR; absTestRootDir = TEST_ROOT_DIR;
} else { } else {
absTestRootDir = fSys.getWorkingDirectory().toString() + "/" absTestRootDir = fSys.getWorkingDirectory().toString() + "/"
+ TEST_ROOT_DIR; + TEST_ROOT_DIR;
} }
} //}
return absTestRootDir; return absTestRootDir;
} }

View File

@ -51,7 +51,7 @@ public class TestChRootedFileSystem {
// ChRoot to the root of the testDirectory // ChRoot to the root of the testDirectory
fSys = new ChRootedFileSystem(fSysTarget, chrootedTo); fSys = new ChRootedFileSystem(chrootedTo.toUri(), conf);
} }
@After @After

View File

@ -0,0 +1,102 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.viewfs;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.*;
import static org.junit.Assert.*;
/**
* Verify that viewfs propagates certain methods to the underlying fs
*/
public class TestViewFileSystemDelegation { //extends ViewFileSystemTestSetup {
static Configuration conf;
static FileSystem viewFs;
static FakeFileSystem fs1;
static FakeFileSystem fs2;
@BeforeClass
public static void setup() throws Exception {
conf = ViewFileSystemTestSetup.configWithViewfsScheme();
fs1 = setupFileSystem(new URI("fs1:/"), FakeFileSystem.class);
fs2 = setupFileSystem(new URI("fs2:/"), FakeFileSystem.class);
viewFs = FileSystem.get(FsConstants.VIEWFS_URI, conf);
}
static FakeFileSystem setupFileSystem(URI uri, Class clazz)
throws Exception {
String scheme = uri.getScheme();
conf.set("fs."+scheme+".impl", clazz.getName());
FakeFileSystem fs = (FakeFileSystem)FileSystem.get(uri, conf);
assertEquals(uri, fs.getUri());
Path targetPath = FileSystemTestHelper.getAbsoluteTestRootPath(fs);
ConfigUtil.addLink(conf, "/mounts/"+scheme, targetPath.toUri());
return fs;
}
@Test
public void testSanity() {
assertEquals("fs1:/", fs1.getUri().toString());
assertEquals("fs2:/", fs2.getUri().toString());
}
@Test
public void testVerifyChecksum() throws Exception {
checkVerifyChecksum(false);
checkVerifyChecksum(true);
}
void checkVerifyChecksum(boolean flag) {
viewFs.setVerifyChecksum(flag);
assertEquals(flag, fs1.getVerifyChecksum());
assertEquals(flag, fs2.getVerifyChecksum());
}
static class FakeFileSystem extends LocalFileSystem {
boolean verifyChecksum = true;
URI uri;
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
super.initialize(uri, conf);
this.uri = uri;
}
@Override
public URI getUri() {
return uri;
}
@Override
public void setVerifyChecksum(boolean verifyChecksum) {
this.verifyChecksum = verifyChecksum;
}
public boolean getVerifyChecksum(){
return verifyChecksum;
}
}
}