HADOOP-14455. ViewFileSystem#rename should support be supported within same nameservice with different mountpoints. Contributed by Brahma Reddy Battula.
This commit is contained in:
parent
1a79dcfc45
commit
6d983cca52
|
@ -66,4 +66,6 @@ public interface Constants {
|
||||||
|
|
||||||
static public final FsPermission PERMISSION_555 =
|
static public final FsPermission PERMISSION_555 =
|
||||||
new FsPermission((short) 0555);
|
new FsPermission((short) 0555);
|
||||||
|
|
||||||
|
String CONFIG_VIEWFS_RENAME_STRATEGY = "fs.viewfs.rename.strategy";
|
||||||
}
|
}
|
||||||
|
|
|
@ -126,7 +126,8 @@ public class ViewFileSystem extends FileSystem {
|
||||||
Configuration config;
|
Configuration config;
|
||||||
InodeTree<FileSystem> fsState; // the fs state; ie the mount table
|
InodeTree<FileSystem> fsState; // the fs state; ie the mount table
|
||||||
Path homeDir = null;
|
Path homeDir = null;
|
||||||
|
// Default to rename within same mountpoint
|
||||||
|
private RenameStrategy renameStrategy = RenameStrategy.SAME_MOUNTPOINT;
|
||||||
/**
|
/**
|
||||||
* Make the path Absolute and get the path-part of a pathname.
|
* Make the path Absolute and get the path-part of a pathname.
|
||||||
* Checks that URI matches this file system
|
* Checks that URI matches this file system
|
||||||
|
@ -207,6 +208,9 @@ public class ViewFileSystem extends FileSystem {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
workingDir = this.getHomeDirectory();
|
workingDir = this.getHomeDirectory();
|
||||||
|
renameStrategy = RenameStrategy.valueOf(
|
||||||
|
conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
|
||||||
|
RenameStrategy.SAME_MOUNTPOINT.toString()));
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
throw new IOException("URISyntax exception: " + theUri);
|
throw new IOException("URISyntax exception: " + theUri);
|
||||||
}
|
}
|
||||||
|
@ -490,27 +494,55 @@ public class ViewFileSystem extends FileSystem {
|
||||||
if (resDst.isInternalDir()) {
|
if (resDst.isInternalDir()) {
|
||||||
throw readOnlyMountTable("rename", dst);
|
throw readOnlyMountTable("rename", dst);
|
||||||
}
|
}
|
||||||
/**
|
|
||||||
// Alternate 1: renames within same file system - valid but we disallow
|
URI srcUri = resSrc.targetFileSystem.getUri();
|
||||||
// Alternate 2: (as described in next para - valid but we have disallowed it
|
URI dstUri = resDst.targetFileSystem.getUri();
|
||||||
//
|
|
||||||
// Note we compare the URIs. the URIs include the link targets.
|
verifyRenameStrategy(srcUri, dstUri,
|
||||||
// hence we allow renames across mount links as long as the mount links
|
resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy);
|
||||||
// point to the same target.
|
|
||||||
if (!resSrc.targetFileSystem.getUri().equals(
|
ChRootedFileSystem srcFS = (ChRootedFileSystem) resSrc.targetFileSystem;
|
||||||
resDst.targetFileSystem.getUri())) {
|
ChRootedFileSystem dstFS = (ChRootedFileSystem) resDst.targetFileSystem;
|
||||||
throw new IOException("Renames across Mount points not supported");
|
return srcFS.getMyFs().rename(srcFS.fullPath(resSrc.remainingPath),
|
||||||
|
dstFS.fullPath(resDst.remainingPath));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void verifyRenameStrategy(URI srcUri, URI dstUri,
|
||||||
|
boolean isSrcDestSame, ViewFileSystem.RenameStrategy renameStrategy)
|
||||||
|
throws IOException {
|
||||||
|
switch (renameStrategy) {
|
||||||
|
case SAME_FILESYSTEM_ACROSS_MOUNTPOINT:
|
||||||
|
if (srcUri.getAuthority() != null) {
|
||||||
|
if (!(srcUri.getScheme().equals(dstUri.getScheme()) && srcUri
|
||||||
|
.getAuthority().equals(dstUri.getAuthority()))) {
|
||||||
|
throw new IOException("Renames across Mount points not supported");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
case SAME_TARGET_URI_ACROSS_MOUNTPOINT:
|
||||||
|
// Alternate 2: Rename across mountpoints with same target.
|
||||||
|
// i.e. Rename across alias mountpoints.
|
||||||
|
//
|
||||||
|
// Note we compare the URIs. the URIs include the link targets.
|
||||||
|
// hence we allow renames across mount links as long as the mount links
|
||||||
|
// point to the same target.
|
||||||
|
if (!srcUri.equals(dstUri)) {
|
||||||
|
throw new IOException("Renames across Mount points not supported");
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
case SAME_MOUNTPOINT:
|
||||||
|
//
|
||||||
|
// Alternate 3 : renames ONLY within the the same mount links.
|
||||||
|
//
|
||||||
|
if (!isSrcDestSame) {
|
||||||
|
throw new IOException("Renames across Mount points not supported");
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException ("Unexpected rename strategy");
|
||||||
}
|
}
|
||||||
*/
|
|
||||||
|
|
||||||
//
|
|
||||||
// Alternate 3 : renames ONLY within the the same mount links.
|
|
||||||
//
|
|
||||||
if (resSrc.targetFileSystem !=resDst.targetFileSystem) {
|
|
||||||
throw new IOException("Renames across Mount points not supported");
|
|
||||||
}
|
|
||||||
return resSrc.targetFileSystem.rename(resSrc.remainingPath,
|
|
||||||
resDst.remainingPath);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -1241,4 +1273,9 @@ public class ViewFileSystem extends FileSystem {
|
||||||
return allPolicies;
|
return allPolicies;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum RenameStrategy {
|
||||||
|
SAME_MOUNTPOINT, SAME_TARGET_URI_ACROSS_MOUNTPOINT,
|
||||||
|
SAME_FILESYSTEM_ACROSS_MOUNTPOINT
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -157,7 +157,9 @@ public class ViewFs extends AbstractFileSystem {
|
||||||
final Configuration config;
|
final Configuration config;
|
||||||
InodeTree<AbstractFileSystem> fsState; // the fs state; ie the mount table
|
InodeTree<AbstractFileSystem> fsState; // the fs state; ie the mount table
|
||||||
Path homeDir = null;
|
Path homeDir = null;
|
||||||
|
private ViewFileSystem.RenameStrategy renameStrategy =
|
||||||
|
ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT;
|
||||||
|
|
||||||
static AccessControlException readOnlyMountTable(final String operation,
|
static AccessControlException readOnlyMountTable(final String operation,
|
||||||
final String p) {
|
final String p) {
|
||||||
return new AccessControlException(
|
return new AccessControlException(
|
||||||
|
@ -237,6 +239,9 @@ public class ViewFs extends AbstractFileSystem {
|
||||||
// return MergeFs.createMergeFs(mergeFsURIList, config);
|
// return MergeFs.createMergeFs(mergeFsURIList, config);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
renameStrategy = ViewFileSystem.RenameStrategy.valueOf(
|
||||||
|
conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
|
||||||
|
ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT.toString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -495,37 +500,23 @@ public class ViewFs extends AbstractFileSystem {
|
||||||
+ " is readOnly");
|
+ " is readOnly");
|
||||||
}
|
}
|
||||||
|
|
||||||
InodeTree.ResolveResult<AbstractFileSystem> resDst =
|
InodeTree.ResolveResult<AbstractFileSystem> resDst =
|
||||||
fsState.resolve(getUriPath(dst), false);
|
fsState.resolve(getUriPath(dst), false);
|
||||||
if (resDst.isInternalDir()) {
|
if (resDst.isInternalDir()) {
|
||||||
throw new AccessControlException(
|
throw new AccessControlException(
|
||||||
"Cannot Rename within internal dirs of mount table: dest=" + dst
|
"Cannot Rename within internal dirs of mount table: dest=" + dst
|
||||||
+ " is readOnly");
|
+ " is readOnly");
|
||||||
}
|
}
|
||||||
|
//Alternate 1: renames within same file system
|
||||||
/**
|
URI srcUri = resSrc.targetFileSystem.getUri();
|
||||||
// Alternate 1: renames within same file system - valid but we disallow
|
URI dstUri = resDst.targetFileSystem.getUri();
|
||||||
// Alternate 2: (as described in next para - valid but we have disallowed it
|
ViewFileSystem.verifyRenameStrategy(srcUri, dstUri,
|
||||||
//
|
resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy);
|
||||||
// Note we compare the URIs. the URIs include the link targets.
|
|
||||||
// hence we allow renames across mount links as long as the mount links
|
|
||||||
// point to the same target.
|
|
||||||
if (!resSrc.targetFileSystem.getUri().equals(
|
|
||||||
resDst.targetFileSystem.getUri())) {
|
|
||||||
throw new IOException("Renames across Mount points not supported");
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
//
|
|
||||||
// Alternate 3 : renames ONLY within the the same mount links.
|
|
||||||
//
|
|
||||||
|
|
||||||
if (resSrc.targetFileSystem !=resDst.targetFileSystem) {
|
ChRootedFs srcFS = (ChRootedFs) resSrc.targetFileSystem;
|
||||||
throw new IOException("Renames across Mount points not supported");
|
ChRootedFs dstFS = (ChRootedFs) resDst.targetFileSystem;
|
||||||
}
|
srcFS.getMyFs().renameInternal(srcFS.fullPath(resSrc.remainingPath),
|
||||||
|
dstFS.fullPath(resDst.remainingPath), overwrite);
|
||||||
resSrc.targetFileSystem.renameInternal(resSrc.remainingPath,
|
|
||||||
resDst.remainingPath, overwrite);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -800,6 +800,15 @@
|
||||||
(ie client side mount table:).</description>
|
(ie client side mount table:).</description>
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>fs.viewfs.rename.strategy</name>
|
||||||
|
<value>SAME_MOUNTPOINT</value>
|
||||||
|
<description>Allowed rename strategy to rename between multiple mountpoints.
|
||||||
|
Allowed values are SAME_MOUNTPOINT,SAME_TARGET_URI_ACROSS_MOUNTPOINT and
|
||||||
|
SAME_FILESYSTEM_ACROSS_MOUNTPOINT.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<name>fs.AbstractFileSystem.ftp.impl</name>
|
<name>fs.AbstractFileSystem.ftp.impl</name>
|
||||||
<value>org.apache.hadoop.fs.ftp.FtpFs</value>
|
<value>org.apache.hadoop.fs.ftp.FtpFs</value>
|
||||||
|
|
|
@ -95,6 +95,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
|
||||||
xmlPropsToSkipCompare.add("nfs3.mountd.port");
|
xmlPropsToSkipCompare.add("nfs3.mountd.port");
|
||||||
xmlPropsToSkipCompare.add("nfs3.server.port");
|
xmlPropsToSkipCompare.add("nfs3.server.port");
|
||||||
xmlPropsToSkipCompare.add("test.fs.s3n.name");
|
xmlPropsToSkipCompare.add("test.fs.s3n.name");
|
||||||
|
xmlPropsToSkipCompare.add("fs.viewfs.rename.strategy");
|
||||||
|
|
||||||
// S3N/S3A properties are in a different subtree.
|
// S3N/S3A properties are in a different subtree.
|
||||||
// - org.apache.hadoop.fs.s3native.S3NativeFileSystemConfigKeys
|
// - org.apache.hadoop.fs.s3native.S3NativeFileSystemConfigKeys
|
||||||
|
|
|
@ -20,6 +20,7 @@ package org.apache.hadoop.fs.contract;
|
||||||
|
|
||||||
import org.apache.hadoop.fs.FSDataInputStream;
|
import org.apache.hadoop.fs.FSDataInputStream;
|
||||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
|
import org.apache.hadoop.fs.FileContext;
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.LocatedFileStatus;
|
import org.apache.hadoop.fs.LocatedFileStatus;
|
||||||
|
@ -715,6 +716,21 @@ public class ContractTestUtils extends Assert {
|
||||||
assertIsFile(filename, status);
|
assertIsFile(filename, status);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Assert that a file exists and whose {@link FileStatus} entry
|
||||||
|
* declares that this is a file and not a symlink or directory.
|
||||||
|
*
|
||||||
|
* @param fileContext filesystem to resolve path against
|
||||||
|
* @param filename name of the file
|
||||||
|
* @throws IOException IO problems during file operations
|
||||||
|
*/
|
||||||
|
public static void assertIsFile(FileContext fileContext, Path filename)
|
||||||
|
throws IOException {
|
||||||
|
assertPathExists(fileContext, "Expected file", filename);
|
||||||
|
FileStatus status = fileContext.getFileStatus(filename);
|
||||||
|
assertIsFile(filename, status);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Assert that a file exists and whose {@link FileStatus} entry
|
* Assert that a file exists and whose {@link FileStatus} entry
|
||||||
* declares that this is a file and not a symlink or directory.
|
* declares that this is a file and not a symlink or directory.
|
||||||
|
@ -765,6 +781,25 @@ public class ContractTestUtils extends Assert {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Assert that a path exists -but make no assertions as to the
|
||||||
|
* type of that entry.
|
||||||
|
*
|
||||||
|
* @param fileContext fileContext to examine
|
||||||
|
* @param message message to include in the assertion failure message
|
||||||
|
* @param path path in the filesystem
|
||||||
|
* @throws FileNotFoundException raised if the path is missing
|
||||||
|
* @throws IOException IO problems
|
||||||
|
*/
|
||||||
|
public static void assertPathExists(FileContext fileContext, String message,
|
||||||
|
Path path) throws IOException {
|
||||||
|
if (!fileContext.util().exists(path)) {
|
||||||
|
//failure, report it
|
||||||
|
throw new FileNotFoundException(
|
||||||
|
message + ": not found " + path + " in " + path.getParent());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Assert that a path does not exist.
|
* Assert that a path does not exist.
|
||||||
*
|
*
|
||||||
|
@ -785,6 +820,25 @@ public class ContractTestUtils extends Assert {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Assert that a path does not exist.
|
||||||
|
*
|
||||||
|
* @param fileContext fileContext to examine
|
||||||
|
* @param message message to include in the assertion failure message
|
||||||
|
* @param path path in the filesystem
|
||||||
|
* @throws IOException IO problems
|
||||||
|
*/
|
||||||
|
public static void assertPathDoesNotExist(FileContext fileContext,
|
||||||
|
String message, Path path) throws IOException {
|
||||||
|
try {
|
||||||
|
FileStatus status = fileContext.getFileStatus(path);
|
||||||
|
fail(message + ": unexpectedly found " + path + " as " + status);
|
||||||
|
} catch (FileNotFoundException expected) {
|
||||||
|
//this is expected
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Assert that a FileSystem.listStatus on a dir finds the subdir/child entry.
|
* Assert that a FileSystem.listStatus on a dir finds the subdir/child entry.
|
||||||
* @param fs filesystem
|
* @param fs filesystem
|
||||||
|
|
|
@ -41,6 +41,7 @@ import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.RemoteIterator;
|
import org.apache.hadoop.fs.RemoteIterator;
|
||||||
import org.apache.hadoop.fs.Trash;
|
import org.apache.hadoop.fs.Trash;
|
||||||
import org.apache.hadoop.fs.UnsupportedFileSystemException;
|
import org.apache.hadoop.fs.UnsupportedFileSystemException;
|
||||||
|
import org.apache.hadoop.fs.contract.ContractTestUtils;
|
||||||
import org.apache.hadoop.fs.permission.AclEntry;
|
import org.apache.hadoop.fs.permission.AclEntry;
|
||||||
import org.apache.hadoop.fs.permission.AclStatus;
|
import org.apache.hadoop.fs.permission.AclStatus;
|
||||||
import org.apache.hadoop.fs.permission.AclUtil;
|
import org.apache.hadoop.fs.permission.AclUtil;
|
||||||
|
@ -51,6 +52,7 @@ import org.apache.hadoop.security.AccessControlException;
|
||||||
import org.apache.hadoop.security.Credentials;
|
import org.apache.hadoop.security.Credentials;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.security.token.Token;
|
import org.apache.hadoop.security.token.Token;
|
||||||
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
import org.junit.Assume;
|
import org.junit.Assume;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
@ -366,28 +368,83 @@ abstract public class ViewFileSystemBaseTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
// rename across mount points that point to same target also fail
|
// rename across mount points that point to same target also fail
|
||||||
@Test(expected=IOException.class)
|
@Test
|
||||||
public void testRenameAcrossMounts1() throws IOException {
|
public void testRenameAcrossMounts1() throws IOException {
|
||||||
fileSystemTestHelper.createFile(fsView, "/user/foo");
|
fileSystemTestHelper.createFile(fsView, "/user/foo");
|
||||||
fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
|
try {
|
||||||
/* - code if we had wanted this to succeed
|
fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
|
||||||
Assert.assertFalse(fSys.exists(new Path("/user/foo")));
|
ContractTestUtils.fail("IOException is not thrown on rename operation");
|
||||||
Assert.assertFalse(fSysLocal.exists(new Path(targetTestRoot,"user/foo")));
|
} catch (IOException e) {
|
||||||
Assert.assertTrue(fSys.isFile(FileSystemTestHelper.getTestRootPath(fSys,"/user2/fooBarBar")));
|
GenericTestUtils
|
||||||
Assert.assertTrue(fSysLocal.isFile(new Path(targetTestRoot,"user/fooBarBar")));
|
.assertExceptionContains("Renames across Mount points not supported",
|
||||||
*/
|
e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// rename across mount points fail if the mount link targets are different
|
// rename across mount points fail if the mount link targets are different
|
||||||
// even if the targets are part of the same target FS
|
// even if the targets are part of the same target FS
|
||||||
|
|
||||||
@Test(expected=IOException.class)
|
@Test
|
||||||
public void testRenameAcrossMounts2() throws IOException {
|
public void testRenameAcrossMounts2() throws IOException {
|
||||||
fileSystemTestHelper.createFile(fsView, "/user/foo");
|
fileSystemTestHelper.createFile(fsView, "/user/foo");
|
||||||
fsView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
|
try {
|
||||||
|
fsView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
|
||||||
|
ContractTestUtils.fail("IOException is not thrown on rename operation");
|
||||||
|
} catch (IOException e) {
|
||||||
|
GenericTestUtils
|
||||||
|
.assertExceptionContains("Renames across Mount points not supported",
|
||||||
|
e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled
|
||||||
|
// to rename across mount points that point to same target URI
|
||||||
|
@Test
|
||||||
|
public void testRenameAcrossMounts3() throws IOException {
|
||||||
|
Configuration conf2 = new Configuration(conf);
|
||||||
|
conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
|
||||||
|
ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT
|
||||||
|
.toString());
|
||||||
|
FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2);
|
||||||
|
fileSystemTestHelper.createFile(fsView2, "/user/foo");
|
||||||
|
fsView2.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
|
||||||
|
ContractTestUtils
|
||||||
|
.assertPathDoesNotExist(fsView2, "src should not exist after rename",
|
||||||
|
new Path("/user/foo"));
|
||||||
|
ContractTestUtils
|
||||||
|
.assertPathDoesNotExist(fsTarget, "src should not exist after rename",
|
||||||
|
new Path(targetTestRoot, "user/foo"));
|
||||||
|
ContractTestUtils.assertIsFile(fsView2,
|
||||||
|
fileSystemTestHelper.getTestRootPath(fsView2, "/user2/fooBarBar"));
|
||||||
|
ContractTestUtils
|
||||||
|
.assertIsFile(fsTarget, new Path(targetTestRoot, "user/fooBarBar"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled
|
||||||
|
// to rename across mount points where the mount link targets are different
|
||||||
|
// but are part of the same target FS
|
||||||
|
@Test
|
||||||
|
public void testRenameAcrossMounts4() throws IOException {
|
||||||
|
Configuration conf2 = new Configuration(conf);
|
||||||
|
conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
|
||||||
|
ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT
|
||||||
|
.toString());
|
||||||
|
FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2);
|
||||||
|
fileSystemTestHelper.createFile(fsView2, "/user/foo");
|
||||||
|
fsView2.rename(new Path("/user/foo"), new Path("/data/fooBar"));
|
||||||
|
ContractTestUtils
|
||||||
|
.assertPathDoesNotExist(fsView2, "src should not exist after rename",
|
||||||
|
new Path("/user/foo"));
|
||||||
|
ContractTestUtils
|
||||||
|
.assertPathDoesNotExist(fsTarget, "src should not exist after rename",
|
||||||
|
new Path(targetTestRoot, "user/foo"));
|
||||||
|
ContractTestUtils.assertIsFile(fsView2,
|
||||||
|
fileSystemTestHelper.getTestRootPath(fsView2, "/data/fooBar"));
|
||||||
|
ContractTestUtils
|
||||||
|
.assertIsFile(fsTarget, new Path(targetTestRoot, "data/fooBar"));
|
||||||
|
}
|
||||||
|
|
||||||
static protected boolean SupportsBlocks = false; // local fs use 1 block
|
static protected boolean SupportsBlocks = false; // local fs use 1 block
|
||||||
// override for HDFS
|
// override for HDFS
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -58,6 +58,7 @@ import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.FsConstants;
|
import org.apache.hadoop.fs.FsConstants;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.UnresolvedLinkException;
|
import org.apache.hadoop.fs.UnresolvedLinkException;
|
||||||
|
import org.apache.hadoop.fs.contract.ContractTestUtils;
|
||||||
import org.apache.hadoop.fs.local.LocalConfigKeys;
|
import org.apache.hadoop.fs.local.LocalConfigKeys;
|
||||||
import org.apache.hadoop.fs.permission.AclEntry;
|
import org.apache.hadoop.fs.permission.AclEntry;
|
||||||
import org.apache.hadoop.fs.permission.AclStatus;
|
import org.apache.hadoop.fs.permission.AclStatus;
|
||||||
|
@ -66,6 +67,7 @@ import org.apache.hadoop.fs.viewfs.ViewFs.MountPoint;
|
||||||
import org.apache.hadoop.security.AccessControlException;
|
import org.apache.hadoop.security.AccessControlException;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.security.token.Token;
|
import org.apache.hadoop.security.token.Token;
|
||||||
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
@ -345,33 +347,93 @@ abstract public class ViewFsBaseTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
// rename across mount points that point to same target also fail
|
// rename across mount points that point to same target also fail
|
||||||
@Test(expected=IOException.class)
|
@Test
|
||||||
public void testRenameAcrossMounts1() throws IOException {
|
public void testRenameAcrossMounts1() throws IOException {
|
||||||
fileContextTestHelper.createFile(fcView, "/user/foo");
|
fileContextTestHelper.createFile(fcView, "/user/foo");
|
||||||
fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
|
try {
|
||||||
/* - code if we had wanted this to succeed
|
fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
|
||||||
Assert.assertFalse(exists(fc, new Path("/user/foo")));
|
ContractTestUtils.fail("IOException is not thrown on rename operation");
|
||||||
Assert.assertFalse(exists(fclocal, new Path(targetTestRoot,"user/foo")));
|
} catch (IOException e) {
|
||||||
Assert.assertTrue(isFile(fc,
|
GenericTestUtils
|
||||||
FileContextTestHelper.getTestRootPath(fc,"/user2/fooBarBar")));
|
.assertExceptionContains("Renames across Mount points not supported",
|
||||||
Assert.assertTrue(isFile(fclocal,
|
e);
|
||||||
new Path(targetTestRoot,"user/fooBarBar")));
|
}
|
||||||
*/
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// rename across mount points fail if the mount link targets are different
|
// rename across mount points fail if the mount link targets are different
|
||||||
// even if the targets are part of the same target FS
|
// even if the targets are part of the same target FS
|
||||||
|
|
||||||
@Test(expected=IOException.class)
|
@Test
|
||||||
public void testRenameAcrossMounts2() throws IOException {
|
public void testRenameAcrossMounts2() throws IOException {
|
||||||
fileContextTestHelper.createFile(fcView, "/user/foo");
|
fileContextTestHelper.createFile(fcView, "/user/foo");
|
||||||
fcView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
|
try {
|
||||||
|
fcView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
|
||||||
|
ContractTestUtils.fail("IOException is not thrown on rename operation");
|
||||||
|
} catch (IOException e) {
|
||||||
|
GenericTestUtils
|
||||||
|
.assertExceptionContains("Renames across Mount points not supported",
|
||||||
|
e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled
|
||||||
|
// to rename across mount points that point to same target URI
|
||||||
|
@Test
|
||||||
|
public void testRenameAcrossMounts3() throws IOException {
|
||||||
|
Configuration conf2 = new Configuration(conf);
|
||||||
|
conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
|
||||||
|
ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT
|
||||||
|
.toString());
|
||||||
|
|
||||||
|
FileContext fcView2 =
|
||||||
|
FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2);
|
||||||
|
String user1Path = "/user/foo";
|
||||||
|
fileContextTestHelper.createFile(fcView2, user1Path);
|
||||||
|
String user2Path = "/user2/fooBarBar";
|
||||||
|
Path user2Dst = new Path(user2Path);
|
||||||
|
fcView2.rename(new Path(user1Path), user2Dst);
|
||||||
|
ContractTestUtils
|
||||||
|
.assertPathDoesNotExist(fcView2, "src should not exist after rename",
|
||||||
|
new Path(user1Path));
|
||||||
|
ContractTestUtils
|
||||||
|
.assertPathDoesNotExist(fcTarget, "src should not exist after rename",
|
||||||
|
new Path(targetTestRoot, "user/foo"));
|
||||||
|
ContractTestUtils.assertIsFile(fcView2,
|
||||||
|
fileContextTestHelper.getTestRootPath(fcView2, user2Path));
|
||||||
|
ContractTestUtils
|
||||||
|
.assertIsFile(fcTarget, new Path(targetTestRoot, "user/fooBarBar"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled
|
||||||
|
// to rename across mount points if the mount link targets are different
|
||||||
|
// but are part of the same target FS
|
||||||
|
@Test
|
||||||
|
public void testRenameAcrossMounts4() throws IOException {
|
||||||
|
Configuration conf2 = new Configuration(conf);
|
||||||
|
conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
|
||||||
|
ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT
|
||||||
|
.toString());
|
||||||
|
FileContext fcView2 =
|
||||||
|
FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2);
|
||||||
|
String userPath = "/user/foo";
|
||||||
|
fileContextTestHelper.createFile(fcView2, userPath);
|
||||||
|
String anotherMountPath = "/data/fooBar";
|
||||||
|
Path anotherDst = new Path(anotherMountPath);
|
||||||
|
fcView2.rename(new Path(userPath), anotherDst);
|
||||||
|
|
||||||
|
ContractTestUtils
|
||||||
|
.assertPathDoesNotExist(fcView2, "src should not exist after rename",
|
||||||
|
new Path(userPath));
|
||||||
|
ContractTestUtils
|
||||||
|
.assertPathDoesNotExist(fcTarget, "src should not exist after rename",
|
||||||
|
new Path(targetTestRoot, "user/foo"));
|
||||||
|
ContractTestUtils.assertIsFile(fcView2,
|
||||||
|
fileContextTestHelper.getTestRootPath(fcView2, anotherMountPath));
|
||||||
|
ContractTestUtils
|
||||||
|
.assertIsFile(fcView2, new Path(targetTestRoot, "data/fooBar"));
|
||||||
|
}
|
||||||
|
|
||||||
static protected boolean SupportsBlocks = false; // local fs use 1 block
|
static protected boolean SupportsBlocks = false; // local fs use 1 block
|
||||||
// override for HDFS
|
// override for HDFS
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileSystemTestHelper;
|
||||||
import org.apache.hadoop.fs.FsConstants;
|
import org.apache.hadoop.fs.FsConstants;
|
||||||
import org.apache.hadoop.fs.FsShell;
|
import org.apache.hadoop.fs.FsShell;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.fs.contract.ContractTestUtils;
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
|
@ -45,6 +46,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology;
|
||||||
import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
|
import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
|
||||||
import org.apache.hadoop.hdfs.client.HdfsAdmin;
|
import org.apache.hadoop.hdfs.client.HdfsAdmin;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.AfterClass;
|
import org.junit.AfterClass;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
|
@ -247,4 +249,24 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest {
|
||||||
Assert.assertTrue("File checksum not matching!",
|
Assert.assertTrue("File checksum not matching!",
|
||||||
fileChecksumViaViewFs.equals(fileChecksumViaTargetFs));
|
fileChecksumViaViewFs.equals(fileChecksumViaTargetFs));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//Rename should fail on across different fileSystems
|
||||||
|
@Test
|
||||||
|
public void testRenameAccorssFilesystem() throws IOException {
|
||||||
|
//data is mountpoint in nn1
|
||||||
|
Path mountDataRootPath = new Path("/data");
|
||||||
|
//mountOnNn2 is nn2 mountpoint
|
||||||
|
Path fsTargetFilePath = new Path("/mountOnNn2");
|
||||||
|
Path filePath = new Path(mountDataRootPath + "/ttest");
|
||||||
|
Path hdfFilepath = new Path(fsTargetFilePath + "/ttest2");
|
||||||
|
fsView.create(filePath);
|
||||||
|
try {
|
||||||
|
fsView.rename(filePath, hdfFilepath);
|
||||||
|
ContractTestUtils.fail("Should thrown IOE on Renames across filesytems");
|
||||||
|
} catch (IOException e) {
|
||||||
|
GenericTestUtils
|
||||||
|
.assertExceptionContains("Renames across Mount points not supported",
|
||||||
|
e);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue