Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-9631. ViewFs should use underlying FileSystem's server side defaults. Contributed by Lohit Vijayarenu and Erik Krogen.

Browse Source 
(cherry picked from commit 59d69257a888347f0fb9c51bb000afc986b64f98)
(cherry picked from commit f98f4bb6ca45e6f807ca747d673690232e39da52)
(cherry picked from commit 8ca9915d0f833bff6dae75fdcd071b6ae79b6320)
This commit is contained in:
Zhe Zhang 2017-03-22 22:21:30 -07:00
parent 1d035c8149
commit 4cc53b51f6
12 changed files with 131 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -120,6 +120,9 @@ Release 2.7.4 - UNRELEASED
HADOOP-13433 Race in UGI.reloginFromKeytab. (Duo Zhang via xiao)
HADOOP-9631. ViewFs should use underlying FileSystem's server side defaults.
(Lohit Vijayarenu and Erik Krogen via zhz)
Release 2.7.3 - 2016-08-25
INCOMPATIBLE CHANGES

16
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
View File

@ -449,9 +449,21 @@ public Path getHomeDirectory() {
* @return server default configuration values
*
* @throws IOException an I/O error occurred
* @deprecated use {@link #getServerDefaults(Path)} instead
*/
@Deprecated
public abstract FsServerDefaults getServerDefaults() throws IOException;
/**
* Return a set of server default configuration values based on path.
* @param f path to fetch server defaults
* @return server default configuration values for path
* @throws IOException an I/O error occurred
*/
public FsServerDefaults getServerDefaults(final Path f) throws IOException {
return getServerDefaults();
}
/**
* Return the fully-qualified path of path f resolving the path
* through any internal symlinks or mount point
@ -547,7 +559,7 @@ public final FSDataOutputStream create(final Path f,
}
FsServerDefaults ssDef = getServerDefaults();
FsServerDefaults ssDef = getServerDefaults(f);
if (ssDef.getBlockSize() % ssDef.getBytesPerChecksum() != 0) {
throw new IOException("Internal error: default blockSize is" +
" not a multiple of default bytesPerChecksum ");
@ -625,7 +637,7 @@ public abstract boolean delete(final Path f, final boolean recursive)
*/
public FSDataInputStream open(final Path f) throws AccessControlException,
FileNotFoundException, UnresolvedLinkException, IOException {
return open(f, getServerDefaults().getFileBufferSize());
return open(f, getServerDefaults(f).getFileBufferSize());
}
/**

13
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
View File

@ -57,7 +57,7 @@ public ChecksumFs(AbstractFileSystem theFs)
throws IOException, URISyntaxException {
super(theFs);
defaultBytesPerChecksum =
getMyFs().getServerDefaults().getBytesPerChecksum();
getMyFs().getServerDefaults(new Path("/")).getBytesPerChecksum();
}
/**
@ -96,9 +96,10 @@ public int getBytesPerSum() {
return defaultBytesPerChecksum;
}
private int getSumBufferSize(int bytesPerSum, int bufferSize)
private int getSumBufferSize(int bytesPerSum, int bufferSize, Path file)
throws IOException {
int defaultBufferSize = getMyFs().getServerDefaults().getFileBufferSize();
int defaultBufferSize = getMyFs().getServerDefaults(file)
.getFileBufferSize();
int proportionalBufferSize = bufferSize / bytesPerSum;
return Math.max(bytesPerSum,
Math.max(proportionalBufferSize, defaultBufferSize));
@ -121,7 +122,7 @@ private static class ChecksumFSInputChecker extends FSInputChecker {
public ChecksumFSInputChecker(ChecksumFs fs, Path file)
throws IOException, UnresolvedLinkException {
this(fs, file, fs.getServerDefaults().getFileBufferSize());
this(fs, file, fs.getServerDefaults(file).getFileBufferSize());
}
public ChecksumFSInputChecker(ChecksumFs fs, Path file, int bufferSize)
@ -132,7 +133,7 @@ public ChecksumFSInputChecker(ChecksumFs fs, Path file, int bufferSize)
Path sumFile = fs.getChecksumFile(file);
try {
int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(),
bufferSize);
bufferSize, file);
sums = fs.getRawFs().open(sumFile, sumBufferSize);
byte[] version = new byte[CHECKSUM_VERSION.length];
@ -355,7 +356,7 @@ public ChecksumFSOutputSummer(final ChecksumFs fs, final Path file,
// Now create the chekcsumfile; adjust the buffsize
int bytesPerSum = fs.getBytesPerSum();
int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize);
int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize, file);
this.sums = fs.getRawFs().createInternal(fs.getChecksumFile(file),
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
absolutePermission, sumBufferSize, replication, blockSize, progress,

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
View File

@ -149,10 +149,16 @@ public FsStatus getFsStatus(final Path f) throws IOException {
}
@Override
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
return fsImpl.getServerDefaults();
}
@Override
public FsServerDefaults getServerDefaults(final Path f) throws IOException {
return fsImpl.getServerDefaults(f);
}
@Override
public Path getHomeDirectory() {
return fsImpl.getHomeDirectory();

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
View File

@ -146,10 +146,15 @@ public FsStatus getFsStatus() throws IOException {
}
@Override
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
return myFs.getServerDefaults();
}
@Override
public FsServerDefaults getServerDefaults(final Path f) throws IOException {
return myFs.getServerDefaults(f);
}
@Override
public Path resolvePath(final Path p) throws FileNotFoundException,

7
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java
View File

@ -29,6 +29,7 @@
import org.apache.hadoop.fs.DelegateToFileSystem;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
/**
* The FtpFs implementation of AbstractFileSystem.
@ -57,7 +58,13 @@ public int getUriDefaultPort() {
}
@Override
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
return FtpConfigKeys.getServerDefaults();
}
@Override
public FsServerDefaults getServerDefaults(final Path f) throws IOException {
return FtpConfigKeys.getServerDefaults();
}
}

8
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java
View File

@ -28,6 +28,7 @@
import org.apache.hadoop.fs.DelegateToFileSystem;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
/**
@ -63,6 +64,13 @@ public int getUriDefaultPort() {
}
@Override
public FsServerDefaults getServerDefaults(final Path f)
throws IOException {
return LocalConfigKeys.getServerDefaults();
}
@Override
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
return LocalConfigKeys.getServerDefaults();
}

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
View File

@ -221,10 +221,16 @@ public FsStatus getFsStatus() throws IOException {
}
@Override
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
return myFs.getServerDefaults();
}
@Override
public FsServerDefaults getServerDefaults(final Path f) throws IOException {
return myFs.getServerDefaults(fullPath(f));
}
@Override
public int getUriDefaultPort() {
return myFs.getUriDefaultPort();

20
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
View File

@ -239,10 +239,22 @@ AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList)
}
@Override
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
return LocalConfigKeys.getServerDefaults();
}
@Override
public FsServerDefaults getServerDefaults(final Path f) throws IOException {
InodeTree.ResolveResult<AbstractFileSystem> res;
try {
res = fsState.resolve(getUriPath(f), true);
} catch (FileNotFoundException fnfe) {
return LocalConfigKeys.getServerDefaults();
}
return res.targetFileSystem.getServerDefaults(res.remainingPath);
}
@Override
public int getUriDefaultPort() {
return -1;
@ -871,8 +883,14 @@ public FsStatus getFsStatus() {
}
@Override
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
throw new IOException("FsServerDefaults not implemented yet");
return LocalConfigKeys.getServerDefaults();
}
@Override
public FsServerDefaults getServerDefaults(final Path f) throws IOException {
return LocalConfigKeys.getServerDefaults();
}
@Override

1
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
View File

@ -117,6 +117,7 @@ public FsStatus getFsStatus() throws IOException {
}
@Override
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
// deliberately empty
return null;

49
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
View File

@ -38,6 +38,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -45,8 +46,10 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextTestHelper;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.FileContextTestHelper.fileType;
@ -54,6 +57,7 @@
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.local.LocalConfigKeys;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.AclUtil;
@ -797,6 +801,51 @@ public void testInternalRemoveXAttr() throws IOException {
fcView.removeXAttr(new Path("/internalDir"), "xattrName");
}
@Test
public void testRespectsServerDefaults() throws Exception {
FsServerDefaults targetDefs =
fcTarget.getDefaultFileSystem().getServerDefaults(new Path("/"));
FsServerDefaults viewDefs =
fcView.getDefaultFileSystem().getServerDefaults(new Path("/data"));
assertEquals(targetDefs.getReplication(), viewDefs.getReplication());
assertEquals(targetDefs.getBlockSize(), viewDefs.getBlockSize());
assertEquals(targetDefs.getBytesPerChecksum(),
viewDefs.getBytesPerChecksum());
assertEquals(targetDefs.getFileBufferSize(),
viewDefs.getFileBufferSize());
assertEquals(targetDefs.getWritePacketSize(),
viewDefs.getWritePacketSize());
assertEquals(targetDefs.getEncryptDataTransfer(),
viewDefs.getEncryptDataTransfer());
assertEquals(targetDefs.getTrashInterval(), viewDefs.getTrashInterval());
assertEquals(targetDefs.getChecksumType(), viewDefs.getChecksumType());
fcView.create(new Path("/data/file"), EnumSet.of(CreateFlag.CREATE))
.close();
FileStatus stat =
fcTarget.getFileStatus(new Path(targetTestRoot, "data/file"));
assertEquals(targetDefs.getReplication(), stat.getReplication());
}
@Test
public void testServerDefaultsInternalDir() throws Exception {
FsServerDefaults localDefs = LocalConfigKeys.getServerDefaults();
FsServerDefaults viewDefs = fcView
.getDefaultFileSystem().getServerDefaults(new Path("/internalDir"));
assertEquals(localDefs.getReplication(), viewDefs.getReplication());
assertEquals(localDefs.getBlockSize(), viewDefs.getBlockSize());
assertEquals(localDefs.getBytesPerChecksum(),
viewDefs.getBytesPerChecksum());
assertEquals(localDefs.getFileBufferSize(),
viewDefs.getFileBufferSize());
assertEquals(localDefs.getWritePacketSize(),
viewDefs.getWritePacketSize());
assertEquals(localDefs.getEncryptDataTransfer(),
viewDefs.getEncryptDataTransfer());
assertEquals(localDefs.getTrashInterval(), viewDefs.getTrashInterval());
assertEquals(localDefs.getChecksumType(), viewDefs.getChecksumType());
}
// Confirm that listLocatedStatus is delegated properly to the underlying
// AbstractFileSystem to allow for optimizations
@Test

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
View File

@ -152,10 +152,16 @@ public FsStatus getFsStatus() throws IOException {
}
@Override
@Deprecated
public FsServerDefaults getServerDefaults() throws IOException {
return dfs.getServerDefaults();
}
@Override
public FsServerDefaults getServerDefaults(final Path f) throws IOException {
return dfs.getServerDefaults();
}
@Override
public RemoteIterator<LocatedFileStatus> listLocatedStatus(
final Path p)