Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-10035. Cleanup TestFilterFileSystem. Contributed by Suresh Srinivas.

This commit is contained in:
Haohui Mai 2015-11-22 17:30:08 -08:00
parent ac0694eb6e
commit 6ed8f24041
2 changed files with 96 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -343,6 +343,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-12572. Update Hadoop's lz4 to r131. (Kevin Bowling via wheat9) HADOOP-12572. Update Hadoop's lz4 to r131. (Kevin Bowling via wheat9)
HADOOP-10035. Cleanup TestFilterFileSystem. (Suresh Srinivas via wheat9)
OPTIMIZATIONS OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp HADOOP-11785. Reduce the number of listStatus operation in distcp

261
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
View File

@ -32,7 +32,6 @@
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.Credentials;
@ -52,190 +51,118 @@ public static void setup() {
conf.setBoolean("fs.flfs.impl.disable.cache", true); conf.setBoolean("fs.flfs.impl.disable.cache", true);
conf.setBoolean("fs.file.impl.disable.cache", true); conf.setBoolean("fs.file.impl.disable.cache", true);
} }
public static class DontCheck { /**
public BlockLocation[] getFileBlockLocations(Path p, * FileSystem methods that must not be overwritten by
long start, long len) { return null; } * {@link FilterFileSystem}. Either because there is a default implementation
public FsServerDefaults getServerDefaults() { return null; } * already available or because it is not relevant.
public long getLength(Path f) { return 0; } */
public FSDataOutputStream append(Path f) { return null; } public static interface MustNotImplement {
public FSDataOutputStream append(Path f, int bufferSize) { return null; } public BlockLocation[] getFileBlockLocations(Path p, long start,
public void rename(final Path src, final Path dst, final Rename... options) { } long len);
public boolean exists(Path f) { return false; } public FSDataOutputStream append(Path f) throws IOException;
public boolean isDirectory(Path f) { return false; } public FSDataOutputStream append(Path f, int bufferSize) throws
public boolean isFile(Path f) { return false; } IOException;
public boolean createNewFile(Path f) { return false; } public long getLength(Path f);
public void rename(Path src, Path dst, Rename... options);
public boolean exists(Path f);
public boolean isDirectory(Path f);
public boolean isFile(Path f);
public boolean createNewFile(Path f);
public FSDataOutputStream createNonRecursive(Path f, public FSDataOutputStream createNonRecursive(Path f,
boolean overwrite, boolean overwrite, int bufferSize, short replication, long blockSize,
int bufferSize, short replication, long blockSize, Progressable progress) throws IOException;
Progressable progress) throws IOException {
return null;
}
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
boolean overwrite, int bufferSize, short replication, long blockSize, boolean overwrite, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException { Progressable progress) throws IOException;
return null;
} public boolean mkdirs(Path f);
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, public FSDataInputStream open(Path f);
EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, public FSDataOutputStream create(Path f);
Progressable progress) throws IOException { public FSDataOutputStream create(Path f, boolean overwrite);
return null; public FSDataOutputStream create(Path f, Progressable progress);
} public FSDataOutputStream create(Path f, short replication);
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, public FSDataOutputStream create(Path f, short replication,
EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress);
Progressable progress, ChecksumOpt checksumOpt) throws IOException { public FSDataOutputStream create(Path f, boolean overwrite,
return null; int bufferSize);
} public FSDataOutputStream create(Path f, boolean overwrite,
public boolean mkdirs(Path f) { return false; } int bufferSize, Progressable progress);
public FSDataInputStream open(Path f) { return null; } public FSDataOutputStream create(Path f, boolean overwrite,
public FSDataOutputStream create(Path f) { return null; } int bufferSize, short replication, long blockSize);
public FSDataOutputStream create(Path f, boolean overwrite) { return null; } public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
public FSDataOutputStream create(Path f, Progressable progress) { short replication, long blockSize, Progressable progress);
return null; public FSDataOutputStream create(Path f, FsPermission permission,
} EnumSet<CreateFlag> flags, int bufferSize, short replication,
public FSDataOutputStream create(Path f, short replication) { long blockSize, Progressable progress);
return null; public String getName();
} public boolean delete(Path f);
public FSDataOutputStream create(Path f, short replication, public short getReplication(Path src);
Progressable progress) { public void processDeleteOnExit();
return null; public FsStatus getStatus();
} public FileStatus[] listStatus(Path f, PathFilter filter);
public FSDataOutputStream create(Path f, public FileStatus[] listStatus(Path[] files);
boolean overwrite, public FileStatus[] listStatus(Path[] files, PathFilter filter);
int bufferSize) { public FileStatus[] globStatus(Path pathPattern);
return null; public FileStatus[] globStatus(Path pathPattern, PathFilter filter);
} public Iterator<LocatedFileStatus> listFiles(Path path,
public FSDataOutputStream create(Path f, boolean isRecursive);
boolean overwrite, public void copyFromLocalFile(Path src, Path dst);
int bufferSize, public void moveFromLocalFile(Path[] srcs, Path dst);
Progressable progress) { public void moveFromLocalFile(Path src, Path dst);
return null; public void copyToLocalFile(Path src, Path dst);
}
public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize,
short replication,
long blockSize) {
return null;
}
public FSDataOutputStream create(Path f,
boolean overwrite,
int bufferSize,
short replication,
long blockSize,
Progressable progress) {
return null;
}
public FSDataOutputStream create(Path f,
FsPermission permission,
boolean overwrite,
int bufferSize,
short replication,
long blockSize,
Progressable progress) {
return null;
}
public FSDataOutputStream create(Path f,
FsPermission permission,
EnumSet<CreateFlag> flags,
int bufferSize,
short replication,
long blockSize,
Progressable progress) throws IOException {
return null;
}
public FSDataOutputStream create(Path f,
FsPermission permission,
EnumSet<CreateFlag> flags,
int bufferSize,
short replication,
long blockSize,
Progressable progress,
ChecksumOpt checksumOpt) throws IOException {
return null;
}
public String getName() { return null; }
public boolean delete(Path f) { return false; }
public short getReplication(Path src) { return 0 ; }
public void processDeleteOnExit() { }
public ContentSummary getContentSummary(Path f) { return null; }
public FsStatus getStatus() { return null; }
public FileStatus[] listStatus(Path f, PathFilter filter) { return null; }
public FileStatus[] listStatus(Path[] files) { return null; }
public FileStatus[] listStatus(Path[] files, PathFilter filter) { return null; }
public FileStatus[] globStatus(Path pathPattern) { return null; }
public FileStatus[] globStatus(Path pathPattern, PathFilter filter) {
return null;
}
public Iterator<LocatedFileStatus> listFiles(
final Path path, final boolean isRecursive) {
return null;
}
public Iterator<LocatedFileStatus> listLocatedStatus(Path f) {
return null;
}
public Iterator<LocatedFileStatus> listLocatedStatus(Path f,
final PathFilter filter) {
return null;
}
public void copyFromLocalFile(Path src, Path dst) { }
public void moveFromLocalFile(Path[] srcs, Path dst) { }
public void moveFromLocalFile(Path src, Path dst) { }
public void copyToLocalFile(Path src, Path dst) { }
public void copyToLocalFile(boolean delSrc, Path src, Path dst, public void copyToLocalFile(boolean delSrc, Path src, Path dst,
boolean useRawLocalFileSystem) { } boolean useRawLocalFileSystem);
public void moveToLocalFile(Path src, Path dst) { } public void moveToLocalFile(Path src, Path dst);
public long getBlockSize(Path f) { return 0; } public long getBlockSize(Path f);
public FSDataOutputStream primitiveCreate(final Path f, public FSDataOutputStream primitiveCreate(Path f,
final EnumSet<CreateFlag> createFlag, EnumSet<CreateFlag> createFlag, CreateOpts... opts);
CreateOpts... opts) { return null; } public void primitiveMkdir(Path f, FsPermission absolutePermission,
public void primitiveMkdir(Path f, FsPermission absolutePermission, boolean createParent);
boolean createParent) { } public int getDefaultPort();
public int getDefaultPort() { return 0; } public String getCanonicalServiceName();
public String getCanonicalServiceName() { return null; } public Token<?> getDelegationToken(String renewer) throws IOException;
public Token<?> getDelegationToken(String renewer) throws IOException { public boolean deleteOnExit(Path f) throws IOException;
return null; public boolean cancelDeleteOnExit(Path f) throws IOException;
}
public boolean deleteOnExit(Path f) throws IOException {
return false;
}
public boolean cancelDeleteOnExit(Path f) throws IOException {
return false;
}
public Token<?>[] addDelegationTokens(String renewer, Credentials creds) public Token<?>[] addDelegationTokens(String renewer, Credentials creds)
throws IOException { throws IOException;
return null; public String getScheme();
} public Path fixRelativePart(Path p);
public String getScheme() { public ContentSummary getContentSummary(Path f);
return "dontcheck";
}
public Path fixRelativePart(Path p) { return null; }
} }
@Test @Test
public void testFilterFileSystem() throws Exception { public void testFilterFileSystem() throws Exception {
int errors = 0;
for (Method m : FileSystem.class.getDeclaredMethods()) { for (Method m : FileSystem.class.getDeclaredMethods()) {
if (Modifier.isStatic(m.getModifiers())) if (Modifier.isStatic(m.getModifiers()) ||
Modifier.isPrivate(m.getModifiers()) ||
Modifier.isFinal(m.getModifiers())) {
continue; continue;
if (Modifier.isPrivate(m.getModifiers())) }
continue;
if (Modifier.isFinal(m.getModifiers()))
continue;
try { try {
DontCheck.class.getMethod(m.getName(), m.getParameterTypes()); MustNotImplement.class.getMethod(m.getName(), m.getParameterTypes());
LOG.info("Skipping " + m); try {
FilterFileSystem.class.getDeclaredMethod(m.getName(),
m.getParameterTypes());
LOG.error("FilterFileSystem MUST NOT implement " + m);
errors++;
} catch (NoSuchMethodException ex) {
// Expected
}
} catch (NoSuchMethodException exc) { } catch (NoSuchMethodException exc) {
LOG.info("Testing " + m);
try{ try{
FilterFileSystem.class.getDeclaredMethod(m.getName(), m.getParameterTypes()); FilterFileSystem.class.getDeclaredMethod(m.getName(), m.getParameterTypes());
} } catch(NoSuchMethodException exc2){
catch(NoSuchMethodException exc2){ LOG.error("FilterFileSystem MUST implement " + m);
LOG.error("FilterFileSystem doesn't implement " + m); errors++;
throw exc2;
} }
} }
} }
assertTrue((errors + " methods were not overridden correctly - see" +
" log"), errors <= 0);
} }
@Test @Test