Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "HADOOP-15637. LocalFs#listLocatedStatus does not filter out hidden .crc files. Contributed by Erik Krogen." 

This reverts commit 6812535b0b.
This commit is contained in:
Chen Liang 2018-07-30 12:05:08 -07:00
parent 8cd2a73777
commit 46fcbc5030
2 changed files with 0 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
View File

@ -27,12 +27,10 @@ import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.NoSuchElementException;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -529,39 +527,4 @@ public abstract class ChecksumFs extends FilterFs {
} }
return results.toArray(new FileStatus[results.size()]); return results.toArray(new FileStatus[results.size()]);
} }
@Override
public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
throws AccessControlException, FileNotFoundException,
UnresolvedLinkException, IOException {
final RemoteIterator<LocatedFileStatus> iter =
getMyFs().listLocatedStatus(f);
return new RemoteIterator<LocatedFileStatus>() {
private LocatedFileStatus next = null;
@Override
public boolean hasNext() throws IOException {
while (next == null && iter.hasNext()) {
LocatedFileStatus unfilteredNext = iter.next();
if (!isChecksumFile(unfilteredNext.getPath())) {
next = unfilteredNext;
}
}
return next != null;
}
@Override
public LocatedFileStatus next() throws IOException {
if (!hasNext()) {
throw new NoSuchElementException();
}
LocatedFileStatus tmp = next;
next = null;
return tmp;
}
};
}
} }

38
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
View File

@ -370,44 +370,6 @@ public abstract class FileContextMainOperationsBaseTest {
Assert.assertFalse(pathsIterator.hasNext()); Assert.assertFalse(pathsIterator.hasNext());
} }
@Test
public void testListFiles() throws Exception {
Path[] testDirs = {
getTestRootPath(fc, "test/dir1"),
getTestRootPath(fc, "test/dir1/dir1"),
getTestRootPath(fc, "test/dir2")
};
Path[] testFiles = {
new Path(testDirs[0], "file1"),
new Path(testDirs[0], "file2"),
new Path(testDirs[1], "file2"),
new Path(testDirs[2], "file1")
};
for (Path path : testDirs) {
fc.mkdir(path, FsPermission.getDefault(), true);
}
for (Path p : testFiles) {
FSDataOutputStream out = fc.create(p).build();
out.writeByte(0);
out.close();
}
RemoteIterator<LocatedFileStatus> filesIterator =
fc.util().listFiles(getTestRootPath(fc, "test"), true);
LocatedFileStatus[] fileStats =
new LocatedFileStatus[testFiles.length];
for (int i = 0; i < fileStats.length; i++) {
assertTrue(filesIterator.hasNext());
fileStats[i] = filesIterator.next();
}
assertFalse(filesIterator.hasNext());
for (Path p : testFiles) {
assertTrue(containsPath(p, fileStats));
}
}
@Test @Test
public void testListStatusFilterWithNoMatches() throws Exception { public void testListStatusFilterWithNoMatches() throws Exception {
Path[] testDirs = { Path[] testDirs = {