diff --git a/CHANGES.txt b/CHANGES.txt index 1840ddac16f..595b7162338 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -24,7 +24,10 @@ Trunk (unreleased changes) (Patrick Kling via eli) HADOOP-7054 Change NN LoadGenerator to use FileContext APIs - (Sanjay Radia) + (Sanjay Radia) + + HADOOP-7060. A more elegant FileSystem#listCorruptFileBlocks API. + (Patrick Kling via hairong) OPTIMIZATIONS diff --git a/src/java/org/apache/hadoop/fs/AbstractFileSystem.java b/src/java/org/apache/hadoop/fs/AbstractFileSystem.java index 8a24d13a853..2dfd4bf8645 100644 --- a/src/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/src/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -835,11 +835,11 @@ public abstract class AbstractFileSystem { UnresolvedLinkException, IOException; /** - * @return a list in which each entry describes a corrupt file/block + * @return an iterator over the corrupt files under the given path + * (may contain duplicates if a file has more than one corrupt block) * @throws IOException */ - public CorruptFileBlocks listCorruptFileBlocks(String path, - String cookie) + public RemoteIterator listCorruptFileBlocks(Path path) throws IOException { throw new UnsupportedOperationException(getClass().getCanonicalName() + " does not support" + diff --git a/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java b/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java index 0b8955fa150..e69de29bb2d 100644 --- a/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java +++ b/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.fs; - -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.Text; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.Arrays; - -/** - * Contains a list of paths corresponding to corrupt files and a cookie - * used for iterative calls to NameNode.listCorruptFileBlocks. - * - */ -public class CorruptFileBlocks implements Writable { - // used for hashCode - private static final int PRIME = 16777619; - - private String[] files; - private String cookie; - - public CorruptFileBlocks() { - this(new String[0], ""); - } - - public CorruptFileBlocks(String[] files, String cookie) { - this.files = files; - this.cookie = cookie; - } - - public String[] getFiles() { - return files; - } - - public String getCookie() { - return cookie; - } - - /** - * {@inheritDoc} - */ - @Override - public void readFields(DataInput in) throws IOException { - int fileCount = in.readInt(); - files = new String[fileCount]; - for (int i = 0; i < fileCount; i++) { - files[i] = Text.readString(in); - } - cookie = Text.readString(in); - } - - /** - * {@inheritDoc} - */ - @Override - public void write(DataOutput out) throws IOException { - out.writeInt(files.length); - for (int i = 0; i < files.length; i++) { - Text.writeString(out, files[i]); - } - Text.writeString(out, cookie); - } - - /** - * {@inheritDoc} - */ - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof CorruptFileBlocks)) { - return false; - } - CorruptFileBlocks other = (CorruptFileBlocks) obj; - return cookie.equals(other.cookie) && - Arrays.equals(files, other.files); - } - - /** - * {@inheritDoc} - */ - public int hashCode() { - int result = cookie.hashCode(); - - for (String file : files) { - result = PRIME * result + file.hashCode(); - } - - return result; - } -} diff --git a/src/java/org/apache/hadoop/fs/FileContext.java b/src/java/org/apache/hadoop/fs/FileContext.java index 43f9245da4a..057244cf412 100644 --- a/src/java/org/apache/hadoop/fs/FileContext.java +++ b/src/java/org/apache/hadoop/fs/FileContext.java @@ -1298,18 +1298,19 @@ public final class FileContext { } /** - * @return a list in which each entry describes a corrupt file/block + * @return an iterator over the corrupt files under the given path + * (may contain duplicates if a file has more than one corrupt block) * @throws IOException */ - public CorruptFileBlocks listCorruptFileBlocks(final String path, - final String cookie) + public RemoteIterator listCorruptFileBlocks(Path path) throws IOException { - final Path absF = fixRelativePart(new Path(path)); - return new FSLinkResolver() { + final Path absF = fixRelativePart(path); + return new FSLinkResolver>() { @Override - public CorruptFileBlocks next(final AbstractFileSystem fs, final Path p) + public RemoteIterator next(final AbstractFileSystem fs, + final Path p) throws IOException, UnresolvedLinkException { - return fs.listCorruptFileBlocks(p.toUri().getPath(), cookie); + return fs.listCorruptFileBlocks(p); } }.resolve(this, absF); } diff --git a/src/java/org/apache/hadoop/fs/FileSystem.java b/src/java/org/apache/hadoop/fs/FileSystem.java index c8f2d36a97a..2a69cbc5f45 100644 --- a/src/java/org/apache/hadoop/fs/FileSystem.java +++ b/src/java/org/apache/hadoop/fs/FileSystem.java @@ -1091,11 +1091,11 @@ public abstract class FileSystem extends Configured implements Closeable { } /** - * @return a list in which each entry describes a corrupt file/block + * @return an iterator over the corrupt files under the given path + * (may contain duplicates if a file has more than one corrupt block) * @throws IOException */ - public CorruptFileBlocks listCorruptFileBlocks(String path, - String cookie) + public RemoteIterator listCorruptFileBlocks(Path path) throws IOException { throw new UnsupportedOperationException(getClass().getCanonicalName() + " does not support" + diff --git a/src/java/org/apache/hadoop/fs/FilterFileSystem.java b/src/java/org/apache/hadoop/fs/FilterFileSystem.java index 88bebb0c142..656bf50d02d 100644 --- a/src/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/src/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -170,10 +170,9 @@ public class FilterFileSystem extends FileSystem { * {@inheritDoc} */ @Override - public CorruptFileBlocks listCorruptFileBlocks(String path, - String cookie) + public RemoteIterator listCorruptFileBlocks(Path path) throws IOException { - return fs.listCorruptFileBlocks(path, cookie); + return fs.listCorruptFileBlocks(path); } /** List files and its block locations in a directory. */ diff --git a/src/java/org/apache/hadoop/fs/FilterFs.java b/src/java/org/apache/hadoop/fs/FilterFs.java index 343289867ee..7e2f146b325 100644 --- a/src/java/org/apache/hadoop/fs/FilterFs.java +++ b/src/java/org/apache/hadoop/fs/FilterFs.java @@ -168,10 +168,9 @@ public abstract class FilterFs extends AbstractFileSystem { * {@inheritDoc} */ @Override - public CorruptFileBlocks listCorruptFileBlocks(String path, - String cookie) + public RemoteIterator listCorruptFileBlocks(Path path) throws IOException { - return myFs.listCorruptFileBlocks(path, cookie); + return myFs.listCorruptFileBlocks(path); } @Override diff --git a/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java b/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java index 14ebc81e311..e69de29bb2d 100644 --- a/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java +++ b/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; - -import static org.junit.Assert.*; -import org.junit.Test; - -import org.apache.hadoop.io.DataOutputBuffer; - -public class TestCorruptFileBlocks { - - /** - * Serialize the cfb given, deserialize and return the result. - */ - static CorruptFileBlocks serializeAndDeserialize(CorruptFileBlocks cfb) - throws IOException { - DataOutputBuffer buf = new DataOutputBuffer(); - cfb.write(buf); - - byte[] data = buf.getData(); - DataInputStream input = new DataInputStream(new ByteArrayInputStream(data)); - - CorruptFileBlocks result = new CorruptFileBlocks(); - result.readFields(input); - - return result; - } - - /** - * Check whether cfb is unchanged after serialization and deserialization. - */ - static boolean checkSerialize(CorruptFileBlocks cfb) - throws IOException { - return cfb.equals(serializeAndDeserialize(cfb)); - } - - /** - * Test serialization and deserializaton of CorruptFileBlocks. - */ - @Test - public void testSerialization() throws IOException { - { - CorruptFileBlocks cfb = new CorruptFileBlocks(); - assertTrue(checkSerialize(cfb)); - } - - { - String[] files = new String[0]; - CorruptFileBlocks cfb = new CorruptFileBlocks(files, ""); - assertTrue(checkSerialize(cfb)); - } - - { - String[] files = { "a", "bb", "ccc" }; - CorruptFileBlocks cfb = new CorruptFileBlocks(files, "test"); - assertTrue(checkSerialize(cfb)); - } - } -} \ No newline at end of file