HADOOP-7060. A more elegant FileSystem#listCorruptFileBlocks API. Contributed by Patrick Kling.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1044146 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -24,7 +24,10 @@ Trunk (unreleased changes)
     (Patrick Kling via eli)
 
     HADOOP-7054 Change NN LoadGenerator to use FileContext APIs
-	(Sanjay Radia)
+	  (Sanjay Radia)
+
+    HADOOP-7060. A more elegant FileSystem#listCorruptFileBlocks API.
+    (Patrick Kling via hairong)
 
   OPTIMIZATIONS
 

src/java/org/apache/hadoop/fs/AbstractFileSystem.java

@@ -835,11 +835,11 @@ public abstract class AbstractFileSystem {
       UnresolvedLinkException, IOException;
 
   /**
-   * @return a list in which each entry describes a corrupt file/block
+   * @return an iterator over the corrupt files under the given path
+   * (may contain duplicates if a file has more than one corrupt block)
    * @throws IOException
    */
-  public CorruptFileBlocks listCorruptFileBlocks(String path,
+  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
-                                                 String cookie)
     throws IOException {
     throw new UnsupportedOperationException(getClass().getCanonicalName() +
                                             " does not support" +

src/java/org/apache/hadoop/fs/CorruptFileBlocks.java

@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.Text;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * Contains a list of paths corresponding to corrupt files and a cookie
- * used for iterative calls to NameNode.listCorruptFileBlocks.
- *
- */
-public class CorruptFileBlocks implements Writable {
-  // used for hashCode
-  private static final int PRIME = 16777619;
-
-  private String[] files;
-  private String cookie;
-
-  public CorruptFileBlocks() {
-    this(new String[0], "");
-  }
-
-  public CorruptFileBlocks(String[] files, String cookie) {
-    this.files = files;
-    this.cookie = cookie;
-  }
-
-  public String[] getFiles() {
-    return files;
-  }
-
-  public String getCookie() {
-    return cookie;
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    int fileCount = in.readInt();
-    files = new String[fileCount];
-    for (int i = 0; i < fileCount; i++) {
-      files[i] = Text.readString(in);
-    }
-    cookie = Text.readString(in);
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(files.length);
-    for (int i = 0; i < files.length; i++) {
-      Text.writeString(out, files[i]);
-    }
-    Text.writeString(out, cookie);
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  public boolean equals(Object obj) {
-    if (this == obj) {
-      return true;
-    }
-    if (!(obj instanceof CorruptFileBlocks)) {
-      return false;
-    }
-    CorruptFileBlocks other = (CorruptFileBlocks) obj;
-    return cookie.equals(other.cookie) &&
-      Arrays.equals(files, other.files);
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  public int hashCode() {
-    int result = cookie.hashCode();
-
-    for (String file : files) {
-      result = PRIME * result + file.hashCode();
-    }
-
-    return result;
-  }
-}

src/java/org/apache/hadoop/fs/FileContext.java

@@ -1298,18 +1298,19 @@ public final class FileContext {
   }
 
   /**
-   * @return a list in which each entry describes a corrupt file/block
+   * @return an iterator over the corrupt files under the given path
+   * (may contain duplicates if a file has more than one corrupt block)
    * @throws IOException
    */
-  public CorruptFileBlocks listCorruptFileBlocks(final String path,
+  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
-                                                 final String cookie)
     throws IOException {
-    final Path absF = fixRelativePart(new Path(path));
+    final Path absF = fixRelativePart(path);
-    return new FSLinkResolver<CorruptFileBlocks>() {
+    return new FSLinkResolver<RemoteIterator<Path>>() {
       @Override
-      public CorruptFileBlocks next(final AbstractFileSystem fs, final Path p) 
+      public RemoteIterator<Path> next(final AbstractFileSystem fs,
+                                       final Path p) 
         throws IOException, UnresolvedLinkException {
-        return fs.listCorruptFileBlocks(p.toUri().getPath(), cookie);
+        return fs.listCorruptFileBlocks(p);
       }
     }.resolve(this, absF);
   }

src/java/org/apache/hadoop/fs/FileSystem.java

@@ -1091,11 +1091,11 @@ public abstract class FileSystem extends Configured implements Closeable {
   }
 
   /**
-   * @return a list in which each entry describes a corrupt file/block
+   * @return an iterator over the corrupt files under the given path
+   * (may contain duplicates if a file has more than one corrupt block)
    * @throws IOException
    */
-  public CorruptFileBlocks listCorruptFileBlocks(String path,
+  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
-                                                 String cookie)
     throws IOException {
     throw new UnsupportedOperationException(getClass().getCanonicalName() +
                                             " does not support" +

src/java/org/apache/hadoop/fs/FilterFileSystem.java

@@ -170,10 +170,9 @@ public class FilterFileSystem extends FileSystem {
    * {@inheritDoc}
    */
   @Override
-  public CorruptFileBlocks listCorruptFileBlocks(String path,
+  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
-                                                 String cookie)
     throws IOException {
-    return fs.listCorruptFileBlocks(path, cookie);
+    return fs.listCorruptFileBlocks(path);
   }
 
   /** List files and its block locations in a directory. */

src/java/org/apache/hadoop/fs/FilterFs.java

@@ -168,10 +168,9 @@ public abstract class FilterFs extends AbstractFileSystem {
    * {@inheritDoc}
    */
   @Override
-  public CorruptFileBlocks listCorruptFileBlocks(String path,
+  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
-                                                 String cookie)
     throws IOException {
-    return myFs.listCorruptFileBlocks(path, cookie);
+    return myFs.listCorruptFileBlocks(path);
   }
 
   @Override

src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java

@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.*;
-import org.junit.Test;
-
-import org.apache.hadoop.io.DataOutputBuffer;
-
-public class TestCorruptFileBlocks {
-
-  /**
-   * Serialize the cfb given, deserialize and return the result.
-   */
-  static CorruptFileBlocks serializeAndDeserialize(CorruptFileBlocks cfb) 
-    throws IOException {
-    DataOutputBuffer buf = new DataOutputBuffer();
-    cfb.write(buf);
-
-    byte[] data = buf.getData();
-    DataInputStream input = new DataInputStream(new ByteArrayInputStream(data));
-
-    CorruptFileBlocks result = new CorruptFileBlocks();
-    result.readFields(input);
-
-    return result;
-  }
-
-  /**
-   * Check whether cfb is unchanged after serialization and deserialization.
-   */
-  static boolean checkSerialize(CorruptFileBlocks cfb)
-    throws IOException {
-    return cfb.equals(serializeAndDeserialize(cfb));
-  }
-
-  /**
-   * Test serialization and deserializaton of CorruptFileBlocks.
-   */
-  @Test
-  public void testSerialization() throws IOException {
-    {
-      CorruptFileBlocks cfb = new CorruptFileBlocks();
-      assertTrue(checkSerialize(cfb));
-    }
-
-    {
-      String[] files = new String[0];
-      CorruptFileBlocks cfb = new CorruptFileBlocks(files, "");
-      assertTrue(checkSerialize(cfb));
-    }
-
-    {
-      String[] files = { "a", "bb", "ccc" };
-      CorruptFileBlocks cfb = new CorruptFileBlocks(files, "test");
-      assertTrue(checkSerialize(cfb));
-    }
-  }
-}