HADOOP-13254. Create framework for configurable disk checkers (yufeigu via rkanter)

(cherry picked from commit 442162048a)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java

@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.io.File;
+
+/**
+ * BasicDiskValidator is a wrapper around {@link DiskChecker}.
+ */
+public class BasicDiskValidator implements DiskValidator {
+  public static final String NAME = "basic";
+
+  @Override
+  public void checkStatus(File dir) throws DiskErrorException {
+    DiskChecker.checkDir(dir);
+  }
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.io.File;
+
+/**
+ * A interface for disk validators.
+ *
+ * The {@link #checkStatus(File)} operation checks status of a file/dir.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface DiskValidator {
+  /**
+   * Check the status of a file/dir.
+   * @param dir a file/dir
+   * @throws DiskErrorException if any disk error
+   */
+  void checkStatus(File dir) throws DiskErrorException;
+}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java

@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * The factory class to create instance of {@link DiskValidator}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class DiskValidatorFactory {
+  @VisibleForTesting
+  static final ConcurrentHashMap<Class<? extends DiskValidator>, DiskValidator>
+      INSTANCES = new ConcurrentHashMap<>();
+
+  private DiskValidatorFactory() {
+  }
+
+  /**
+   * Returns a {@link DiskValidator} instance corresponding to the passed clazz.
+   * @param clazz a class extends {@link DiskValidator}
+   */
+  public static DiskValidator
+      getInstance(Class<? extends DiskValidator> clazz) {
+    DiskValidator diskValidator;
+    if (INSTANCES.containsKey(clazz)) {
+      diskValidator = INSTANCES.get(clazz);
+    } else {
+      diskValidator = ReflectionUtils.newInstance(clazz, null);
+      // check the return of putIfAbsent() to see if any other thread have put
+      // the instance with the same key into INSTANCES
+      DiskValidator diskValidatorRet =
+          INSTANCES.putIfAbsent(clazz, diskValidator);
+      if (diskValidatorRet != null) {
+        diskValidator = diskValidatorRet;
+      }
+    }
+
+    return diskValidator;
+  }
+
+  /**
+   * Returns {@link DiskValidator} instance corresponding to its name.
+   * The diskValidator parameter can be "basic" for {@link BasicDiskValidator}.
+   * @param diskValidator canonical class name, for example, "basic"
+   * @throws DiskErrorException if the class cannot be located
+   */
+  @SuppressWarnings("unchecked")
+  public static DiskValidator getInstance(String diskValidator)
+      throws DiskErrorException {
+    @SuppressWarnings("rawtypes")
+    Class clazz;
+
+    if (diskValidator.equalsIgnoreCase(BasicDiskValidator.NAME)) {
+      clazz = BasicDiskValidator.class;
+    } else {
+      try {
+        clazz = Class.forName(diskValidator);
+      } catch (ClassNotFoundException cnfe) {
+        throw new DiskErrorException(diskValidator
+            + " DiskValidator class not found.", cnfe);
+      }
+    }
+
+    return getInstance(clazz);
+  }
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestBasicDiskValidator.java

@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+import java.io.File;
+
+/**
+ * The class to test BasicDiskValidator.
+ */
+public class TestBasicDiskValidator extends TestDiskChecker {
+  @Override
+  protected void checkDirs(boolean isDir, String perm, boolean success)
+      throws Throwable {
+    File localDir = File.createTempFile("test", "tmp");
+    try {
+      if (isDir) {
+        // reuse the file path generated by File#createTempFile to create a dir
+        localDir.delete();
+        localDir.mkdir();
+      }
+
+      Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
+          localDir.getAbsolutePath()));
+
+      DiskValidatorFactory.getInstance(BasicDiskValidator.NAME).
+          checkStatus(localDir);
+      assertTrue("call to checkDir() succeeded.", success);
+    } catch (DiskErrorException e) {
+      // call to checkDir() succeeded even though it was expected to fail
+      // if success is false, otherwise throw the exception
+      if (success) {
+        throw e;
+      }
+    } finally {
+      localDir.delete();
+    }
+  }
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java

@@ -138,30 +138,30 @@ public class TestDiskChecker {
 
   @Test (timeout = 30000)
   public void testCheckDir_normal_local() throws Throwable {
-    _checkDirs(true, "755", true);
+    checkDirs(true, "755", true);
   }
 
   @Test (timeout = 30000)
   public void testCheckDir_notDir_local() throws Throwable {
-    _checkDirs(false, "000", false);
+    checkDirs(false, "000", false);
   }
 
   @Test (timeout = 30000)
   public void testCheckDir_notReadable_local() throws Throwable {
-    _checkDirs(true, "000", false);
+    checkDirs(true, "000", false);
   }
 
   @Test (timeout = 30000)
   public void testCheckDir_notWritable_local() throws Throwable {
-    _checkDirs(true, "444", false);
+    checkDirs(true, "444", false);
   }
 
   @Test (timeout = 30000)
   public void testCheckDir_notListable_local() throws Throwable {
-    _checkDirs(true, "666", false);
+    checkDirs(true, "666", false);
   }
 
-  private void _checkDirs(boolean isDir, String perm, boolean success)
+  protected void checkDirs(boolean isDir, String perm, boolean success)
       throws Throwable {
     File localDir = File.createTempFile("test", "tmp");
     if (isDir) {

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskValidatorFactory.java

@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.hadoop.util.DiskChecker.DiskErrorException;
+
+/**
+ * The class to test DiskValidatorFactory.
+ */
+public class TestDiskValidatorFactory {
+
+  /**
+   * Trivial tests that make sure
+   * {@link DiskValidatorFactory#getInstance(String)} works as expected.
+   *
+   * @throws DiskErrorException if fail to get the instance.
+   */
+  @Test
+  public void testGetInstance() throws DiskErrorException {
+    DiskValidator diskValidator = DiskValidatorFactory.getInstance("basic");
+    assertNotNull("Fail to get the instance.", diskValidator);
+
+    assertEquals("Fail to create the correct instance.",
+        diskValidator.getClass(), BasicDiskValidator.class);
+
+    assertNotNull("Fail to cache the object", DiskValidatorFactory.INSTANCES.
+        get(BasicDiskValidator.class));
+  }
+
+  /**
+   * To test whether an exception is threw out as expected if trying to create
+   * a non-exist class.
+   * @throws DiskErrorException if fail to get the instance.
+   */
+  @Test(expected = DiskErrorException.class)
+  public void testGetInstanceOfNonExistClass() throws DiskErrorException {
+    DiskValidatorFactory.getInstance("non-exist");
+  }
+}