From 15fe833da7e35568dba32545bfac76e680a3dc68 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Sun, 23 Dec 2012 02:15:17 +0000 Subject: [PATCH] HADOOP-9162. Merge change 1425390 from trunk git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1425392 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../hadoop-common/src/main/bin/hadoop | 3 + .../hadoop/util/NativeLibraryChecker.java | 75 +++++++++++++++++++ .../hadoop/util/TestNativeLibraryChecker.java | 54 +++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f75186ff8a7..1edc788b6a1 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -104,6 +104,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-9147. Add missing fields to FIleStatus.toString. (Jonathan Allen via suresh) + HADOOP-9162. Add utility to check native library availability. + (Binglin Chang via suresh) + OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop index 486465aa139..a680e46e8e2 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -31,6 +31,7 @@ function print_usage(){ echo " fs run a generic filesystem user client" echo " version print the version" echo " jar run a jar file" + echo " checknative [-a|-h] check native hadoop and compression libraries availability" echo " distcp copy file or directories recursively" echo " archive -archiveName NAME -p * create a hadoop archive" echo " classpath prints the class path needed to get the" @@ -100,6 +101,8 @@ case $COMMAND in CLASS=org.apache.hadoop.util.VersionInfo elif [ "$COMMAND" = "jar" ] ; then CLASS=org.apache.hadoop.util.RunJar + elif [ "$COMMAND" = "checknative" ] ; then + CLASS=org.apache.hadoop.util.NativeLibraryChecker elif [ "$COMMAND" = "distcp" ] ; then CLASS=org.apache.hadoop.tools.DistCp CLASSPATH=${CLASSPATH}:${TOOL_PATH} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java new file mode 100644 index 00000000000..f4cf2f53314 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import org.apache.hadoop.util.NativeCodeLoader; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.compress.SnappyCodec; +import org.apache.hadoop.io.compress.zlib.ZlibFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class NativeLibraryChecker { + /** + * A tool to test native library availability, + */ + public static void main(String[] args) { + String usage = "NativeLibraryChecker [-a|-h]\n" + + " -a use -a to check all libraries are available\n" + + " by default just check hadoop library is available\n" + + " exit with error code if check failed\n" + + " -h print this message\n"; + if (args.length > 1 || + (args.length == 1 && + !(args[0].equals("-a") || args[0].equals("-h")))) { + System.err.println(usage); + ExitUtil.terminate(1); + } + boolean checkAll = false; + if (args.length == 1) { + if (args[0].equals("-h")) { + System.out.println(usage); + return; + } + checkAll = true; + } + boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded(); + boolean zlibLoaded = false; + boolean snappyLoaded = false; + // lz4 is linked within libhadoop + boolean lz4Loaded = nativeHadoopLoaded; + if (nativeHadoopLoaded) { + zlibLoaded = ZlibFactory.isNativeZlibLoaded(new Configuration()); + snappyLoaded = NativeCodeLoader.buildSupportsSnappy() && + SnappyCodec.isNativeCodeLoaded(); + } + System.out.println("Native library checking:"); + System.out.printf("hadoop: %b\n", nativeHadoopLoaded); + System.out.printf("zlib: %b\n", zlibLoaded); + System.out.printf("snappy: %b\n", snappyLoaded); + System.out.printf("lz4: %b\n", lz4Loaded); + if ((!nativeHadoopLoaded) || + (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded))) { + // return 1 to indicated check failed + ExitUtil.terminate(1); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java new file mode 100644 index 00000000000..c41b56768b6 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import junit.framework.TestCase; + +import org.apache.hadoop.util.ExitUtil.ExitException; +import org.junit.Test; + +public class TestNativeLibraryChecker extends TestCase { + private void expectExit(String [] args) { + try { + // should throw exit exception + NativeLibraryChecker.main(args); + fail("should call exit"); + } catch (ExitException e) { + // pass + ExitUtil.resetFirstExitException(); + } + } + + @Test + public void testNativeLibraryChecker() { + ExitUtil.disableSystemExit(); + // help should return normally + NativeLibraryChecker.main(new String[] {"-h"}); + // illegal argmuments should exit + expectExit(new String[] {"-a", "-h"}); + expectExit(new String[] {"aaa"}); + if (NativeCodeLoader.isNativeCodeLoaded()) { + // no argument should return normally + NativeLibraryChecker.main(new String[0]); + } else { + // no argument should exit + expectExit(new String[0]); + } + } + +}