HADOOP-11057. checknative command to probe for winutils.exe on windows. Contributed by Xiaoyu Yao.
(cherry picked from commit 6dae4b430c
)
This commit is contained in:
parent
c1df6f3b40
commit
7d9c45f778
|
@ -170,6 +170,9 @@ Release 2.6.0 - UNRELEASED
|
||||||
|
|
||||||
HADOOP-11070. Create MiniKMS for testing. (tucu)
|
HADOOP-11070. Create MiniKMS for testing. (tucu)
|
||||||
|
|
||||||
|
HADOOP-11057. checknative command to probe for winutils.exe on windows.
|
||||||
|
(Xiaoyu Yao via cnauroth)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
|
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
|
||||||
|
|
|
@ -37,7 +37,8 @@ public class NativeLibraryChecker {
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
String usage = "NativeLibraryChecker [-a|-h]\n"
|
String usage = "NativeLibraryChecker [-a|-h]\n"
|
||||||
+ " -a use -a to check all libraries are available\n"
|
+ " -a use -a to check all libraries are available\n"
|
||||||
+ " by default just check hadoop library is available\n"
|
+ " by default just check hadoop library (and\n"
|
||||||
|
+ " winutils.exe on Windows OS) is available\n"
|
||||||
+ " exit with error code 1 if check failed\n"
|
+ " exit with error code 1 if check failed\n"
|
||||||
+ " -h print this message\n";
|
+ " -h print this message\n";
|
||||||
if (args.length > 1 ||
|
if (args.length > 1 ||
|
||||||
|
@ -62,12 +63,16 @@ public static void main(String[] args) {
|
||||||
boolean lz4Loaded = nativeHadoopLoaded;
|
boolean lz4Loaded = nativeHadoopLoaded;
|
||||||
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
|
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
|
||||||
boolean openSslLoaded = false;
|
boolean openSslLoaded = false;
|
||||||
|
boolean winutilsExists = false;
|
||||||
|
|
||||||
String openSslDetail = "";
|
String openSslDetail = "";
|
||||||
String hadoopLibraryName = "";
|
String hadoopLibraryName = "";
|
||||||
String zlibLibraryName = "";
|
String zlibLibraryName = "";
|
||||||
String snappyLibraryName = "";
|
String snappyLibraryName = "";
|
||||||
String lz4LibraryName = "";
|
String lz4LibraryName = "";
|
||||||
String bzip2LibraryName = "";
|
String bzip2LibraryName = "";
|
||||||
|
String winutilsPath = null;
|
||||||
|
|
||||||
if (nativeHadoopLoaded) {
|
if (nativeHadoopLoaded) {
|
||||||
hadoopLibraryName = NativeCodeLoader.getLibraryName();
|
hadoopLibraryName = NativeCodeLoader.getLibraryName();
|
||||||
zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf);
|
zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf);
|
||||||
|
@ -93,6 +98,15 @@ public static void main(String[] args) {
|
||||||
bzip2LibraryName = Bzip2Factory.getLibraryName(conf);
|
bzip2LibraryName = Bzip2Factory.getLibraryName(conf);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// winutils.exe is required on Windows
|
||||||
|
winutilsPath = Shell.getWinUtilsPath();
|
||||||
|
if (winutilsPath != null) {
|
||||||
|
winutilsExists = true;
|
||||||
|
} else {
|
||||||
|
winutilsPath = "";
|
||||||
|
}
|
||||||
|
|
||||||
System.out.println("Native library checking:");
|
System.out.println("Native library checking:");
|
||||||
System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
|
System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
|
||||||
System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
|
System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
|
||||||
|
@ -100,7 +114,11 @@ public static void main(String[] args) {
|
||||||
System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
|
System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
|
||||||
System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
|
System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
|
||||||
System.out.printf("openssl: %b %s\n", openSslLoaded, openSslDetail);
|
System.out.printf("openssl: %b %s\n", openSslLoaded, openSslDetail);
|
||||||
if ((!nativeHadoopLoaded) ||
|
if (Shell.WINDOWS) {
|
||||||
|
System.out.printf("winutils: %b %s\n", winutilsExists, winutilsPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((!nativeHadoopLoaded) || (Shell.WINDOWS && (!winutilsExists)) ||
|
||||||
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
|
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
|
||||||
// return 1 to indicated check failed
|
// return 1 to indicated check failed
|
||||||
ExitUtil.terminate(1);
|
ExitUtil.terminate(1);
|
||||||
|
|
|
@ -17,6 +17,9 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.util;
|
package org.apache.hadoop.util;
|
||||||
|
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.PrintStream;
|
||||||
|
|
||||||
import junit.framework.TestCase;
|
import junit.framework.TestCase;
|
||||||
|
|
||||||
import org.apache.hadoop.util.ExitUtil.ExitException;
|
import org.apache.hadoop.util.ExitUtil.ExitException;
|
||||||
|
@ -51,4 +54,30 @@ public void testNativeLibraryChecker() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNativeLibraryCheckerOutput(){
|
||||||
|
expectOutput(new String[]{"-a"});
|
||||||
|
// no argument
|
||||||
|
expectOutput(new String[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void expectOutput(String [] args) {
|
||||||
|
ExitUtil.disableSystemExit();
|
||||||
|
ByteArrayOutputStream outContent = new ByteArrayOutputStream();
|
||||||
|
PrintStream originalPs = System.out;
|
||||||
|
System.setOut(new PrintStream(outContent));
|
||||||
|
try {
|
||||||
|
NativeLibraryChecker.main(args);
|
||||||
|
} catch (ExitException e) {
|
||||||
|
ExitUtil.resetFirstExitException();
|
||||||
|
} finally {
|
||||||
|
if (Shell.WINDOWS) {
|
||||||
|
assertEquals(outContent.toString().indexOf("winutils: true") != -1, true);
|
||||||
|
}
|
||||||
|
if (NativeCodeLoader.isNativeCodeLoaded()) {
|
||||||
|
assertEquals(outContent.toString().indexOf("hadoop: true") != -1, true);
|
||||||
|
}
|
||||||
|
System.setOut(originalPs);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue