HADOOP-6978. Adds support for NativeIO using JNI. Contributed by Todd Lipcon, Devaraj Das & Owen O'Malley.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1040883 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Devaraj Das
parent
ebb236ef9c
commit
dbd07f9e8c
|
|
@ -75,6 +75,9 @@ Release 0.22.0 - Unreleased
|
|||
HADOOP-7013. Add boolean field isCorrupt to BlockLocation.
|
||||
(Patrick Kling via hairong)
|
||||
|
||||
HADOOP-6978. Adds support for NativeIO using JNI.
|
||||
(Todd Lipcon, Devaraj Das & Owen O'Malley via ddas)
|
||||
|
||||
IMPROVEMENTS
|
||||
|
||||
HADOOP-6644. util.Shell getGROUPS_FOR_USER_COMMAND method name
|
||||
|
|
|
|||
|
|
@ -366,6 +366,7 @@
|
|||
|
||||
<mkdir dir="${build.native}/lib"/>
|
||||
<mkdir dir="${build.native}/src/org/apache/hadoop/io/compress/zlib"/>
|
||||
<mkdir dir="${build.native}/src/org/apache/hadoop/io/nativeio"/>
|
||||
<mkdir dir="${build.native}/src/org/apache/hadoop/security"/>
|
||||
|
||||
<javah
|
||||
|
|
@ -386,6 +387,14 @@
|
|||
>
|
||||
<class name="org.apache.hadoop.security.JniBasedUnixGroupsMapping" />
|
||||
</javah>
|
||||
<javah
|
||||
classpath="${build.classes}"
|
||||
destdir="${build.native}/src/org/apache/hadoop/io/nativeio"
|
||||
force="yes"
|
||||
verbose="yes"
|
||||
>
|
||||
<class name="org.apache.hadoop.io.nativeio.NativeIO" />
|
||||
</javah>
|
||||
|
||||
<exec dir="${build.native}" executable="sh" failonerror="true">
|
||||
<env key="OS_NAME" value="${os.name}"/>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,208 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileDescriptor;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.io.nativeio.Errno;
|
||||
import org.apache.hadoop.io.nativeio.NativeIO;
|
||||
import org.apache.hadoop.io.nativeio.NativeIOException;
|
||||
import org.apache.hadoop.io.nativeio.NativeIO.Stat;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
|
||||
/**
|
||||
* This class provides secure APIs for opening and creating files on the local
|
||||
* disk. The main issue this class tries to handle is that of symlink traversal.
|
||||
* <br/>
|
||||
* An example of such an attack is:
|
||||
* <ol>
|
||||
* <li> Malicious user removes his task's syslog file, and puts a link to the
|
||||
* jobToken file of a target user.</li>
|
||||
* <li> Malicious user tries to open the syslog file via the servlet on the
|
||||
* tasktracker.</li>
|
||||
* <li> The tasktracker is unaware of the symlink, and simply streams the contents
|
||||
* of the jobToken file. The malicious user can now access potentially sensitive
|
||||
* map outputs, etc. of the target user's job.</li>
|
||||
* </ol>
|
||||
* A similar attack is possible involving task log truncation, but in that case
|
||||
* due to an insecure write to a file.
|
||||
* <br/>
|
||||
*/
|
||||
public class SecureIOUtils {
|
||||
|
||||
/**
|
||||
* Ensure that we are set up to run with the appropriate native support code.
|
||||
* If security is disabled, and the support code is unavailable, this class
|
||||
* still tries its best to be secure, but is vulnerable to some race condition
|
||||
* attacks.
|
||||
*
|
||||
* If security is enabled but the support code is unavailable, throws a
|
||||
* RuntimeException since we don't want to run insecurely.
|
||||
*/
|
||||
static {
|
||||
boolean shouldBeSecure = UserGroupInformation.isSecurityEnabled();
|
||||
boolean canBeSecure = NativeIO.isAvailable();
|
||||
|
||||
if (!canBeSecure && shouldBeSecure) {
|
||||
throw new RuntimeException(
|
||||
"Secure IO is not possible without native code extensions.");
|
||||
}
|
||||
|
||||
// Pre-cache an instance of the raw FileSystem since we sometimes
|
||||
// do secure IO in a shutdown hook, where this call could fail.
|
||||
try {
|
||||
rawFilesystem = FileSystem.getLocal(new Configuration()).getRaw();
|
||||
} catch (IOException ie) {
|
||||
throw new RuntimeException(
|
||||
"Couldn't obtain an instance of RawLocalFileSystem.");
|
||||
}
|
||||
|
||||
// SecureIO just skips security checks in the case that security is
|
||||
// disabled
|
||||
skipSecurity = !canBeSecure;
|
||||
}
|
||||
|
||||
private final static boolean skipSecurity;
|
||||
private final static FileSystem rawFilesystem;
|
||||
|
||||
/**
|
||||
* Open the given File for read access, verifying the expected user/group
|
||||
* constraints.
|
||||
* @param f the file that we are trying to open
|
||||
* @param expectedOwner the expected user owner for the file
|
||||
* @param expectedGroup the expected group owner for the file
|
||||
* @throws IOException if an IO Error occurred, or the user/group does not
|
||||
* match
|
||||
*/
|
||||
public static FileInputStream openForRead(File f, String expectedOwner,
|
||||
String expectedGroup) throws IOException {
|
||||
if (skipSecurity) {
|
||||
// Subject to race conditions but this is the best we can do
|
||||
FileStatus status =
|
||||
rawFilesystem.getFileStatus(new Path(f.getAbsolutePath()));
|
||||
checkStat(f, status.getOwner(), status.getGroup(),
|
||||
expectedOwner, expectedGroup);
|
||||
return new FileInputStream(f);
|
||||
}
|
||||
|
||||
FileInputStream fis = new FileInputStream(f);
|
||||
boolean success = false;
|
||||
try {
|
||||
Stat stat = NativeIO.fstat(fis.getFD());
|
||||
checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner,
|
||||
expectedGroup);
|
||||
success = true;
|
||||
return fis;
|
||||
} finally {
|
||||
if (!success) {
|
||||
fis.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static FileOutputStream insecureCreateForWrite(File f,
|
||||
int permissions) throws IOException {
|
||||
// If we can't do real security, do a racy exists check followed by an
|
||||
// open and chmod
|
||||
if (f.exists()) {
|
||||
throw new AlreadyExistsException("File " + f + " already exists");
|
||||
}
|
||||
FileOutputStream fos = new FileOutputStream(f);
|
||||
boolean success = false;
|
||||
try {
|
||||
rawFilesystem.setPermission(new Path(f.getAbsolutePath()),
|
||||
new FsPermission((short)permissions));
|
||||
success = true;
|
||||
return fos;
|
||||
} finally {
|
||||
if (!success) {
|
||||
fos.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the specified File for write access, ensuring that it does not exist.
|
||||
* @param f the file that we want to create
|
||||
* @param permissions we want to have on the file (if security is enabled)
|
||||
*
|
||||
* @throws AlreadyExistsException if the file already exists
|
||||
* @throws IOException if any other error occurred
|
||||
*/
|
||||
public static FileOutputStream createForWrite(File f, int permissions)
|
||||
throws IOException {
|
||||
if (skipSecurity) {
|
||||
return insecureCreateForWrite(f, permissions);
|
||||
} else {
|
||||
// Use the native wrapper around open(2)
|
||||
try {
|
||||
FileDescriptor fd = NativeIO.open(f.getAbsolutePath(),
|
||||
NativeIO.O_WRONLY | NativeIO.O_CREAT | NativeIO.O_EXCL,
|
||||
permissions);
|
||||
return new FileOutputStream(fd);
|
||||
} catch (NativeIOException nioe) {
|
||||
if (nioe.getErrno() == Errno.EEXIST) {
|
||||
throw new AlreadyExistsException(nioe);
|
||||
}
|
||||
throw nioe;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void checkStat(File f, String owner, String group,
|
||||
String expectedOwner,
|
||||
String expectedGroup) throws IOException {
|
||||
if (expectedOwner != null &&
|
||||
!expectedOwner.equals(owner)) {
|
||||
throw new IOException(
|
||||
"Owner '" + owner + "' for path " + f + " did not match " +
|
||||
"expected owner '" + expectedOwner + "'");
|
||||
}
|
||||
if (expectedGroup != null &&
|
||||
!expectedGroup.equals(group)) {
|
||||
throw new IOException(
|
||||
"Group '" + group + "' for path " + f + " did not match " +
|
||||
"expected group '" + expectedGroup + "'");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Signals that an attempt to create a file at a given pathname has failed
|
||||
* because another file already existed at that path.
|
||||
*/
|
||||
public static class AlreadyExistsException extends IOException {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
public AlreadyExistsException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
|
||||
public AlreadyExistsException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io.nativeio;
|
||||
|
||||
/**
|
||||
* Enum representing POSIX errno values.
|
||||
*/
|
||||
public enum Errno {
|
||||
EPERM,
|
||||
ENOENT,
|
||||
ESRCH,
|
||||
EINTR,
|
||||
EIO,
|
||||
ENXIO,
|
||||
E2BIG,
|
||||
ENOEXEC,
|
||||
EBADF,
|
||||
ECHILD,
|
||||
EAGAIN,
|
||||
ENOMEM,
|
||||
EACCES,
|
||||
EFAULT,
|
||||
ENOTBLK,
|
||||
EBUSY,
|
||||
EEXIST,
|
||||
EXDEV,
|
||||
ENODEV,
|
||||
ENOTDIR,
|
||||
EISDIR,
|
||||
EINVAL,
|
||||
ENFILE,
|
||||
EMFILE,
|
||||
ENOTTY,
|
||||
ETXTBSY,
|
||||
EFBIG,
|
||||
ENOSPC,
|
||||
ESPIPE,
|
||||
EROFS,
|
||||
EMLINK,
|
||||
EPIPE,
|
||||
EDOM,
|
||||
ERANGE,
|
||||
|
||||
UNKNOWN;
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io.nativeio;
|
||||
|
||||
import java.io.FileDescriptor;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.hadoop.util.NativeCodeLoader;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
/**
|
||||
* JNI wrappers for various native IO-related calls not available in Java.
|
||||
* These functions should generally be used alongside a fallback to another
|
||||
* more portable mechanism.
|
||||
*/
|
||||
public class NativeIO {
|
||||
// Flags for open() call from bits/fcntl.h
|
||||
public static final int O_RDONLY = 00;
|
||||
public static final int O_WRONLY = 01;
|
||||
public static final int O_RDWR = 02;
|
||||
public static final int O_CREAT = 0100;
|
||||
public static final int O_EXCL = 0200;
|
||||
public static final int O_NOCTTY = 0400;
|
||||
public static final int O_TRUNC = 01000;
|
||||
public static final int O_APPEND = 02000;
|
||||
public static final int O_NONBLOCK = 04000;
|
||||
public static final int O_SYNC = 010000;
|
||||
public static final int O_ASYNC = 020000;
|
||||
public static final int O_FSYNC = O_SYNC;
|
||||
public static final int O_NDELAY = O_NONBLOCK;
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(NativeIO.class);
|
||||
|
||||
private static boolean nativeLoaded = false;
|
||||
|
||||
static {
|
||||
if (NativeCodeLoader.isNativeCodeLoaded()) {
|
||||
try {
|
||||
initNative();
|
||||
nativeLoaded = true;
|
||||
} catch (Throwable t) {
|
||||
// This can happen if the user has an older version of libhadoop.so
|
||||
// installed - in this case we can continue without native IO
|
||||
// after warning
|
||||
LOG.error("Unable to initialize NativeIO libraries", t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true if the JNI-based native IO extensions are available.
|
||||
*/
|
||||
public static boolean isAvailable() {
|
||||
return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded;
|
||||
}
|
||||
|
||||
/** Wrapper around open(2) */
|
||||
public static native FileDescriptor open(String path, int flags, int mode) throws IOException;
|
||||
/** Wrapper around fstat(2) */
|
||||
public static native Stat fstat(FileDescriptor fd) throws IOException;
|
||||
/** Initialize the JNI method ID and class ID cache */
|
||||
private static native void initNative();
|
||||
|
||||
|
||||
/**
|
||||
* Result type of the fstat call
|
||||
*/
|
||||
public static class Stat {
|
||||
private String owner, group;
|
||||
private int mode;
|
||||
|
||||
// Mode constants
|
||||
public static final int S_IFMT = 0170000; /* type of file */
|
||||
public static final int S_IFIFO = 0010000; /* named pipe (fifo) */
|
||||
public static final int S_IFCHR = 0020000; /* character special */
|
||||
public static final int S_IFDIR = 0040000; /* directory */
|
||||
public static final int S_IFBLK = 0060000; /* block special */
|
||||
public static final int S_IFREG = 0100000; /* regular */
|
||||
public static final int S_IFLNK = 0120000; /* symbolic link */
|
||||
public static final int S_IFSOCK = 0140000; /* socket */
|
||||
public static final int S_IFWHT = 0160000; /* whiteout */
|
||||
public static final int S_ISUID = 0004000; /* set user id on execution */
|
||||
public static final int S_ISGID = 0002000; /* set group id on execution */
|
||||
public static final int S_ISVTX = 0001000; /* save swapped text even after use */
|
||||
public static final int S_IRUSR = 0000400; /* read permission, owner */
|
||||
public static final int S_IWUSR = 0000200; /* write permission, owner */
|
||||
public static final int S_IXUSR = 0000100; /* execute/search permission, owner */
|
||||
|
||||
Stat(String owner, String group, int mode) {
|
||||
this.owner = owner;
|
||||
this.group = group;
|
||||
this.mode = mode;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return "Stat(owner='" + owner + "', group='" + group + "'" +
|
||||
", mode=" + mode + ")";
|
||||
}
|
||||
|
||||
public String getOwner() {
|
||||
return owner;
|
||||
}
|
||||
public String getGroup() {
|
||||
return group;
|
||||
}
|
||||
public int getMode() {
|
||||
return mode;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io.nativeio;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* An exception generated by a call to the native IO code.
|
||||
*
|
||||
* These exceptions simply wrap <i>errno</i> result codes.
|
||||
*/
|
||||
public class NativeIOException extends IOException {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private Errno errno;
|
||||
|
||||
public NativeIOException(String msg, Errno errno) {
|
||||
super(msg);
|
||||
this.errno = errno;
|
||||
}
|
||||
|
||||
public Errno getErrno() {
|
||||
return errno;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return errno.toString() + ": " + super.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -33,7 +33,8 @@ export PLATFORM = $(shell echo $$OS_NAME | tr [A-Z] [a-z])
|
|||
|
||||
AM_CPPFLAGS = @JNI_CPPFLAGS@ -I$(HADOOP_NATIVE_SRCDIR)/src \
|
||||
-Isrc/org/apache/hadoop/io/compress/zlib \
|
||||
-Isrc/org/apache/hadoop/security
|
||||
-Isrc/org/apache/hadoop/security \
|
||||
-Isrc/org/apache/hadoop/io/nativeio/
|
||||
AM_LDFLAGS = @JNI_LDFLAGS@ -m$(JVM_DATA_MODEL)
|
||||
AM_CFLAGS = -g -Wall -fPIC -O2 -m$(JVM_DATA_MODEL)
|
||||
|
||||
|
|
@ -41,8 +42,12 @@ lib_LTLIBRARIES = libhadoop.la
|
|||
libhadoop_la_SOURCES = src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c \
|
||||
src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c \
|
||||
src/org/apache/hadoop/security/getGroup.c \
|
||||
src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c
|
||||
libhadoop_la_LDFLAGS = -version-info 1:0:0
|
||||
src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c \
|
||||
src/org/apache/hadoop/io/nativeio/file_descriptor.c \
|
||||
src/org/apache/hadoop/io/nativeio/errno_enum.c \
|
||||
src/org/apache/hadoop/io/nativeio/NativeIO.c
|
||||
|
||||
libhadoop_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS)
|
||||
libhadoop_la_LIBADD = -ldl -ljvm
|
||||
|
||||
#
|
||||
|
|
|
|||
|
|
@ -93,7 +93,8 @@ libLTLIBRARIES_INSTALL = $(INSTALL)
|
|||
LTLIBRARIES = $(lib_LTLIBRARIES)
|
||||
libhadoop_la_DEPENDENCIES =
|
||||
am_libhadoop_la_OBJECTS = ZlibCompressor.lo ZlibDecompressor.lo \
|
||||
getGroup.lo JniBasedUnixGroupsMapping.lo
|
||||
getGroup.lo JniBasedUnixGroupsMapping.lo file_descriptor.lo \
|
||||
errno_enum.lo NativeIO.lo
|
||||
libhadoop_la_OBJECTS = $(am_libhadoop_la_OBJECTS)
|
||||
DEFAULT_INCLUDES = -I. -I$(srcdir) -I.
|
||||
depcomp = $(SHELL) $(top_srcdir)/config/depcomp
|
||||
|
|
@ -222,7 +223,8 @@ sysconfdir = @sysconfdir@
|
|||
target_alias = @target_alias@
|
||||
AM_CPPFLAGS = @JNI_CPPFLAGS@ -I$(HADOOP_NATIVE_SRCDIR)/src \
|
||||
-Isrc/org/apache/hadoop/io/compress/zlib \
|
||||
-Isrc/org/apache/hadoop/security
|
||||
-Isrc/org/apache/hadoop/security \
|
||||
-Isrc/org/apache/hadoop/io/nativeio/
|
||||
|
||||
AM_LDFLAGS = @JNI_LDFLAGS@ -m$(JVM_DATA_MODEL)
|
||||
AM_CFLAGS = -g -Wall -fPIC -O2 -m$(JVM_DATA_MODEL)
|
||||
|
|
@ -230,9 +232,12 @@ lib_LTLIBRARIES = libhadoop.la
|
|||
libhadoop_la_SOURCES = src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c \
|
||||
src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c \
|
||||
src/org/apache/hadoop/security/getGroup.c \
|
||||
src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c
|
||||
src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c \
|
||||
src/org/apache/hadoop/io/nativeio/file_descriptor.c \
|
||||
src/org/apache/hadoop/io/nativeio/errno_enum.c \
|
||||
src/org/apache/hadoop/io/nativeio/NativeIO.c
|
||||
|
||||
libhadoop_la_LDFLAGS = -version-info 1:0:0
|
||||
libhadoop_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS)
|
||||
libhadoop_la_LIBADD = -ldl -ljvm
|
||||
all: config.h
|
||||
$(MAKE) $(AM_MAKEFLAGS) all-am
|
||||
|
|
@ -326,8 +331,11 @@ distclean-compile:
|
|||
-rm -f *.tab.c
|
||||
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/JniBasedUnixGroupsMapping.Plo@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NativeIO.Plo@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ZlibCompressor.Plo@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ZlibDecompressor.Plo@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/errno_enum.Plo@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/file_descriptor.Plo@am__quote@
|
||||
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/getGroup.Plo@am__quote@
|
||||
|
||||
.c.o:
|
||||
|
|
@ -379,6 +387,27 @@ JniBasedUnixGroupsMapping.lo: src/org/apache/hadoop/security/JniBasedUnixGroupsM
|
|||
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
|
||||
@am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o JniBasedUnixGroupsMapping.lo `test -f 'src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c' || echo '$(srcdir)/'`src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c
|
||||
|
||||
file_descriptor.lo: src/org/apache/hadoop/io/nativeio/file_descriptor.c
|
||||
@am__fastdepCC_TRUE@ if $(LIBTOOL) --tag=CC --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT file_descriptor.lo -MD -MP -MF "$(DEPDIR)/file_descriptor.Tpo" -c -o file_descriptor.lo `test -f 'src/org/apache/hadoop/io/nativeio/file_descriptor.c' || echo '$(srcdir)/'`src/org/apache/hadoop/io/nativeio/file_descriptor.c; \
|
||||
@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/file_descriptor.Tpo" "$(DEPDIR)/file_descriptor.Plo"; else rm -f "$(DEPDIR)/file_descriptor.Tpo"; exit 1; fi
|
||||
@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/org/apache/hadoop/io/nativeio/file_descriptor.c' object='file_descriptor.lo' libtool=yes @AMDEPBACKSLASH@
|
||||
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
|
||||
@am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o file_descriptor.lo `test -f 'src/org/apache/hadoop/io/nativeio/file_descriptor.c' || echo '$(srcdir)/'`src/org/apache/hadoop/io/nativeio/file_descriptor.c
|
||||
|
||||
errno_enum.lo: src/org/apache/hadoop/io/nativeio/errno_enum.c
|
||||
@am__fastdepCC_TRUE@ if $(LIBTOOL) --tag=CC --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT errno_enum.lo -MD -MP -MF "$(DEPDIR)/errno_enum.Tpo" -c -o errno_enum.lo `test -f 'src/org/apache/hadoop/io/nativeio/errno_enum.c' || echo '$(srcdir)/'`src/org/apache/hadoop/io/nativeio/errno_enum.c; \
|
||||
@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/errno_enum.Tpo" "$(DEPDIR)/errno_enum.Plo"; else rm -f "$(DEPDIR)/errno_enum.Tpo"; exit 1; fi
|
||||
@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/org/apache/hadoop/io/nativeio/errno_enum.c' object='errno_enum.lo' libtool=yes @AMDEPBACKSLASH@
|
||||
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
|
||||
@am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o errno_enum.lo `test -f 'src/org/apache/hadoop/io/nativeio/errno_enum.c' || echo '$(srcdir)/'`src/org/apache/hadoop/io/nativeio/errno_enum.c
|
||||
|
||||
NativeIO.lo: src/org/apache/hadoop/io/nativeio/NativeIO.c
|
||||
@am__fastdepCC_TRUE@ if $(LIBTOOL) --tag=CC --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT NativeIO.lo -MD -MP -MF "$(DEPDIR)/NativeIO.Tpo" -c -o NativeIO.lo `test -f 'src/org/apache/hadoop/io/nativeio/NativeIO.c' || echo '$(srcdir)/'`src/org/apache/hadoop/io/nativeio/NativeIO.c; \
|
||||
@am__fastdepCC_TRUE@ then mv -f "$(DEPDIR)/NativeIO.Tpo" "$(DEPDIR)/NativeIO.Plo"; else rm -f "$(DEPDIR)/NativeIO.Tpo"; exit 1; fi
|
||||
@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='src/org/apache/hadoop/io/nativeio/NativeIO.c' object='NativeIO.lo' libtool=yes @AMDEPBACKSLASH@
|
||||
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
|
||||
@am__fastdepCC_FALSE@ $(LIBTOOL) --tag=CC --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o NativeIO.lo `test -f 'src/org/apache/hadoop/io/nativeio/NativeIO.c' || echo '$(srcdir)/'`src/org/apache/hadoop/io/nativeio/NativeIO.c
|
||||
|
||||
mostlyclean-libtool:
|
||||
-rm -f *.lo
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,10 @@
|
|||
/* The 'actual' dynamic-library for '-lz' */
|
||||
#undef HADOOP_ZLIB_LIBRARY
|
||||
|
||||
/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you
|
||||
don't. */
|
||||
#undef HAVE_DECL_STRERROR_R
|
||||
|
||||
/* Define to 1 if you have the <dlfcn.h> header file. */
|
||||
#undef HAVE_DLFCN_H
|
||||
|
||||
|
|
@ -39,6 +43,9 @@
|
|||
/* Define to 1 if you have the <stdlib.h> header file. */
|
||||
#undef HAVE_STDLIB_H
|
||||
|
||||
/* Define to 1 if you have the `strerror_r' function. */
|
||||
#undef HAVE_STRERROR_R
|
||||
|
||||
/* Define to 1 if you have the <strings.h> header file. */
|
||||
#undef HAVE_STRINGS_H
|
||||
|
||||
|
|
@ -81,8 +88,17 @@
|
|||
/* Define to 1 if you have the ANSI C header files. */
|
||||
#undef STDC_HEADERS
|
||||
|
||||
/* Define to 1 if strerror_r returns char *. */
|
||||
#undef STRERROR_R_CHAR_P
|
||||
|
||||
/* Version number of package */
|
||||
#undef VERSION
|
||||
|
||||
/* Number of bits in a file offset, on hosts where this is settable. */
|
||||
#undef _FILE_OFFSET_BITS
|
||||
|
||||
/* Define for large files, on AIX-style hosts. */
|
||||
#undef _LARGE_FILES
|
||||
|
||||
/* Define to empty if `const' does not conform to ANSI C. */
|
||||
#undef const
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -38,6 +38,7 @@ AC_INIT(src/org_apache_hadoop.h)
|
|||
AC_CONFIG_SRCDIR([src/org_apache_hadoop.h])
|
||||
AC_CONFIG_AUX_DIR(config)
|
||||
AC_CONFIG_HEADER([config.h])
|
||||
AC_SYS_LARGEFILE
|
||||
|
||||
AM_INIT_AUTOMAKE(hadoop,1.0.0)
|
||||
|
||||
|
|
@ -95,6 +96,9 @@ AC_C_CONST
|
|||
# Checks for library functions.
|
||||
AC_CHECK_FUNCS([memset])
|
||||
|
||||
# Check for nonstandard STRERROR_R
|
||||
AC_FUNC_STRERROR_R
|
||||
|
||||
AC_CONFIG_FILES([Makefile])
|
||||
AC_OUTPUT
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ AM_LDFLAGS = @JNI_LDFLAGS@ -m$(JVM_DATA_MODEL)
|
|||
|
||||
lib_LTLIBRARIES = libhadoop.la
|
||||
libhadoop_la_SOURCES =
|
||||
libhadoop_la_LDFLAGS = -version-info 1:0:0
|
||||
libhadoop_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS)
|
||||
libhadoop_la_LIBADD = $(HADOOP_OBJS) -ldl -ljvm
|
||||
|
||||
#
|
||||
|
|
|
|||
|
|
@ -0,0 +1,277 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// get the autoconf settings
|
||||
#include "config.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <grp.h>
|
||||
#include <jni.h>
|
||||
#include <pwd.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "org_apache_hadoop.h"
|
||||
#include "org_apache_hadoop_io_nativeio_NativeIO.h"
|
||||
#include "file_descriptor.h"
|
||||
#include "errno_enum.h"
|
||||
|
||||
// the NativeIO$Stat inner class and its constructor
|
||||
static jclass stat_clazz;
|
||||
static jmethodID stat_ctor;
|
||||
|
||||
// the NativeIOException class and its constructor
|
||||
static jclass nioe_clazz;
|
||||
static jmethodID nioe_ctor;
|
||||
|
||||
// Internal functions
|
||||
static void throw_ioe(JNIEnv* env, int errnum);
|
||||
static ssize_t get_pw_buflen();
|
||||
|
||||
|
||||
static void stat_init(JNIEnv *env) {
|
||||
// Init Stat
|
||||
jclass clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$Stat");
|
||||
PASS_EXCEPTIONS(env);
|
||||
stat_clazz = (*env)->NewGlobalRef(env, clazz);
|
||||
stat_ctor = (*env)->GetMethodID(env, stat_clazz, "<init>",
|
||||
"(Ljava/lang/String;Ljava/lang/String;I)V");
|
||||
}
|
||||
|
||||
static void stat_deinit(JNIEnv *env) {
|
||||
if (stat_clazz != NULL) {
|
||||
(*env)->DeleteGlobalRef(env, stat_clazz);
|
||||
stat_clazz = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static void nioe_init(JNIEnv *env) {
|
||||
// Init NativeIOException
|
||||
nioe_clazz = (*env)->FindClass(
|
||||
env, "org/apache/hadoop/io/nativeio/NativeIOException");
|
||||
PASS_EXCEPTIONS(env);
|
||||
|
||||
nioe_clazz = (*env)->NewGlobalRef(env, nioe_clazz);
|
||||
nioe_ctor = (*env)->GetMethodID(env, nioe_clazz, "<init>",
|
||||
"(Ljava/lang/String;Lorg/apache/hadoop/io/nativeio/Errno;)V");
|
||||
}
|
||||
|
||||
static void nioe_deinit(JNIEnv *env) {
|
||||
if (nioe_clazz != NULL) {
|
||||
(*env)->DeleteGlobalRef(env, nioe_clazz);
|
||||
nioe_clazz = NULL;
|
||||
}
|
||||
nioe_ctor = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* private static native void initNative();
|
||||
*
|
||||
* We rely on this function rather than lazy initialization because
|
||||
* the lazy approach may have a race if multiple callers try to
|
||||
* init at the same time.
|
||||
*/
|
||||
JNIEXPORT void JNICALL
|
||||
Java_org_apache_hadoop_io_nativeio_NativeIO_initNative(
|
||||
JNIEnv *env, jclass clazz) {
|
||||
|
||||
stat_init(env);
|
||||
PASS_EXCEPTIONS_GOTO(env, error);
|
||||
nioe_init(env);
|
||||
PASS_EXCEPTIONS_GOTO(env, error);
|
||||
fd_init(env);
|
||||
PASS_EXCEPTIONS_GOTO(env, error);
|
||||
errno_enum_init(env);
|
||||
PASS_EXCEPTIONS_GOTO(env, error);
|
||||
return;
|
||||
error:
|
||||
// these are all idempodent and safe to call even if the
|
||||
// class wasn't initted yet
|
||||
stat_deinit(env);
|
||||
nioe_deinit(env);
|
||||
fd_deinit(env);
|
||||
errno_enum_deinit(env);
|
||||
}
|
||||
|
||||
/*
|
||||
* public static native Stat fstat(FileDescriptor fd);
|
||||
*/
|
||||
JNIEXPORT jobject JNICALL
|
||||
Java_org_apache_hadoop_io_nativeio_NativeIO_fstat(
|
||||
JNIEnv *env, jclass clazz, jobject fd_object)
|
||||
{
|
||||
jobject ret = NULL;
|
||||
char *pw_buf = NULL;
|
||||
|
||||
int fd = fd_get(env, fd_object);
|
||||
PASS_EXCEPTIONS_GOTO(env, cleanup);
|
||||
|
||||
struct stat s;
|
||||
int rc = fstat(fd, &s);
|
||||
if (rc != 0) {
|
||||
throw_ioe(env, errno);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
size_t pw_buflen = get_pw_buflen();
|
||||
if ((pw_buf = malloc(pw_buflen)) == NULL) {
|
||||
THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// Grab username
|
||||
struct passwd pwd, *pwdp;
|
||||
while ((rc = getpwuid_r(s.st_uid, &pwd, pw_buf, pw_buflen, &pwdp)) != 0) {
|
||||
if (rc != ERANGE) {
|
||||
throw_ioe(env, rc);
|
||||
goto cleanup;
|
||||
}
|
||||
free(pw_buf);
|
||||
pw_buflen *= 2;
|
||||
if ((pw_buf = malloc(pw_buflen)) == NULL) {
|
||||
THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer");
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
assert(pwdp == &pwd);
|
||||
|
||||
jstring jstr_username = (*env)->NewStringUTF(env, pwd.pw_name);
|
||||
if (jstr_username == NULL) goto cleanup;
|
||||
|
||||
// Grab group
|
||||
struct group grp, *grpp;
|
||||
while ((rc = getgrgid_r(s.st_gid, &grp, pw_buf, pw_buflen, &grpp)) != 0) {
|
||||
if (rc != ERANGE) {
|
||||
throw_ioe(env, rc);
|
||||
goto cleanup;
|
||||
}
|
||||
free(pw_buf);
|
||||
pw_buflen *= 2;
|
||||
if ((pw_buf = malloc(pw_buflen)) == NULL) {
|
||||
THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for pw buffer");
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
assert(grpp == &grp);
|
||||
|
||||
jstring jstr_groupname = (*env)->NewStringUTF(env, grp.gr_name);
|
||||
PASS_EXCEPTIONS_GOTO(env, cleanup);
|
||||
|
||||
// Construct result
|
||||
ret = (*env)->NewObject(env, stat_clazz, stat_ctor,
|
||||
jstr_username, jstr_groupname, s.st_mode);
|
||||
|
||||
cleanup:
|
||||
if (pw_buf != NULL) free(pw_buf);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* public static native FileDescriptor open(String path, int flags, int mode);
|
||||
*/
|
||||
JNIEXPORT jobject JNICALL
|
||||
Java_org_apache_hadoop_io_nativeio_NativeIO_open(
|
||||
JNIEnv *env, jclass clazz, jstring j_path,
|
||||
jint flags, jint mode)
|
||||
{
|
||||
jobject ret = NULL;
|
||||
|
||||
const char *path = (*env)->GetStringUTFChars(env, j_path, NULL);
|
||||
if (path == NULL) goto cleanup; // JVM throws Exception for us
|
||||
|
||||
int fd;
|
||||
if (flags & O_CREAT) {
|
||||
fd = open(path, flags, mode);
|
||||
} else {
|
||||
fd = open(path, flags);
|
||||
}
|
||||
|
||||
if (fd == -1) {
|
||||
throw_ioe(env, errno);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
ret = fd_create(env, fd);
|
||||
|
||||
cleanup:
|
||||
if (path != NULL) {
|
||||
(*env)->ReleaseStringUTFChars(env, j_path, path);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* Throw a java.IO.IOException, generating the message from errno.
|
||||
*/
|
||||
static void throw_ioe(JNIEnv* env, int errnum)
|
||||
{
|
||||
const char* message;
|
||||
char buffer[80];
|
||||
jstring jstr_message;
|
||||
|
||||
buffer[0] = 0;
|
||||
#ifdef STRERROR_R_CHAR_P
|
||||
// GNU strerror_r
|
||||
message = strerror_r(errnum, buffer, sizeof(buffer));
|
||||
assert (message != NULL);
|
||||
#else
|
||||
int ret = strerror_r(errnum, buffer, sizeof(buffer));
|
||||
if (ret == 0) {
|
||||
message = buffer;
|
||||
} else {
|
||||
message = "Unknown error";
|
||||
}
|
||||
#endif
|
||||
jobject errno_obj = errno_to_enum(env, errnum);
|
||||
|
||||
if ((jstr_message = (*env)->NewStringUTF(env, message)) == NULL)
|
||||
goto err;
|
||||
|
||||
jthrowable obj = (jthrowable)(*env)->NewObject(env, nioe_clazz, nioe_ctor,
|
||||
jstr_message, errno_obj);
|
||||
if (obj == NULL) goto err;
|
||||
|
||||
(*env)->Throw(env, obj);
|
||||
return;
|
||||
|
||||
err:
|
||||
if (jstr_message != NULL)
|
||||
(*env)->ReleaseStringUTFChars(env, jstr_message, message);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Determine how big a buffer we need for reentrant getpwuid_r and getgrnam_r
|
||||
*/
|
||||
ssize_t get_pw_buflen() {
|
||||
size_t ret = 0;
|
||||
#ifdef _SC_GETPW_R_SIZE_MAX
|
||||
ret = sysconf(_SC_GETPW_R_SIZE_MAX);
|
||||
#endif
|
||||
return (ret > 512) ? ret : 512;
|
||||
}
|
||||
/**
|
||||
* vim: sw=2: ts=2: et:
|
||||
*/
|
||||
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include <assert.h>
|
||||
#include <errno.h>
|
||||
#include <jni.h>
|
||||
|
||||
#include "org_apache_hadoop.h"
|
||||
|
||||
typedef struct errno_mapping {
|
||||
int errno_val;
|
||||
char *errno_str;
|
||||
} errno_mapping_t;
|
||||
|
||||
// Macro to define structs like {FOO, "FOO"} for each errno value
|
||||
#define MAPPING(x) {x, #x}
|
||||
static errno_mapping_t ERRNO_MAPPINGS[] = {
|
||||
MAPPING(EPERM),
|
||||
MAPPING(ENOENT),
|
||||
MAPPING(ESRCH),
|
||||
MAPPING(EINTR),
|
||||
MAPPING(EIO),
|
||||
MAPPING(ENXIO),
|
||||
MAPPING(E2BIG),
|
||||
MAPPING(ENOEXEC),
|
||||
MAPPING(EBADF),
|
||||
MAPPING(ECHILD),
|
||||
MAPPING(EAGAIN),
|
||||
MAPPING(ENOMEM),
|
||||
MAPPING(EACCES),
|
||||
MAPPING(EFAULT),
|
||||
MAPPING(ENOTBLK),
|
||||
MAPPING(EBUSY),
|
||||
MAPPING(EEXIST),
|
||||
MAPPING(EXDEV),
|
||||
MAPPING(ENODEV),
|
||||
MAPPING(ENOTDIR),
|
||||
MAPPING(EISDIR),
|
||||
MAPPING(EINVAL),
|
||||
MAPPING(ENFILE),
|
||||
MAPPING(EMFILE),
|
||||
MAPPING(ENOTTY),
|
||||
MAPPING(ETXTBSY),
|
||||
MAPPING(EFBIG),
|
||||
MAPPING(ENOSPC),
|
||||
MAPPING(ESPIPE),
|
||||
MAPPING(EROFS),
|
||||
MAPPING(EMLINK),
|
||||
MAPPING(EPIPE),
|
||||
MAPPING(EDOM),
|
||||
MAPPING(ERANGE),
|
||||
{-1, NULL}
|
||||
};
|
||||
|
||||
static jclass enum_class;
|
||||
static jmethodID enum_valueOf;
|
||||
static jclass errno_class;
|
||||
|
||||
void errno_enum_init(JNIEnv *env) {
|
||||
if (enum_class != NULL) return;
|
||||
|
||||
enum_class = (*env)->FindClass(env, "java/lang/Enum");
|
||||
PASS_EXCEPTIONS(env);
|
||||
enum_class = (*env)->NewGlobalRef(env, enum_class);
|
||||
enum_valueOf = (*env)->GetStaticMethodID(env, enum_class,
|
||||
"valueOf", "(Ljava/lang/Class;Ljava/lang/String;)Ljava/lang/Enum;");
|
||||
PASS_EXCEPTIONS(env);
|
||||
|
||||
errno_class = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/Errno");
|
||||
PASS_EXCEPTIONS(env);
|
||||
errno_class = (*env)->NewGlobalRef(env, errno_class);
|
||||
}
|
||||
|
||||
void errno_enum_deinit(JNIEnv *env) {
|
||||
if (enum_class != NULL) {
|
||||
(*env)->DeleteGlobalRef(env, enum_class);
|
||||
enum_class = NULL;
|
||||
}
|
||||
if (errno_class != NULL) {
|
||||
(*env)->DeleteGlobalRef(env, errno_class);
|
||||
errno_class = NULL;
|
||||
}
|
||||
enum_valueOf = NULL;
|
||||
}
|
||||
|
||||
|
||||
static char *errno_to_string(int errnum) {
|
||||
int i;
|
||||
for (i = 0; ERRNO_MAPPINGS[i].errno_str != NULL; i++) {
|
||||
if (ERRNO_MAPPINGS[i].errno_val == errnum)
|
||||
return ERRNO_MAPPINGS[i].errno_str;
|
||||
}
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
||||
jobject errno_to_enum(JNIEnv *env, int errnum) {
|
||||
char *str = errno_to_string(errnum);
|
||||
assert(str != NULL);
|
||||
|
||||
jstring jstr = (*env)->NewStringUTF(env, str);
|
||||
PASS_EXCEPTIONS_RET(env, NULL);
|
||||
|
||||
return (*env)->CallStaticObjectMethod(
|
||||
env, enum_class, enum_valueOf, errno_class, jstr);
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef ERRNO_ENUM_H
|
||||
#define ERRNO_ENUM_H
|
||||
|
||||
#include <jni.h>
|
||||
|
||||
void errno_enum_init(JNIEnv *env);
|
||||
void errno_enum_deinit(JNIEnv *env);
|
||||
jobject errno_to_enum(JNIEnv *env, int errnum);
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <jni.h>
|
||||
#include "file_descriptor.h"
|
||||
#include "org_apache_hadoop.h"
|
||||
|
||||
// class of java.io.FileDescriptor
|
||||
static jclass fd_class;
|
||||
// the internal field for the integer fd
|
||||
static jfieldID fd_descriptor;
|
||||
// the no-argument constructor
|
||||
static jmethodID fd_constructor;
|
||||
|
||||
|
||||
void fd_init(JNIEnv* env)
|
||||
{
|
||||
if (fd_class != NULL) return; // already initted
|
||||
|
||||
fd_class = (*env)->FindClass(env, "java/io/FileDescriptor");
|
||||
PASS_EXCEPTIONS(env);
|
||||
fd_class = (*env)->NewGlobalRef(env, fd_class);
|
||||
|
||||
fd_descriptor = (*env)->GetFieldID(env, fd_class, "fd", "I");
|
||||
PASS_EXCEPTIONS(env);
|
||||
fd_constructor = (*env)->GetMethodID(env, fd_class, "<init>", "()V");
|
||||
}
|
||||
|
||||
void fd_deinit(JNIEnv *env) {
|
||||
if (fd_class != NULL) {
|
||||
(*env)->DeleteGlobalRef(env, fd_class);
|
||||
fd_class = NULL;
|
||||
}
|
||||
fd_descriptor = NULL;
|
||||
fd_constructor = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Given an instance 'obj' of java.io.FileDescriptor, return the
|
||||
* underlying fd, or throw if unavailable
|
||||
*/
|
||||
int fd_get(JNIEnv* env, jobject obj) {
|
||||
return (*env)->GetIntField(env, obj, fd_descriptor);
|
||||
}
|
||||
|
||||
/*
|
||||
* Create a FileDescriptor object corresponding to the given int fd
|
||||
*/
|
||||
jobject fd_create(JNIEnv *env, int fd) {
|
||||
jobject obj = (*env)->NewObject(env, fd_class, fd_constructor);
|
||||
PASS_EXCEPTIONS_RET(env, NULL);
|
||||
|
||||
(*env)->SetIntField(env, obj, fd_descriptor, fd);
|
||||
return obj;
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef FILE_DESCRIPTOR_H
|
||||
#define FILE_DESCRIPTOR_H
|
||||
|
||||
#include <jni.h>
|
||||
|
||||
void fd_init(JNIEnv *env);
|
||||
void fd_deinit(JNIEnv *env);
|
||||
|
||||
int fd_get(JNIEnv* env, jobject obj);
|
||||
jobject fd_create(JNIEnv *env, int fd);
|
||||
|
||||
#endif
|
||||
|
|
@ -50,6 +50,22 @@
|
|||
} \
|
||||
}
|
||||
|
||||
/* Helper macro to return if an exception is pending */
|
||||
#define PASS_EXCEPTIONS(env) \
|
||||
{ \
|
||||
if ((*env)->ExceptionCheck(env)) return; \
|
||||
}
|
||||
|
||||
#define PASS_EXCEPTIONS_GOTO(env, target) \
|
||||
{ \
|
||||
if ((*env)->ExceptionCheck(env)) goto target; \
|
||||
}
|
||||
|
||||
#define PASS_EXCEPTIONS_RET(env, ret) \
|
||||
{ \
|
||||
if ((*env)->ExceptionCheck(env)) return (ret); \
|
||||
}
|
||||
|
||||
/**
|
||||
* A helper function to dlsym a 'symbol' from a given library-handle.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io;
|
||||
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.io.nativeio.NativeIO;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assume.*;
|
||||
import static org.junit.Assert.*;
|
||||
import java.io.IOException;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
|
||||
public class TestSecureIOUtils {
|
||||
private static String realOwner, realGroup;
|
||||
private static final File testFilePath =
|
||||
new File(System.getProperty("test.build.data"), "TestSecureIOContext");
|
||||
|
||||
@BeforeClass
|
||||
public static void makeTestFile() throws Exception {
|
||||
FileOutputStream fos = new FileOutputStream(testFilePath);
|
||||
fos.write("hello".getBytes("UTF-8"));
|
||||
fos.close();
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
FileSystem rawFS = FileSystem.getLocal(conf).getRaw();
|
||||
FileStatus stat = rawFS.getFileStatus(
|
||||
new Path(testFilePath.toString()));
|
||||
realOwner = stat.getOwner();
|
||||
realGroup = stat.getGroup();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReadUnrestricted() throws IOException {
|
||||
SecureIOUtils.openForRead(testFilePath, null, null).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReadCorrectlyRestrictedWithSecurity() throws IOException {
|
||||
SecureIOUtils
|
||||
.openForRead(testFilePath, realOwner, realGroup).close();
|
||||
}
|
||||
|
||||
@Test(expected=IOException.class)
|
||||
public void testReadIncorrectlyRestrictedWithSecurity() throws IOException {
|
||||
SecureIOUtils
|
||||
.openForRead(testFilePath, "invalidUser", null).close();
|
||||
fail("Didn't throw expection for wrong ownership!");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateForWrite() throws IOException {
|
||||
try {
|
||||
SecureIOUtils.createForWrite(testFilePath, 0777);
|
||||
fail("Was able to create file at " + testFilePath);
|
||||
} catch (SecureIOUtils.AlreadyExistsException aee) {
|
||||
// expected
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,137 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io.nativeio;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileDescriptor;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assume.*;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.util.NativeCodeLoader;
|
||||
|
||||
public class TestNativeIO {
|
||||
static final Log LOG = LogFactory.getLog(TestNativeIO.class);
|
||||
|
||||
static final File TEST_DIR = new File(
|
||||
System.getProperty("test.build.data"), "testnativeio");
|
||||
|
||||
@Before
|
||||
public void checkLoaded() {
|
||||
assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setupTestDir() throws IOException {
|
||||
FileUtil.fullyDelete(TEST_DIR);
|
||||
TEST_DIR.mkdirs();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFstat() throws Exception {
|
||||
FileOutputStream fos = new FileOutputStream(
|
||||
new File(TEST_DIR, "testfstat"));
|
||||
NativeIO.Stat stat = NativeIO.fstat(fos.getFD());
|
||||
fos.close();
|
||||
LOG.info("Stat: " + String.valueOf(stat));
|
||||
|
||||
assertEquals(System.getProperty("user.name"), stat.getOwner());
|
||||
assertNotNull(stat.getGroup());
|
||||
assertTrue(!"".equals(stat.getGroup()));
|
||||
assertEquals("Stat mode field should indicate a regular file",
|
||||
NativeIO.Stat.S_IFREG, stat.getMode() & NativeIO.Stat.S_IFMT);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFstatClosedFd() throws Exception {
|
||||
FileOutputStream fos = new FileOutputStream(
|
||||
new File(TEST_DIR, "testfstat2"));
|
||||
fos.close();
|
||||
try {
|
||||
NativeIO.Stat stat = NativeIO.fstat(fos.getFD());
|
||||
} catch (NativeIOException nioe) {
|
||||
LOG.info("Got expected exception", nioe);
|
||||
assertEquals(Errno.EBADF, nioe.getErrno());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOpenMissingWithoutCreate() throws Exception {
|
||||
LOG.info("Open a missing file without O_CREAT and it should fail");
|
||||
try {
|
||||
FileDescriptor fd = NativeIO.open(
|
||||
new File(TEST_DIR, "doesntexist").getAbsolutePath(),
|
||||
NativeIO.O_WRONLY, 0700);
|
||||
fail("Able to open a new file without O_CREAT");
|
||||
} catch (NativeIOException nioe) {
|
||||
LOG.info("Got expected exception", nioe);
|
||||
assertEquals(Errno.ENOENT, nioe.getErrno());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOpenWithCreate() throws Exception {
|
||||
LOG.info("Test creating a file with O_CREAT");
|
||||
FileDescriptor fd = NativeIO.open(
|
||||
new File(TEST_DIR, "testWorkingOpen").getAbsolutePath(),
|
||||
NativeIO.O_WRONLY | NativeIO.O_CREAT, 0700);
|
||||
assertNotNull(true);
|
||||
assertTrue(fd.valid());
|
||||
FileOutputStream fos = new FileOutputStream(fd);
|
||||
fos.write("foo".getBytes());
|
||||
fos.close();
|
||||
|
||||
assertFalse(fd.valid());
|
||||
|
||||
LOG.info("Test exclusive create");
|
||||
try {
|
||||
fd = NativeIO.open(
|
||||
new File(TEST_DIR, "testWorkingOpen").getAbsolutePath(),
|
||||
NativeIO.O_WRONLY | NativeIO.O_CREAT | NativeIO.O_EXCL, 0700);
|
||||
fail("Was able to create existing file with O_EXCL");
|
||||
} catch (NativeIOException nioe) {
|
||||
LOG.info("Got expected exception for failed exclusive create", nioe);
|
||||
assertEquals(Errno.EEXIST, nioe.getErrno());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that opens and closes a file 10000 times - this would crash with
|
||||
* "Too many open files" if we leaked fds using this access pattern.
|
||||
*/
|
||||
@Test
|
||||
public void testFDDoesntLeak() throws IOException {
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
FileDescriptor fd = NativeIO.open(
|
||||
new File(TEST_DIR, "testNoFdLeak").getAbsolutePath(),
|
||||
NativeIO.O_WRONLY | NativeIO.O_CREAT, 0700);
|
||||
assertNotNull(true);
|
||||
assertTrue(fd.valid());
|
||||
FileOutputStream fos = new FileOutputStream(fd);
|
||||
fos.write("foo".getBytes());
|
||||
fos.close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue