HADOOP-7824. NativeIO.java flags and identifiers must be set correctly for each platform, not hardcoded to their Linux values (Martin Walsh via Colin P. McCabe)
(cherry picked from commit 21d10ccc6e463cf250414264c78acb4a6e7c83e3)
This commit is contained in:
Colin Patrick Mccabe
parent
508da4d461
commit
07a042aa3c
@ -535,6 +535,10 @@ Release 2.8.0 - UNRELEASED
|
||||
HADOOP-10945. 4-digit octal umask permissions throws a parse error (Chang
|
||||
Li via jlowe)
|
||||
|
||||
HADOOP-7824. NativeIO.java flags and identifiers must be set correctly for
|
||||
each platform, not hardcoded to their Linux values (Martin Walsh via Colin
|
||||
P. McCabe)
|
||||
|
||||
Release 2.7.2 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
||||
@ -29,6 +29,8 @@
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.io.nativeio.NativeIO;
|
||||
|
||||
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_WILLNEED;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||
|
||||
@ -204,7 +206,7 @@ public void run() {
|
||||
// other FD, which may be wasted work, but won't cause a problem.
|
||||
try {
|
||||
NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
|
||||
fd, off, len, NativeIO.POSIX.POSIX_FADV_WILLNEED);
|
||||
fd, off, len, POSIX_FADV_WILLNEED);
|
||||
} catch (IOException ioe) {
|
||||
if (canceled) {
|
||||
// no big deal - the reader canceled the request and closed
|
||||
|
||||
@ -56,51 +56,54 @@
|
||||
@InterfaceStability.Unstable
|
||||
public class NativeIO {
|
||||
public static class POSIX {
|
||||
// Flags for open() call from bits/fcntl.h
|
||||
public static final int O_RDONLY = 00;
|
||||
public static final int O_WRONLY = 01;
|
||||
public static final int O_RDWR = 02;
|
||||
public static final int O_CREAT = 0100;
|
||||
public static final int O_EXCL = 0200;
|
||||
public static final int O_NOCTTY = 0400;
|
||||
public static final int O_TRUNC = 01000;
|
||||
public static final int O_APPEND = 02000;
|
||||
public static final int O_NONBLOCK = 04000;
|
||||
public static final int O_SYNC = 010000;
|
||||
// Flags for open() call from bits/fcntl.h - Set by JNI
|
||||
public static int O_RDONLY = -1;
|
||||
public static int O_WRONLY = -1;
|
||||
public static int O_RDWR = -1;
|
||||
public static int O_CREAT = -1;
|
||||
public static int O_EXCL = -1;
|
||||
public static int O_NOCTTY = -1;
|
||||
public static int O_TRUNC = -1;
|
||||
public static int O_APPEND = -1;
|
||||
public static int O_NONBLOCK = -1;
|
||||
public static int O_SYNC = -1;
|
||||
|
||||
// Flags for posix_fadvise() from bits/fcntl.h
|
||||
// Flags for posix_fadvise() from bits/fcntl.h - Set by JNI
|
||||
/* No further special treatment. */
|
||||
public static final int POSIX_FADV_NORMAL = 0;
|
||||
public static int POSIX_FADV_NORMAL = -1;
|
||||
/* Expect random page references. */
|
||||
public static final int POSIX_FADV_RANDOM = 1;
|
||||
public static int POSIX_FADV_RANDOM = -1;
|
||||
/* Expect sequential page references. */
|
||||
public static final int POSIX_FADV_SEQUENTIAL = 2;
|
||||
public static int POSIX_FADV_SEQUENTIAL = -1;
|
||||
/* Will need these pages. */
|
||||
public static final int POSIX_FADV_WILLNEED = 3;
|
||||
public static int POSIX_FADV_WILLNEED = -1;
|
||||
/* Don't need these pages. */
|
||||
public static final int POSIX_FADV_DONTNEED = 4;
|
||||
public static int POSIX_FADV_DONTNEED = -1;
|
||||
/* Data will be accessed once. */
|
||||
public static final int POSIX_FADV_NOREUSE = 5;
|
||||
public static int POSIX_FADV_NOREUSE = -1;
|
||||
|
||||
|
||||
// Updated by JNI when supported by glibc. Leave defaults in case kernel
|
||||
// supports sync_file_range, but glibc does not.
|
||||
/* Wait upon writeout of all pages
|
||||
in the range before performing the
|
||||
write. */
|
||||
public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1;
|
||||
public static int SYNC_FILE_RANGE_WAIT_BEFORE = 1;
|
||||
/* Initiate writeout of all those
|
||||
dirty pages in the range which are
|
||||
not presently under writeback. */
|
||||
public static final int SYNC_FILE_RANGE_WRITE = 2;
|
||||
|
||||
public static int SYNC_FILE_RANGE_WRITE = 2;
|
||||
/* Wait upon writeout of all pages in
|
||||
the range after performing the
|
||||
write. */
|
||||
public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4;
|
||||
public static int SYNC_FILE_RANGE_WAIT_AFTER = 4;
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(NativeIO.class);
|
||||
|
||||
// Set to true via JNI if possible
|
||||
public static boolean fadvisePossible = false;
|
||||
|
||||
private static boolean nativeLoaded = false;
|
||||
private static boolean fadvisePossible = true;
|
||||
private static boolean syncFileRangePossible = true;
|
||||
|
||||
static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY =
|
||||
@ -262,8 +265,6 @@ static void posixFadviseIfPossible(String identifier,
|
||||
if (nativeLoaded && fadvisePossible) {
|
||||
try {
|
||||
posix_fadvise(fd, offset, len, flags);
|
||||
} catch (UnsupportedOperationException uoe) {
|
||||
fadvisePossible = false;
|
||||
} catch (UnsatisfiedLinkError ule) {
|
||||
fadvisePossible = false;
|
||||
}
|
||||
@ -344,21 +345,21 @@ public static class Stat {
|
||||
private String owner, group;
|
||||
private int mode;
|
||||
|
||||
// Mode constants
|
||||
public static final int S_IFMT = 0170000; /* type of file */
|
||||
public static final int S_IFIFO = 0010000; /* named pipe (fifo) */
|
||||
public static final int S_IFCHR = 0020000; /* character special */
|
||||
public static final int S_IFDIR = 0040000; /* directory */
|
||||
public static final int S_IFBLK = 0060000; /* block special */
|
||||
public static final int S_IFREG = 0100000; /* regular */
|
||||
public static final int S_IFLNK = 0120000; /* symbolic link */
|
||||
public static final int S_IFSOCK = 0140000; /* socket */
|
||||
public static final int S_ISUID = 0004000; /* set user id on execution */
|
||||
public static final int S_ISGID = 0002000; /* set group id on execution */
|
||||
public static final int S_ISVTX = 0001000; /* save swapped text even after use */
|
||||
public static final int S_IRUSR = 0000400; /* read permission, owner */
|
||||
public static final int S_IWUSR = 0000200; /* write permission, owner */
|
||||
public static final int S_IXUSR = 0000100; /* execute/search permission, owner */
|
||||
// Mode constants - Set by JNI
|
||||
public static int S_IFMT = -1; /* type of file */
|
||||
public static int S_IFIFO = -1; /* named pipe (fifo) */
|
||||
public static int S_IFCHR = -1; /* character special */
|
||||
public static int S_IFDIR = -1; /* directory */
|
||||
public static int S_IFBLK = -1; /* block special */
|
||||
public static int S_IFREG = -1; /* regular */
|
||||
public static int S_IFLNK = -1; /* symbolic link */
|
||||
public static int S_IFSOCK = -1; /* socket */
|
||||
public static int S_ISUID = -1; /* set user id on execution */
|
||||
public static int S_ISGID = -1; /* set group id on execution */
|
||||
public static int S_ISVTX = -1; /* save swapped text even after use */
|
||||
public static int S_IRUSR = -1; /* read permission, owner */
|
||||
public static int S_IWUSR = -1; /* write permission, owner */
|
||||
public static int S_IXUSR = -1; /* execute/search permission, owner */
|
||||
|
||||
Stat(int ownerId, int groupId, int mode) {
|
||||
this.ownerId = ownerId;
|
||||
|
||||
@ -58,6 +58,15 @@
|
||||
#define MMAP_PROT_WRITE org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_WRITE
|
||||
#define MMAP_PROT_EXEC org_apache_hadoop_io_nativeio_NativeIO_POSIX_MMAP_PROT_EXEC
|
||||
|
||||
#define NATIVE_IO_POSIX_CLASS "org/apache/hadoop/io/nativeio/NativeIO$POSIX"
|
||||
#define NATIVE_IO_STAT_CLASS "org/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat"
|
||||
|
||||
#define SET_INT_OR_RETURN(E, C, F) \
|
||||
{ \
|
||||
setStaticInt(E, C, #F, F); \
|
||||
if ((*E)->ExceptionCheck(E)) return; \
|
||||
}
|
||||
|
||||
// the NativeIO$POSIX$Stat inner class and its constructor
|
||||
static jclass stat_clazz;
|
||||
static jmethodID stat_ctor;
|
||||
@ -101,12 +110,90 @@ static int workaround_non_threadsafe_calls(JNIEnv *env, jclass clazz) {
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a static boolean field to the specified value.
|
||||
*/
|
||||
static void setStaticBoolean(JNIEnv *env, jclass clazz, char *field,
|
||||
jboolean val) {
|
||||
jfieldID fid = (*env)->GetStaticFieldID(env, clazz, field, "Z");
|
||||
if (fid != NULL) {
|
||||
(*env)->SetStaticBooleanField(env, clazz, fid, val);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a static int field to the specified value.
|
||||
*/
|
||||
static void setStaticInt(JNIEnv *env, jclass clazz, char *field,
|
||||
jint val) {
|
||||
jfieldID fid = (*env)->GetStaticFieldID(env, clazz, field, "I");
|
||||
if (fid != NULL) {
|
||||
(*env)->SetStaticIntField(env, clazz, fid, val);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialises a list of java constants that are platform specific.
|
||||
* These are only initialized in UNIX.
|
||||
* Any exceptions that occur will be dealt at the level above.
|
||||
**/
|
||||
static void consts_init(JNIEnv *env) {
|
||||
jclass clazz = (*env)->FindClass(env, NATIVE_IO_POSIX_CLASS);
|
||||
if (clazz == NULL) {
|
||||
return; // exception has been raised
|
||||
}
|
||||
SET_INT_OR_RETURN(env, clazz, O_RDONLY);
|
||||
SET_INT_OR_RETURN(env, clazz, O_WRONLY);
|
||||
SET_INT_OR_RETURN(env, clazz, O_RDWR);
|
||||
SET_INT_OR_RETURN(env, clazz, O_CREAT);
|
||||
SET_INT_OR_RETURN(env, clazz, O_EXCL);
|
||||
SET_INT_OR_RETURN(env, clazz, O_NOCTTY);
|
||||
SET_INT_OR_RETURN(env, clazz, O_TRUNC);
|
||||
SET_INT_OR_RETURN(env, clazz, O_APPEND);
|
||||
SET_INT_OR_RETURN(env, clazz, O_NONBLOCK);
|
||||
SET_INT_OR_RETURN(env, clazz, O_SYNC);
|
||||
#ifdef HAVE_POSIX_FADVISE
|
||||
setStaticBoolean(env, clazz, "fadvisePossible", JNI_TRUE);
|
||||
SET_INT_OR_RETURN(env, clazz, POSIX_FADV_NORMAL);
|
||||
SET_INT_OR_RETURN(env, clazz, POSIX_FADV_RANDOM);
|
||||
SET_INT_OR_RETURN(env, clazz, POSIX_FADV_SEQUENTIAL);
|
||||
SET_INT_OR_RETURN(env, clazz, POSIX_FADV_WILLNEED);
|
||||
SET_INT_OR_RETURN(env, clazz, POSIX_FADV_DONTNEED);
|
||||
SET_INT_OR_RETURN(env, clazz, POSIX_FADV_NOREUSE);
|
||||
#else
|
||||
setStaticBoolean(env, clazz, "fadvisePossible", JNI_FALSE);
|
||||
#endif
|
||||
#ifdef HAVE_SYNC_FILE_RANGE
|
||||
SET_INT_OR_RETURN(env, clazz, SYNC_FILE_RANGE_WAIT_BEFORE);
|
||||
SET_INT_OR_RETURN(env, clazz, SYNC_FILE_RANGE_WRITE);
|
||||
SET_INT_OR_RETURN(env, clazz, SYNC_FILE_RANGE_WAIT_AFTER);
|
||||
#endif
|
||||
clazz = (*env)->FindClass(env, NATIVE_IO_STAT_CLASS);
|
||||
if (clazz == NULL) {
|
||||
return; // exception has been raised
|
||||
}
|
||||
SET_INT_OR_RETURN(env, clazz, S_IFMT);
|
||||
SET_INT_OR_RETURN(env, clazz, S_IFIFO);
|
||||
SET_INT_OR_RETURN(env, clazz, S_IFCHR);
|
||||
SET_INT_OR_RETURN(env, clazz, S_IFDIR);
|
||||
SET_INT_OR_RETURN(env, clazz, S_IFBLK);
|
||||
SET_INT_OR_RETURN(env, clazz, S_IFREG);
|
||||
SET_INT_OR_RETURN(env, clazz, S_IFLNK);
|
||||
SET_INT_OR_RETURN(env, clazz, S_IFSOCK);
|
||||
SET_INT_OR_RETURN(env, clazz, S_ISUID);
|
||||
SET_INT_OR_RETURN(env, clazz, S_ISGID);
|
||||
SET_INT_OR_RETURN(env, clazz, S_ISVTX);
|
||||
SET_INT_OR_RETURN(env, clazz, S_IRUSR);
|
||||
SET_INT_OR_RETURN(env, clazz, S_IWUSR);
|
||||
SET_INT_OR_RETURN(env, clazz, S_IXUSR);
|
||||
}
|
||||
|
||||
static void stat_init(JNIEnv *env, jclass nativeio_class) {
|
||||
jclass clazz = NULL;
|
||||
jclass obj_class = NULL;
|
||||
jmethodID obj_ctor = NULL;
|
||||
// Init Stat
|
||||
clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat");
|
||||
clazz = (*env)->FindClass(env, NATIVE_IO_STAT_CLASS);
|
||||
if (!clazz) {
|
||||
return; // exception has been raised
|
||||
}
|
||||
@ -180,39 +267,6 @@ static void nioe_deinit(JNIEnv *env) {
|
||||
nioe_ctor = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Compatibility mapping for fadvise flags. Return the proper value from fnctl.h.
|
||||
* If the value is not known, return the argument unchanged.
|
||||
*/
|
||||
static int map_fadvise_flag(jint flag) {
|
||||
#ifdef HAVE_POSIX_FADVISE
|
||||
switch(flag) {
|
||||
case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_NORMAL:
|
||||
return POSIX_FADV_NORMAL;
|
||||
break;
|
||||
case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_RANDOM:
|
||||
return POSIX_FADV_RANDOM;
|
||||
break;
|
||||
case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_SEQUENTIAL:
|
||||
return POSIX_FADV_SEQUENTIAL;
|
||||
break;
|
||||
case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_WILLNEED:
|
||||
return POSIX_FADV_WILLNEED;
|
||||
break;
|
||||
case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_DONTNEED:
|
||||
return POSIX_FADV_DONTNEED;
|
||||
break;
|
||||
case org_apache_hadoop_io_nativeio_NativeIO_POSIX_POSIX_FADV_NOREUSE:
|
||||
return POSIX_FADV_NOREUSE;
|
||||
break;
|
||||
default:
|
||||
return flag;
|
||||
}
|
||||
#else
|
||||
return flag;
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* private static native void initNative();
|
||||
*
|
||||
@ -223,6 +277,10 @@ static int map_fadvise_flag(jint flag) {
|
||||
JNIEXPORT void JNICALL
|
||||
Java_org_apache_hadoop_io_nativeio_NativeIO_initNative(
|
||||
JNIEnv *env, jclass clazz) {
|
||||
#ifdef UNIX
|
||||
consts_init(env);
|
||||
PASS_EXCEPTIONS_GOTO(env, error);
|
||||
#endif
|
||||
stat_init(env, clazz);
|
||||
PASS_EXCEPTIONS_GOTO(env, error);
|
||||
nioe_init(env);
|
||||
@ -345,7 +403,7 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_posix_1fadvise(
|
||||
PASS_EXCEPTIONS(env);
|
||||
|
||||
int err = 0;
|
||||
if ((err = posix_fadvise(fd, (off_t)offset, (off_t)len, map_fadvise_flag(flags)))) {
|
||||
if ((err = posix_fadvise(fd, (off_t)offset, (off_t)len, flags))) {
|
||||
#ifdef __FreeBSD__
|
||||
throw_ioe(env, errno);
|
||||
#else
|
||||
@ -448,22 +506,6 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_mlock_1native(
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef __FreeBSD__
|
||||
static int toFreeBSDFlags(int flags)
|
||||
{
|
||||
int rc = flags & 03;
|
||||
if ( flags & 0100 ) rc |= O_CREAT;
|
||||
if ( flags & 0200 ) rc |= O_EXCL;
|
||||
if ( flags & 0400 ) rc |= O_NOCTTY;
|
||||
if ( flags & 01000 ) rc |= O_TRUNC;
|
||||
if ( flags & 02000 ) rc |= O_APPEND;
|
||||
if ( flags & 04000 ) rc |= O_NONBLOCK;
|
||||
if ( flags &010000 ) rc |= O_SYNC;
|
||||
if ( flags &020000 ) rc |= O_ASYNC;
|
||||
return rc;
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Class: org_apache_hadoop_io_nativeio_NativeIO_POSIX
|
||||
* Method: open
|
||||
@ -479,9 +521,6 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_open(
|
||||
jint flags, jint mode)
|
||||
{
|
||||
#ifdef UNIX
|
||||
#ifdef __FreeBSD__
|
||||
flags = toFreeBSDFlags(flags);
|
||||
#endif
|
||||
jobject ret = NULL;
|
||||
|
||||
const char *path = (*env)->GetStringUTFChars(env, j_path, NULL);
|
||||
|
||||
@ -55,6 +55,9 @@
|
||||
import org.apache.hadoop.util.NativeCodeLoader;
|
||||
import org.apache.hadoop.util.Time;
|
||||
|
||||
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.*;
|
||||
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.Stat.*;
|
||||
|
||||
public class TestNativeIO {
|
||||
static final Log LOG = LogFactory.getLog(TestNativeIO.class);
|
||||
|
||||
@ -93,9 +96,8 @@ public void testFstat() throws Exception {
|
||||
assertEquals(expectedOwner, owner);
|
||||
assertNotNull(stat.getGroup());
|
||||
assertTrue(!stat.getGroup().isEmpty());
|
||||
assertEquals("Stat mode field should indicate a regular file",
|
||||
NativeIO.POSIX.Stat.S_IFREG,
|
||||
stat.getMode() & NativeIO.POSIX.Stat.S_IFMT);
|
||||
assertEquals("Stat mode field should indicate a regular file", S_IFREG,
|
||||
stat.getMode() & S_IFMT);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -128,8 +130,7 @@ public void run() {
|
||||
assertNotNull(stat.getGroup());
|
||||
assertTrue(!stat.getGroup().isEmpty());
|
||||
assertEquals("Stat mode field should indicate a regular file",
|
||||
NativeIO.POSIX.Stat.S_IFREG,
|
||||
stat.getMode() & NativeIO.POSIX.Stat.S_IFMT);
|
||||
S_IFREG, stat.getMode() & S_IFMT);
|
||||
} catch (Throwable t) {
|
||||
thrown.set(t);
|
||||
}
|
||||
@ -338,8 +339,7 @@ public void testOpenMissingWithoutCreate() throws Exception {
|
||||
LOG.info("Open a missing file without O_CREAT and it should fail");
|
||||
try {
|
||||
FileDescriptor fd = NativeIO.POSIX.open(
|
||||
new File(TEST_DIR, "doesntexist").getAbsolutePath(),
|
||||
NativeIO.POSIX.O_WRONLY, 0700);
|
||||
new File(TEST_DIR, "doesntexist").getAbsolutePath(), O_WRONLY, 0700);
|
||||
fail("Able to open a new file without O_CREAT");
|
||||
} catch (NativeIOException nioe) {
|
||||
LOG.info("Got expected exception", nioe);
|
||||
@ -356,7 +356,7 @@ public void testOpenWithCreate() throws Exception {
|
||||
LOG.info("Test creating a file with O_CREAT");
|
||||
FileDescriptor fd = NativeIO.POSIX.open(
|
||||
new File(TEST_DIR, "testWorkingOpen").getAbsolutePath(),
|
||||
NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT, 0700);
|
||||
O_WRONLY | O_CREAT, 0700);
|
||||
assertNotNull(true);
|
||||
assertTrue(fd.valid());
|
||||
FileOutputStream fos = new FileOutputStream(fd);
|
||||
@ -369,7 +369,7 @@ public void testOpenWithCreate() throws Exception {
|
||||
try {
|
||||
fd = NativeIO.POSIX.open(
|
||||
new File(TEST_DIR, "testWorkingOpen").getAbsolutePath(),
|
||||
NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT | NativeIO.POSIX.O_EXCL, 0700);
|
||||
O_WRONLY | O_CREAT | O_EXCL, 0700);
|
||||
fail("Was able to create existing file with O_EXCL");
|
||||
} catch (NativeIOException nioe) {
|
||||
LOG.info("Got expected exception for failed exclusive create", nioe);
|
||||
@ -390,7 +390,7 @@ public void testFDDoesntLeak() throws IOException {
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
FileDescriptor fd = NativeIO.POSIX.open(
|
||||
new File(TEST_DIR, "testNoFdLeak").getAbsolutePath(),
|
||||
NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT, 0700);
|
||||
O_WRONLY | O_CREAT, 0700);
|
||||
assertNotNull(true);
|
||||
assertTrue(fd.valid());
|
||||
FileOutputStream fos = new FileOutputStream(fd);
|
||||
@ -436,8 +436,7 @@ public void testPosixFadvise() throws Exception {
|
||||
FileInputStream fis = new FileInputStream("/dev/zero");
|
||||
try {
|
||||
NativeIO.POSIX.posix_fadvise(
|
||||
fis.getFD(), 0, 0,
|
||||
NativeIO.POSIX.POSIX_FADV_SEQUENTIAL);
|
||||
fis.getFD(), 0, 0, POSIX_FADV_SEQUENTIAL);
|
||||
} catch (UnsupportedOperationException uoe) {
|
||||
// we should just skip the unit test on machines where we don't
|
||||
// have fadvise support
|
||||
@ -450,20 +449,14 @@ public void testPosixFadvise() throws Exception {
|
||||
}
|
||||
|
||||
try {
|
||||
NativeIO.POSIX.posix_fadvise(
|
||||
fis.getFD(), 0, 1024,
|
||||
NativeIO.POSIX.POSIX_FADV_SEQUENTIAL);
|
||||
|
||||
NativeIO.POSIX.posix_fadvise(fis.getFD(), 0, 1024, POSIX_FADV_SEQUENTIAL);
|
||||
fail("Did not throw on bad file");
|
||||
} catch (NativeIOException nioe) {
|
||||
assertEquals(Errno.EBADF, nioe.getErrno());
|
||||
}
|
||||
|
||||
try {
|
||||
NativeIO.POSIX.posix_fadvise(
|
||||
null, 0, 1024,
|
||||
NativeIO.POSIX.POSIX_FADV_SEQUENTIAL);
|
||||
|
||||
NativeIO.POSIX.posix_fadvise(null, 0, 1024, POSIX_FADV_SEQUENTIAL);
|
||||
fail("Did not throw on null file");
|
||||
} catch (NullPointerException npe) {
|
||||
// expected
|
||||
@ -476,9 +469,8 @@ public void testSyncFileRange() throws Exception {
|
||||
new File(TEST_DIR, "testSyncFileRange"));
|
||||
try {
|
||||
fos.write("foo".getBytes());
|
||||
NativeIO.POSIX.sync_file_range(
|
||||
fos.getFD(), 0, 1024,
|
||||
NativeIO.POSIX.SYNC_FILE_RANGE_WRITE);
|
||||
NativeIO.POSIX.sync_file_range(fos.getFD(), 0, 1024,
|
||||
SYNC_FILE_RANGE_WRITE);
|
||||
// no way to verify that this actually has synced,
|
||||
// but if it doesn't throw, we can assume it worked
|
||||
} catch (UnsupportedOperationException uoe) {
|
||||
@ -489,9 +481,8 @@ public void testSyncFileRange() throws Exception {
|
||||
fos.close();
|
||||
}
|
||||
try {
|
||||
NativeIO.POSIX.sync_file_range(
|
||||
fos.getFD(), 0, 1024,
|
||||
NativeIO.POSIX.SYNC_FILE_RANGE_WRITE);
|
||||
NativeIO.POSIX.sync_file_range(fos.getFD(), 0, 1024,
|
||||
SYNC_FILE_RANGE_WRITE);
|
||||
fail("Did not throw on bad file");
|
||||
} catch (NativeIOException nioe) {
|
||||
assertEquals(Errno.EBADF, nioe.getErrno());
|
||||
@ -657,4 +648,51 @@ public void testCopyFileUnbuffered() throws Exception {
|
||||
FileUtils.deleteQuietly(TEST_DIR);
|
||||
}
|
||||
}
|
||||
|
||||
@Test (timeout=10000)
|
||||
public void testNativePosixConsts() {
|
||||
assumeTrue("Native POSIX constants not required for Windows",
|
||||
!Path.WINDOWS);
|
||||
assertTrue("Native 0_RDONLY const not set", O_RDONLY >= 0);
|
||||
assertTrue("Native 0_WRONLY const not set", O_WRONLY >= 0);
|
||||
assertTrue("Native 0_RDWR const not set", O_RDWR >= 0);
|
||||
assertTrue("Native 0_CREAT const not set", O_CREAT >= 0);
|
||||
assertTrue("Native 0_EXCL const not set", O_EXCL >= 0);
|
||||
assertTrue("Native 0_NOCTTY const not set", O_NOCTTY >= 0);
|
||||
assertTrue("Native 0_TRUNC const not set", O_TRUNC >= 0);
|
||||
assertTrue("Native 0_APPEND const not set", O_APPEND >= 0);
|
||||
assertTrue("Native 0_NONBLOCK const not set", O_NONBLOCK >= 0);
|
||||
assertTrue("Native 0_SYNC const not set", O_SYNC >= 0);
|
||||
assertTrue("Native S_IFMT const not set", S_IFMT >= 0);
|
||||
assertTrue("Native S_IFIFO const not set", S_IFIFO >= 0);
|
||||
assertTrue("Native S_IFCHR const not set", S_IFCHR >= 0);
|
||||
assertTrue("Native S_IFDIR const not set", S_IFDIR >= 0);
|
||||
assertTrue("Native S_IFBLK const not set", S_IFBLK >= 0);
|
||||
assertTrue("Native S_IFREG const not set", S_IFREG >= 0);
|
||||
assertTrue("Native S_IFLNK const not set", S_IFLNK >= 0);
|
||||
assertTrue("Native S_IFSOCK const not set", S_IFSOCK >= 0);
|
||||
assertTrue("Native S_ISUID const not set", S_ISUID >= 0);
|
||||
assertTrue("Native S_ISGID const not set", S_ISGID >= 0);
|
||||
assertTrue("Native S_ISVTX const not set", S_ISVTX >= 0);
|
||||
assertTrue("Native S_IRUSR const not set", S_IRUSR >= 0);
|
||||
assertTrue("Native S_IWUSR const not set", S_IWUSR >= 0);
|
||||
assertTrue("Native S_IXUSR const not set", S_IXUSR >= 0);
|
||||
}
|
||||
|
||||
@Test (timeout=10000)
|
||||
public void testNativeFadviseConsts() {
|
||||
assumeTrue("Fadvise constants not supported", fadvisePossible);
|
||||
assertTrue("Native POSIX_FADV_NORMAL const not set",
|
||||
POSIX_FADV_NORMAL >= 0);
|
||||
assertTrue("Native POSIX_FADV_RANDOM const not set",
|
||||
POSIX_FADV_RANDOM >= 0);
|
||||
assertTrue("Native POSIX_FADV_SEQUENTIAL const not set",
|
||||
POSIX_FADV_SEQUENTIAL >= 0);
|
||||
assertTrue("Native POSIX_FADV_WILLNEED const not set",
|
||||
POSIX_FADV_WILLNEED >= 0);
|
||||
assertTrue("Native POSIX_FADV_DONTNEED const not set",
|
||||
POSIX_FADV_DONTNEED >= 0);
|
||||
assertTrue("Native POSIX_FADV_NOREUSE const not set",
|
||||
POSIX_FADV_NOREUSE >= 0);
|
||||
}
|
||||
}
|
||||
|
||||
@ -58,6 +58,9 @@
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.util.Time;
|
||||
|
||||
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
|
||||
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.SYNC_FILE_RANGE_WRITE;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
/** A class that receives a block and writes to its own disk, meanwhile
|
||||
@ -791,12 +794,12 @@ private void manageWriterOsCache(long offsetInBlock) {
|
||||
this.datanode.getFSDataset().submitBackgroundSyncFileRangeRequest(
|
||||
block, outFd, lastCacheManagementOffset,
|
||||
offsetInBlock - lastCacheManagementOffset,
|
||||
NativeIO.POSIX.SYNC_FILE_RANGE_WRITE);
|
||||
SYNC_FILE_RANGE_WRITE);
|
||||
} else {
|
||||
NativeIO.POSIX.syncFileRangeIfPossible(outFd,
|
||||
lastCacheManagementOffset, offsetInBlock
|
||||
- lastCacheManagementOffset,
|
||||
NativeIO.POSIX.SYNC_FILE_RANGE_WRITE);
|
||||
lastCacheManagementOffset,
|
||||
offsetInBlock - lastCacheManagementOffset,
|
||||
SYNC_FILE_RANGE_WRITE);
|
||||
}
|
||||
}
|
||||
//
|
||||
@ -812,8 +815,7 @@ private void manageWriterOsCache(long offsetInBlock) {
|
||||
long dropPos = lastCacheManagementOffset - CACHE_DROP_LAG_BYTES;
|
||||
if (dropPos > 0 && dropCacheBehindWrites) {
|
||||
NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(
|
||||
block.getBlockName(), outFd, 0, dropPos,
|
||||
NativeIO.POSIX.POSIX_FADV_DONTNEED);
|
||||
block.getBlockName(), outFd, 0, dropPos, POSIX_FADV_DONTNEED);
|
||||
}
|
||||
lastCacheManagementOffset = offsetInBlock;
|
||||
long duration = Time.monotonicNow() - begin;
|
||||
|
||||
@ -51,6 +51,9 @@
|
||||
import org.apache.htrace.Trace;
|
||||
import org.apache.htrace.TraceScope;
|
||||
|
||||
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
|
||||
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_SEQUENTIAL;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
@ -411,8 +414,7 @@ public void close() throws IOException {
|
||||
try {
|
||||
NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(
|
||||
block.getBlockName(), blockInFd, lastCacheDropOffset,
|
||||
offset - lastCacheDropOffset,
|
||||
NativeIO.POSIX.POSIX_FADV_DONTNEED);
|
||||
offset - lastCacheDropOffset, POSIX_FADV_DONTNEED);
|
||||
} catch (Exception e) {
|
||||
LOG.warn("Unable to drop cache on file close", e);
|
||||
}
|
||||
@ -729,8 +731,7 @@ private long doSendBlock(DataOutputStream out, OutputStream baseStream,
|
||||
if (isLongRead() && blockInFd != null) {
|
||||
// Advise that this file descriptor will be accessed sequentially.
|
||||
NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(
|
||||
block.getBlockName(), blockInFd, 0, 0,
|
||||
NativeIO.POSIX.POSIX_FADV_SEQUENTIAL);
|
||||
block.getBlockName(), blockInFd, 0, 0, POSIX_FADV_SEQUENTIAL);
|
||||
}
|
||||
|
||||
// Trigger readahead of beginning of file if configured.
|
||||
@ -818,7 +819,7 @@ private void manageOsCache() throws IOException {
|
||||
long dropLength = offset - lastCacheDropOffset;
|
||||
NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(
|
||||
block.getBlockName(), blockInFd, lastCacheDropOffset,
|
||||
dropLength, NativeIO.POSIX.POSIX_FADV_DONTNEED);
|
||||
dropLength, POSIX_FADV_DONTNEED);
|
||||
lastCacheDropOffset = offset;
|
||||
}
|
||||
}
|
||||
|
||||
@ -39,6 +39,9 @@
|
||||
import org.apache.hadoop.io.nativeio.NativeIO;
|
||||
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.CacheManipulator;
|
||||
import org.apache.hadoop.io.nativeio.NativeIOException;
|
||||
|
||||
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
@ -78,7 +81,7 @@ private static class Stats {
|
||||
synchronized void fadvise(int offset, int len, int flags) {
|
||||
LOG.debug("got fadvise(offset=" + offset + ", len=" + len +
|
||||
",flags=" + flags + ")");
|
||||
if (flags == NativeIO.POSIX.POSIX_FADV_DONTNEED) {
|
||||
if (flags == POSIX_FADV_DONTNEED) {
|
||||
for (int i = 0; i < len; i++) {
|
||||
dropped[(offset + i)] = true;
|
||||
}
|
||||
|
||||
@ -27,6 +27,9 @@
|
||||
import org.apache.hadoop.io.ReadaheadPool;
|
||||
import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
|
||||
import org.apache.hadoop.io.nativeio.NativeIO;
|
||||
|
||||
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
|
||||
|
||||
import org.jboss.netty.handler.stream.ChunkedFile;
|
||||
|
||||
public class FadvisedChunkedFile extends ChunkedFile {
|
||||
@ -72,7 +75,7 @@ public void close() throws Exception {
|
||||
NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
|
||||
fd,
|
||||
getStartOffset(), getEndOffset() - getStartOffset(),
|
||||
NativeIO.POSIX.POSIX_FADV_DONTNEED);
|
||||
POSIX_FADV_DONTNEED);
|
||||
} catch (Throwable t) {
|
||||
LOG.warn("Failed to manage OS cache for " + identifier, t);
|
||||
}
|
||||
|
||||
@ -30,6 +30,9 @@
|
||||
import org.apache.hadoop.io.ReadaheadPool;
|
||||
import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
|
||||
import org.apache.hadoop.io.nativeio.NativeIO;
|
||||
|
||||
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
|
||||
|
||||
import org.jboss.netty.channel.DefaultFileRegion;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
@ -155,8 +158,7 @@ public void transferSuccessful() {
|
||||
if (manageOsCache && getCount() > 0) {
|
||||
try {
|
||||
NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
|
||||
fd, getPosition(), getCount(),
|
||||
NativeIO.POSIX.POSIX_FADV_DONTNEED);
|
||||
fd, getPosition(), getCount(), POSIX_FADV_DONTNEED);
|
||||
} catch (Throwable t) {
|
||||
LOG.warn("Failed to manage OS cache for " + identifier, t);
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user