LUCENE-8982: Separate out native code to another module to allow cpp build with gradle (#2068)

* LUCENE-8982: Separate out native code to another module to allow cpp build with gradle
This commit is contained in:
zacharymorn 2020-11-16 00:40:03 -08:00 committed by GitHub
parent 3ae0ca23d9
commit ebc87a8a27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 348 additions and 47 deletions

View File

@ -154,6 +154,9 @@ apply from: file('gradle/generation/snowball.gradle')
apply from: file('gradle/generation/kuromoji.gradle')
apply from: file('gradle/generation/nori.gradle')
// Shared configuration of subprojects containing native code.
apply from: file('gradle/native/disable-native.gradle')
// Additional development aids.
apply from: file('gradle/maven/maven-local.gradle')
apply from: file('gradle/testing/per-project-summary.gradle')

View File

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This is the master switch to disable all tasks that compile
// native (cpp) code.
rootProject.ext {
buildNative = propertyOrDefault("build.native", true).toBoolean()
}
// Explicitly list all projects that should be configured for native extensions.
// We could scan for projects with a the cpp-library plugin but this is faster.
def nativeProjects = allprojects.findAll {it.path in [
":lucene:misc:native"
]}
def javaProjectsWithNativeDeps = allprojects.findAll {it.path in [
":lucene:misc"
]}
// Set up defaults for projects with native dependencies.
configure(javaProjectsWithNativeDeps, {
configurations {
nativeDeps {
attributes {
attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage.class, Usage.NATIVE_RUNTIME))
attributes.attribute(CppBinary.OPTIMIZED_ATTRIBUTE, false)
}
}
}
plugins.withType(JavaPlugin) {
ext {
testOptions += [
[propName: 'tests.native', value: buildNative, description: "Enable tests that require native extensions."]
]
nativeDepsDir = file("${buildDir}/nativeDeps")
}
// Only copy and attach native deps if native build is enabled.
if (buildNative) {
task copyNativeDeps(type: Sync) {
from configurations.nativeDeps
into nativeDepsDir
}
tasks.withType(Test) {
dependsOn copyNativeDeps
systemProperty "java.library.path", nativeDepsDir
}
}
}
})
// If native build is disabled we just disable all tasks in the active task set that
// originate from "native" projects.
//
// Perhaps there is a cleaner way to do it but removing their references from
// settings.gradle would remove them from IDE detection, dependency resolution, etc.
// This way seems better.
if (!buildNative) {
gradle.taskGraph.whenReady { taskGraph ->
def tasks = taskGraph.getAllTasks()
tasks.findAll { task -> task.project in nativeProjects }.each { task ->
task.enabled = false
}
}
}

View File

@ -62,6 +62,10 @@ grant {
permission java.lang.RuntimePermission "getClassLoader";
permission java.lang.RuntimePermission "setContextClassLoader";
// Needed for loading native library (lucene:misc:native) in lucene:misc
permission java.lang.RuntimePermission "loadLibrary.LuceneNativeIO";
permission java.lang.RuntimePermission "writeFileDescriptor";
// TestLockFactoriesMultiJVM opens a random port on 127.0.0.1 (port 0 = ephemeral port range):
permission java.net.SocketPermission "127.0.0.1:0", "accept,listen,resolve";

View File

@ -18,6 +18,10 @@ System Requirements
API Changes
* LUCENE-8982: Separate out native code to another module to allow cpp
build with gradle. This also changes the name of the native "posix-support"
library to LuceneNativeIO. (Zachary Chen, Dawid Weiss)
* LUCENE-9562: All binary analysis packages (and corresponding
Maven artifacts) with names containing '-analyzers-' have been renamed
to '-analysis-'. (Dawid Weiss)

View File

@ -22,4 +22,6 @@ description = 'Index tools and other miscellaneous code'
dependencies {
api project(':lucene:core')
testImplementation project(':lucene:test-framework')
}
nativeDeps project(":lucene:misc:native")
}

View File

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* This gets separated out from misc module into a native module due to incompatibility between cpp-library and java-library plugins.
* For details, please see https://github.com/gradle/gradle-native/issues/352#issuecomment-461724948
*/
import org.apache.tools.ant.taskdefs.condition.Os
description = 'Module for native code'
apply plugin: 'cpp-library'
library {
baseName = 'LuceneNativeIO'
// Native build for Windows platform will be added in later stage
targetMachines = [
machines.linux.x86_64,
machines.macOS.x86_64,
machines.windows.x86_64
]
// Point at platform-specific sources. Other platforms will be ignored
// (plugin won't find the toolchain).
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
source.from file("${projectDir}/src/main/windows")
} else if (Os.isFamily(Os.FAMILY_UNIX) || Os.isFamily(Os.FAMILY_MAC)) {
source.from file("${projectDir}/src/main/posix")
}
}
tasks.withType(CppCompile).configureEach {
def javaHome = rootProject.ext.runtimeJava.getInstallationDirectory().getAsFile().getPath()
// Assume standard openjdk layout. This means only one architecture-specific include folder
// is present.
systemIncludes.from file("${javaHome}/include")
for (def path : [
file("${javaHome}/include/win32"),
file("${javaHome}/include/darwin"),
file("${javaHome}/include/linux"),
file("${javaHome}/include/solaris")]) {
if (path.exists()) {
systemIncludes.from path
}
}
compilerArgs.add '-fPIC'
}
tasks.withType(LinkSharedLibrary).configureEach {
linkerArgs.add '-lstdc++'
}

View File

@ -38,12 +38,12 @@
#ifdef LINUX
/*
* Class: org_apache_lucene_store_NativePosixUtil
* Class: org_apache_lucene_misc_store_NativePosixUtil
* Method: posix_fadvise
* Signature: (Ljava/io/FileDescriptor;JJI)V
*/
extern "C"
JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_posix_1fadvise(JNIEnv *env, jclass _ignore, jobject fileDescriptor, jlong offset, jlong len, jint advice)
JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_NativePosixUtil_posix_1fadvise(JNIEnv *env, jclass _ignore, jobject fileDescriptor, jlong offset, jlong len, jint advice)
{
jfieldID field_fd;
jmethodID const_fdesc;
@ -103,12 +103,12 @@ JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_posix_1fadvi
#endif
/*
* Class: org_apache_lucene_store_NativePosixUtil
* Class: org_apache_lucene_misc_store_NativePosixUtil
* Method: open_direct
* Signature: (Ljava/lang/String;Z)Ljava/io/FileDescriptor;
*/
extern "C"
JNIEXPORT jobject JNICALL Java_org_apache_lucene_store_NativePosixUtil_open_1direct(JNIEnv *env, jclass _ignore, jstring filename, jboolean readOnly)
JNIEXPORT jobject JNICALL Java_org_apache_lucene_misc_store_NativePosixUtil_open_1direct(JNIEnv *env, jclass _ignore, jstring filename, jboolean readOnly)
{
jfieldID field_fd;
jmethodID const_fdesc;
@ -169,12 +169,12 @@ JNIEXPORT jobject JNICALL Java_org_apache_lucene_store_NativePosixUtil_open_1dir
}
/*
* Class: org_apache_lucene_store_NativePosixUtil
* Class: org_apache_lucene_misc_store_NativePosixUtil
* Method: pread
* Signature: (Ljava/io/FileDescriptor;JLjava/nio/ByteBuffer;)I
*/
extern "C"
JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_NativePosixUtil_pread(JNIEnv *env, jclass _ignore, jobject jfd, jlong pos, jobject byteBuf)
JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_NativePosixUtil_pread(JNIEnv *env, jclass _ignore, jobject jfd, jlong pos, jobject byteBuf)
{
// get int fd:
jclass class_fdesc = env->FindClass("java/io/FileDescriptor");
@ -214,12 +214,12 @@ JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_NativePosixUtil_pread(JNIEn
}
/*
* Class: org_apache_lucene_store_NativePosixUtil
* Class: org_apache_lucene_misc_store_NativePosixUtil
* Method: posix_madvise
* Signature: (Ljava/nio/ByteBuffer;I)I
*/
extern "C"
JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_posix_1madvise(JNIEnv *env, jclass _ignore, jobject buffer, jint advice) {
JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_NativePosixUtil_posix_1madvise(JNIEnv *env, jclass _ignore, jobject buffer, jint advice) {
void *p = env->GetDirectBufferAddress(buffer);
if (p == NULL) {
return -1;
@ -280,12 +280,12 @@ JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_posix_1madvi
/*
* Class: org_apache_lucene_store_NativePosixUtil
* Class: org_apache_lucene_misc_store_NativePosixUtil
* Method: madvise
* Signature: (Ljava/nio/ByteBuffer;I)I
*/
extern "C"
JNIEXPORT jint JNICALL Java_org_apache_lucene_store_NativePosixUtil_madvise(JNIEnv *env, jclass _ignore, jobject buffer, jint advice) {
JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_NativePosixUtil_madvise(JNIEnv *env, jclass _ignore, jobject buffer, jint advice) {
void *p = env->GetDirectBufferAddress(buffer);
if (p == NULL) {
return -1;

View File

@ -56,11 +56,11 @@ void throwException(JNIEnv *env, const char *clazz, const char *msg)
/**
* Opens a handle to a file.
*
* Class: org_apache_lucene_store_WindowsDirectory
* Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: open
* Signature: (Ljava/lang/String;)J
*/
JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_WindowsDirectory_open
JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_open
(JNIEnv *env, jclass ignored, jstring filename)
{
char *fname;
@ -95,11 +95,11 @@ JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_WindowsDirectory_open
* Reads data into the byte array, starting at offset, for length characters.
* The read is positioned at pos.
*
* Class: org_apache_lucene_store_WindowsDirectory
* Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: read
* Signature: (J[BIIJ)I
*/
JNIEXPORT jint JNICALL Java_org_apache_lucene_store_WindowsDirectory_read
JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_read
(JNIEnv *env, jclass ignored, jlong fd, jbyteArray bytes, jint offset, jint length, jlong pos)
{
OVERLAPPED io = { 0 };
@ -140,11 +140,11 @@ JNIEXPORT jint JNICALL Java_org_apache_lucene_store_WindowsDirectory_read
/**
* Closes a handle to a file
*
* Class: org_apache_lucene_store_WindowsDirectory
* Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: close
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_apache_lucene_store_WindowsDirectory_close
JNIEXPORT void JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_close
(JNIEnv *env, jclass ignored, jlong fd)
{
if (!CloseHandle((HANDLE) fd)) {
@ -155,11 +155,11 @@ JNIEXPORT void JNICALL Java_org_apache_lucene_store_WindowsDirectory_close
/**
* Returns the length in bytes of a file.
*
* Class: org_apache_lucene_store_WindowsDirectory
* Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: length
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL Java_org_apache_lucene_store_WindowsDirectory_length
JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_length
(JNIEnv *env, jclass ignored, jlong fd)
{
BY_HANDLE_FILE_INFORMATION info;

View File

@ -33,7 +33,7 @@ public final class NativePosixUtil {
public final static int NOREUSE = 5;
static {
System.loadLibrary("NativePosixUtil");
System.loadLibrary("LuceneNativeIO");
}
private static native int posix_fadvise(FileDescriptor fd, long offset, long len, int advise) throws IOException;

View File

@ -56,10 +56,9 @@ import org.apache.lucene.util.SuppressForbidden;
*
* <p>To use this you must compile
* NativePosixUtil.cpp (exposes Linux-specific APIs through
* JNI) for your platform, by running <code>ant
* build-native-unix</code>, and then putting the resulting
* <code>libNativePosixUtil.so</code> (from
* <code>lucene/build/native</code>) onto your dynamic
* JNI) for your platform, by running <code>./gradlew build</code>, and then putting the resulting
* <code>libLuceneNativeIO.so</code> or <code>libLuceneNativeIO.dylib</code>
* (from <code>lucene/misc/native/build/lib/release/platform/</code>) onto your dynamic
* linker search path.
*
* <p><b>WARNING</b>: this code is very new and quite easily

View File

@ -34,14 +34,10 @@ import java.nio.file.Path;
* <p>
* Steps:
* <ol>
* <li>Compile the source code to create WindowsDirectory.dll:
* <blockquote>
* c:\mingw\bin\g++ -Wall -D_JNI_IMPLEMENTATION_ -Wl,--kill-at
* -I"%JAVA_HOME%\include" -I"%JAVA_HOME%\include\win32" -static-libgcc
* -static-libstdc++ -shared WindowsDirectory.cpp -o WindowsDirectory.dll
* </blockquote>
* For 64-bit JREs, use mingw64, with the -m64 option.
* <li>Put WindowsDirectory.dll into some directory in your windows PATH
* <li>Compile the source code to create libLuceneNativeIO.dll: <code>./gradlew build</code>
* <li>Put the resulting <code>libLuceneNativeIO.dll</code>
* (from <code>lucene/misc/native/build/lib/release/platform/</code>)
* into some directory in your windows PATH
* <li>Open indexes with WindowsDirectory and use it.
* </ol>
* @lucene.experimental
@ -50,7 +46,7 @@ public class WindowsDirectory extends FSDirectory {
private static final int DEFAULT_BUFFERSIZE = 4096; /* default pgsize on ia32/amd64 */
static {
System.loadLibrary("WindowsDirectory");
System.loadLibrary("LuceneNativeIO");
}
/** Create a new WindowsDirectory for the named location.

View File

@ -42,26 +42,25 @@ used during merging of segments larger than a specified size (default
searching, keeping search more responsive while large merges run.
<p>
See <a target=_top href="http://blog.mikemccandless.com/2010/06/lucene-and-fadvisemadvise.html">this blog post</a>
See <a target="_top" href="http://blog.mikemccandless.com/2010/06/lucene-and-fadvisemadvise.html">this blog post</a>
for details.
Steps to build:
<p>Steps to build (from the project's root directory):
<ul>
<li> <code>cd lucene/misc/</code>
<li> To compile NativePosixUtil.cpp -&gt; libNativePosixUtil.so, run<code> ant build-native-unix</code>.
<li>Compile both the native library part (<code>libLuceneNativeIO</code>) and Java sources with:
<code>./gradlew -p lucene/misc build</code>.</li>
<li><code>libNativePosixUtil.so</code> will be located in the <code>lucene/build/native/</code> folder
<li>The native library will be located in the <code>lucene/misc/native/build/lib/main/release/<i>your-platform</i></code> folder.</li>
<li> Make sure libNativePosixUtil.so is on your LD_LIBRARY_PATH so java can find it (something like <code>export LD_LIBRARY_PATH=/path/to/dir:$LD_LIBRARY_PATH</code>, where /path/to/dir contains libNativePosixUtil.so)
<li> <code>ant jar</code> to compile the java source and put that JAR on your CLASSPATH
<li>On Unix-ish systems, make sure <code>libNativePosixUtil.so</code> is on your
<code>LD_LIBRARY_PATH</code> so java can find it (something like <code>export LD_LIBRARY_PATH=/path/to/dir:$LD_LIBRARY_PATH</code>,
where /path/to/dir contains <code>libLuceneNativeIO.so</code>).</li>
</ul>
<p>
NativePosixUtil.cpp/java also expose access to the posix_madvise,
madvise, posix_fadvise functions, which are somewhat more cross
platform than O_DIRECT, however, in testing (see above link), these
The native library exposes access to the <code>posix_madvise</code>,
<code>madvise</code>, <code>posix_fadvise</code> functions, which are somewhat more cross
platform than <code>O_DIRECT</code>, however, in testing (see above link), these
APIs did not seem to help prevent buffer cache eviction.
</body>

View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.misc.store;
import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
import org.apache.lucene.util.Constants;
import org.junit.Assume;
import java.util.Set;
public class NativeLibEnableRule extends TestRuleAdapter {
enum OperatingSystem {
LINUX(Constants.LINUX),
WINDOWS(Constants.WINDOWS),
SUN_OS(Constants.SUN_OS),
MAC(Constants.MAC_OS_X),
FREE_BSD(Constants.FREE_BSD);
public final boolean enabled;
OperatingSystem(boolean enabled) {
this.enabled = enabled;
}
}
private final Set<OperatingSystem> runOn;
public NativeLibEnableRule(Set<OperatingSystem> runOn) {
this.runOn = runOn;
}
@Override
protected void before() {
Assume.assumeTrue("Test ignored (tests.native is false)",
Boolean.parseBoolean(System.getProperty("tests.native", "false")));
Assume.assumeTrue("Test ignored, only applies to architectures: " + runOn,
runOn.stream().anyMatch(os -> os.enabled));
}
}

View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.misc.store;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Rule;
import org.junit.rules.TestRule;
import java.io.IOException;
import java.util.EnumSet;
public class NativeUnixDirectoryTest extends LuceneTestCase {
@Rule
public static TestRule requiresNative = new NativeLibEnableRule(
EnumSet.of(NativeLibEnableRule.OperatingSystem.MAC,
NativeLibEnableRule.OperatingSystem.FREE_BSD,
NativeLibEnableRule.OperatingSystem.LINUX));
public void testLibraryLoaded() throws IOException {
try (ByteBuffersDirectory ramDir = new ByteBuffersDirectory();
Directory dir = new NativeUnixDirectory(RandomizedTest.newTempDir(LifecycleScope.TEST), ramDir)) {
MergeInfo mergeInfo = new MergeInfo(1000, Integer.MAX_VALUE, true, 1);
dir.createOutput("test", new IOContext(mergeInfo)).close();
}
}
}

View File

@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.misc.store;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Rule;
import org.junit.rules.TestRule;
import java.io.IOException;
import java.util.EnumSet;
public class WindowsDirectoryTest extends LuceneTestCase {
@Rule
public static TestRule requiresNative = new NativeLibEnableRule(
EnumSet.of(NativeLibEnableRule.OperatingSystem.WINDOWS));
public void testLibraryLoaded() throws IOException {
try (Directory dir = new WindowsDirectory(RandomizedTest.newTempDir(LifecycleScope.TEST))) {
dir.createOutput("test", IOContext.DEFAULT).close();
}
}
}

View File

@ -32,7 +32,9 @@ def includeInBinaries = project(":lucene").subprojects.findAll {subproject ->
":lucene:packaging",
":lucene:documentation",
// Exclude parent container project of analysis modules (no artifacts).
":lucene:analysis"
":lucene:analysis",
// Exclude native module, which requires manual copying and enabling
":lucene:misc:native"
])
}

View File

@ -42,6 +42,7 @@ include "lucene:join"
include "lucene:luke"
include "lucene:memory"
include "lucene:misc"
include "lucene:misc:native"
include "lucene:monitor"
include "lucene:queries"
include "lucene:queryparser"
@ -73,4 +74,4 @@ include "solr:example"
include "solr:documentation"
include "solr:packaging"
include "solr:docker"
include "solr:docker:package"
include "solr:docker:package"