diff --git a/build.gradle b/build.gradle index dd5f63365ef..fedea545eea 100644 --- a/build.gradle +++ b/build.gradle @@ -158,9 +158,6 @@ apply from: file('gradle/generation/antlr.gradle') apply from: file('gradle/datasets/external-datasets.gradle') -// Shared configuration of subprojects containing native code. -apply from: file('gradle/native/disable-native.gradle') - // Additional development aids. apply from: file('gradle/testing/per-project-summary.gradle') apply from: file('gradle/testing/slowest-tests-at-end.gradle') diff --git a/gradle/native/disable-native.gradle b/gradle/native/disable-native.gradle deleted file mode 100644 index 3cc239dbacb..00000000000 --- a/gradle/native/disable-native.gradle +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.tools.ant.taskdefs.condition.Os - -// This is the master switch to disable all tasks that compile -// native (cpp) code. -rootProject.ext { - buildNative = propertyOrDefault("build.native", false).toBoolean() -} - -// Explicitly list all projects that should be configured for native extensions. -// We could scan for projects with a the cpp-library plugin but this is faster. -def nativeProjects = allprojects.findAll {it.path in [ - ":lucene:misc:native" -]} - -def javaProjectsWithNativeDeps = allprojects.findAll {it.path in [ - ":lucene:misc" -]} - -// Set up defaults for projects with native dependencies. -configure(javaProjectsWithNativeDeps, { - configurations { - nativeDeps { - attributes { - attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage.class, Usage.NATIVE_RUNTIME)) - attributes.attribute(CppBinary.OPTIMIZED_ATTRIBUTE, false) - } - } - } - - plugins.withType(JavaPlugin) { - ext { - testOptions += [ - [propName: 'tests.native', value: buildNative, description: "Enable tests that require native extensions."] - ] - - nativeDepsDir = file("${buildDir}/nativeDeps") - } - - // Only copy and attach native deps if native build is enabled. - if (buildNative && Os.isFamily(Os.FAMILY_WINDOWS)) { - task copyNativeDeps(type: Sync) { - from configurations.nativeDeps - into nativeDepsDir - } - - tasks.withType(Test) { - dependsOn copyNativeDeps - systemProperty "java.library.path", nativeDepsDir - } - } - } -}) - -// If native build is disabled we just disable all tasks in the active task set that -// originate from "native" projects. -// -// Perhaps there is a cleaner way to do it but removing their references from -// settings.gradle would remove them from IDE detection, dependency resolution, etc. -// This way seems better. -if (!buildNative) { - gradle.taskGraph.whenReady { taskGraph -> - def tasks = taskGraph.getAllTasks() - tasks.findAll { task -> task.project in nativeProjects }.each { task -> - task.enabled = false - } - } -} diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index d717c44bfa2..7b2e50dcbcf 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -90,6 +90,9 @@ API Changes * GITHUB#11742: MatchingFacetSetsCounts#getTopChildren now properly returns "top" children instead of all children. (Greg Miller) +* GITHUB#11772: Removed native subproject and WindowsDirectory implementation from lucene.misc. Recommendation: + use MMapDirectory implementation on Windows. (Robert Muir, Uwe Schindler, Dawid Weiss) + Bug Fixes --------------------- * GITHUB#11726: Indexing term vectors on large documents could fail due to diff --git a/lucene/misc/build.gradle b/lucene/misc/build.gradle index 1c4468a504b..a21dfeaafa7 100644 --- a/lucene/misc/build.gradle +++ b/lucene/misc/build.gradle @@ -22,6 +22,4 @@ description = 'Index tools and other miscellaneous code' dependencies { moduleApi project(':lucene:core') moduleTestImplementation project(':lucene:test-framework') - - nativeDeps project(":lucene:misc:native") } \ No newline at end of file diff --git a/lucene/misc/native/build.gradle b/lucene/misc/native/build.gradle deleted file mode 100644 index 664410ed903..00000000000 --- a/lucene/misc/native/build.gradle +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * This gets separated out from misc module into a native module due to incompatibility between cpp-library and java-library plugins. - * For details, please see https://github.com/gradle/gradle-native/issues/352#issuecomment-461724948 - */ -import org.apache.tools.ant.taskdefs.condition.Os - -description = 'Module for native code' - -apply plugin: 'cpp-library' - -library { - baseName = 'LuceneNativeIO' - - // Native build for Windows platform will be added in later stage - targetMachines = [ - machines.windows.x86_64 - ] - - // Point at platform-specific sources. Other platforms will be ignored - // (plugin won't find the toolchain). - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - source.from file("${projectDir}/src/main/windows") - } -} - -tasks.withType(CppCompile).configureEach { - def javaHome = rootProject.ext.runtimeJavaHome - - // Assume standard openjdk layout. This means only one architecture-specific include folder is present. - systemIncludes.from file("${javaHome}/include") - - for (def path : [file("${javaHome}/include/win32")]) { - if (path.exists()) { - systemIncludes.from path - } - } - - compilerArgs.add '-fPIC' -} - -tasks.withType(LinkSharedLibrary).configureEach { - linkerArgs.add '-lstdc++' -} diff --git a/lucene/misc/native/src/main/windows/WindowsDirectory.cpp b/lucene/misc/native/src/main/windows/WindowsDirectory.cpp deleted file mode 100644 index 20f79fe1954..00000000000 --- a/lucene/misc/native/src/main/windows/WindowsDirectory.cpp +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -#include -#include "windows.h" - -/** - * Windows Native IO methods. - */ -extern "C" { - -/** - * Utility to format a Windows system error code into an exception. - */ -void throwIOException(JNIEnv *env, DWORD error) -{ - jclass ioex; - char *msg; - - ioex = env->FindClass("java/io/IOException"); - - if (ioex != NULL) { - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR) &msg, 0, NULL ); - env->ThrowNew(ioex, msg); - LocalFree(msg); - } -} - -/** - * Utility to throw Exceptions on bad input - */ -void throwException(JNIEnv *env, const char *clazz, const char *msg) -{ - jclass exc = env->FindClass(clazz); - - if (exc != NULL) { - env->ThrowNew(exc, msg); - } -} - -/** - * Opens a handle to a file. - * - * Class: org_apache_lucene_misc_store_WindowsDirectory - * Method: open - * Signature: (Ljava/lang/String;)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_open - (JNIEnv *env, jclass ignored, jstring filename) -{ - char *fname; - HANDLE handle; - - if (filename == NULL) { - throwException(env, "java/lang/NullPointerException", "filename must not be null"); - return -1; - } - - fname = (char *) env->GetStringUTFChars(filename, NULL); - - if (fname == NULL) { - throwException(env, "java/lang/IllegalArgumentException", "invalid filename"); - return -1; - } - - handle = CreateFile(fname, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, - NULL, OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, NULL); - - env->ReleaseStringUTFChars(filename, fname); - - if (handle == INVALID_HANDLE_VALUE) { - throwIOException(env, GetLastError()); - return -1; - } - - return (jlong) handle; -} - -/** - * Reads data into the byte array, starting at offset, for length characters. - * The read is positioned at pos. - * - * Class: org_apache_lucene_misc_store_WindowsDirectory - * Method: read - * Signature: (J[BIIJ)I - */ -JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_read - (JNIEnv *env, jclass ignored, jlong fd, jbyteArray bytes, jint offset, jint length, jlong pos) -{ - OVERLAPPED io = { 0 }; - DWORD numRead = -1; - - io.Offset = (DWORD) (pos & 0xFFFFFFFF); - io.OffsetHigh = (DWORD) ((pos >> 0x20) & 0x7FFFFFFF); - - if (bytes == NULL) { - throwException(env, "java/lang/NullPointerException", "bytes must not be null"); - return -1; - } - - if (length <= 4096) { /* For small buffers, avoid GetByteArrayElements' copy */ - char buffer[length]; - - if (ReadFile((HANDLE) fd, &buffer, length, &numRead, &io)) { - env->SetByteArrayRegion(bytes, offset, numRead, (const jbyte *) buffer); - } else { - throwIOException(env, GetLastError()); - numRead = -1; - } - - } else { - jbyte *buffer = env->GetByteArrayElements (bytes, NULL); - - if (!ReadFile((HANDLE) fd, (void *)(buffer+offset), length, &numRead, &io)) { - throwIOException(env, GetLastError()); - numRead = -1; - } - - env->ReleaseByteArrayElements(bytes, buffer, numRead == 0 || numRead == -1 ? JNI_ABORT : 0); - } - - return numRead; -} - -/** - * Closes a handle to a file - * - * Class: org_apache_lucene_misc_store_WindowsDirectory - * Method: close - * Signature: (J)V - */ -JNIEXPORT void JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_close - (JNIEnv *env, jclass ignored, jlong fd) -{ - if (!CloseHandle((HANDLE) fd)) { - throwIOException(env, GetLastError()); - } -} - -/** - * Returns the length in bytes of a file. - * - * Class: org_apache_lucene_misc_store_WindowsDirectory - * Method: length - * Signature: (J)J - */ -JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_length - (JNIEnv *env, jclass ignored, jlong fd) -{ - BY_HANDLE_FILE_INFORMATION info; - - if (GetFileInformationByHandle((HANDLE) fd, (LPBY_HANDLE_FILE_INFORMATION) &info)) { - return (jlong) (((DWORDLONG) info.nFileSizeHigh << 0x20) + info.nFileSizeLow); - } else { - throwIOException(env, GetLastError()); - return -1; - } -} - -} /* extern "C" */ diff --git a/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java b/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java deleted file mode 100644 index c41401af3c9..00000000000 --- a/lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.misc.store; - -import java.io.EOFException; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Path; -import org.apache.lucene.store.BufferedIndexInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FSLockFactory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.LockFactory; - -/** - * Native {@link Directory} implementation for Microsoft Windows. - * - *

Steps: - * - *

    - *
  1. Compile the source code to create libLuceneNativeIO.dll: ./gradlew build - *
  2. Put the resulting libLuceneNativeIO.dll (from - * lucene/misc/native/build/lib/release/platform/) into some directory in your windows - * PATH - *
  3. Open indexes with WindowsDirectory and use it. - *
- * - * @lucene.experimental - */ -public class WindowsDirectory extends FSDirectory { - private static final int DEFAULT_BUFFERSIZE = 4096; /* default pgsize on ia32/amd64 */ - - static { - System.loadLibrary("LuceneNativeIO"); - } - - /** - * Create a new WindowsDirectory for the named location. - * - * @param path the path of the directory - * @param lockFactory the lock factory to use - * @throws IOException If there is a low-level I/O error - */ - public WindowsDirectory(Path path, LockFactory lockFactory) throws IOException { - super(path, lockFactory); - } - - /** - * Create a new WindowsDirectory for the named location and {@link FSLockFactory#getDefault()}. - * - * @param path the path of the directory - * @throws IOException If there is a low-level I/O error - */ - public WindowsDirectory(Path path) throws IOException { - this(path, FSLockFactory.getDefault()); - } - - @Override - public IndexInput openInput(String name, IOContext context) throws IOException { - ensureOpen(); - return new WindowsIndexInput( - getDirectory().resolve(name), - Math.max(BufferedIndexInput.bufferSize(context), DEFAULT_BUFFERSIZE)); - } - - static class WindowsIndexInput extends BufferedIndexInput { - private final long fd; - private final long length; - boolean isClone; - boolean isOpen; - - public WindowsIndexInput(Path file, int bufferSize) throws IOException { - super("WindowsIndexInput(path=\"" + file + "\")", bufferSize); - fd = WindowsDirectory.open(file.toString()); - length = WindowsDirectory.length(fd); - isOpen = true; - } - - @Override - protected void readInternal(ByteBuffer b) throws IOException { - int bytesRead; - try { - bytesRead = - WindowsDirectory.read(fd, b.array(), b.position(), b.remaining(), getFilePointer()); - } catch (IOException ioe) { - throw new IOException(ioe.getMessage() + ": " + this, ioe); - } - - if (bytesRead != length) { - throw new EOFException("read past EOF: " + this); - } - } - - @Override - protected void seekInternal(long pos) throws IOException {} - - @Override - public synchronized void close() throws IOException { - // NOTE: we synchronize and track "isOpen" because Lucene sometimes closes IIs twice! - if (!isClone && isOpen) { - WindowsDirectory.close(fd); - isOpen = false; - } - } - - @Override - public long length() { - return length; - } - - @Override - public WindowsIndexInput clone() { - WindowsIndexInput clone = (WindowsIndexInput) super.clone(); - clone.isClone = true; - return clone; - } - } - - /** Opens a handle to a file. */ - private static native long open(String filename) throws IOException; - - /** Reads data from a file at pos into bytes */ - private static native int read(long fd, byte[] bytes, int offset, int length, long pos) - throws IOException; - - /** Closes a handle to a file */ - private static native void close(long fd) throws IOException; - - /** Returns the length of a file */ - private static native long length(long fd) throws IOException; -} diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/NativeLibEnableRule.java b/lucene/misc/src/test/org/apache/lucene/misc/store/NativeLibEnableRule.java deleted file mode 100644 index 5586b4f5665..00000000000 --- a/lucene/misc/src/test/org/apache/lucene/misc/store/NativeLibEnableRule.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.misc.store; - -import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; -import java.util.Set; -import org.apache.lucene.util.Constants; -import org.junit.Assume; - -public class NativeLibEnableRule extends TestRuleAdapter { - enum OperatingSystem { - LINUX(Constants.LINUX), - WINDOWS(Constants.WINDOWS), - SUN_OS(Constants.SUN_OS), - MAC(Constants.MAC_OS_X), - FREE_BSD(Constants.FREE_BSD); - - public final boolean enabled; - - OperatingSystem(boolean enabled) { - this.enabled = enabled; - } - } - - private final Set runOn; - - public NativeLibEnableRule(Set runOn) { - this.runOn = runOn; - } - - @Override - protected void before() { - Assume.assumeTrue( - "Test ignored (tests.native is false)", - Boolean.parseBoolean(System.getProperty("tests.native", "false"))); - - Assume.assumeTrue( - "Test ignored, only applies to architectures: " + runOn, - runOn.stream().anyMatch(os -> os.enabled)); - } -} diff --git a/lucene/misc/src/test/org/apache/lucene/misc/store/TestWindowsDirectory.java b/lucene/misc/src/test/org/apache/lucene/misc/store/TestWindowsDirectory.java deleted file mode 100644 index c4326469f08..00000000000 --- a/lucene/misc/src/test/org/apache/lucene/misc/store/TestWindowsDirectory.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.misc.store; - -import com.carrotsearch.randomizedtesting.LifecycleScope; -import com.carrotsearch.randomizedtesting.RandomizedTest; -import java.io.IOException; -import java.util.EnumSet; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.tests.util.LuceneTestCase; -import org.junit.Rule; -import org.junit.rules.TestRule; - -public class TestWindowsDirectory extends LuceneTestCase { - @Rule - public static TestRule requiresNative = - new NativeLibEnableRule(EnumSet.of(NativeLibEnableRule.OperatingSystem.WINDOWS)); - - public void testLibraryLoaded() throws IOException { - try (Directory dir = new WindowsDirectory(RandomizedTest.newTempDir(LifecycleScope.TEST))) { - dir.createOutput("test", IOContext.DEFAULT).close(); - } - } -} diff --git a/settings.gradle b/settings.gradle index 971fbb6c5bc..4d22af5fa2e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -55,7 +55,6 @@ include "lucene:join" include "lucene:luke" include "lucene:memory" include "lucene:misc" -include "lucene:misc:native" include "lucene:monitor" include "lucene:queries" include "lucene:queryparser"