Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-11172: remove WindowsDirectory and native subproject. (#11774)

This commit is contained in:
Dawid Weiss 2022-09-15 16:22:46 +02:00 committed by GitHub
parent 0587844742
commit 9acc653995
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 3 additions and 566 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build.gradle
View File

@ -158,9 +158,6 @@ apply from: file('gradle/generation/antlr.gradle')
apply from: file('gradle/datasets/external-datasets.gradle')
// Shared configuration of subprojects containing native code.
apply from: file('gradle/native/disable-native.gradle')
// Additional development aids.
apply from: file('gradle/testing/per-project-summary.gradle')
apply from: file('gradle/testing/slowest-tests-at-end.gradle')

84
gradle/native/disable-native.gradle
View File

@ -1,84 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.tools.ant.taskdefs.condition.Os
// This is the master switch to disable all tasks that compile
// native (cpp) code.
rootProject.ext {
buildNative = propertyOrDefault("build.native", false).toBoolean()
}
// Explicitly list all projects that should be configured for native extensions.
// We could scan for projects with a the cpp-library plugin but this is faster.
def nativeProjects = allprojects.findAll {it.path in [
":lucene:misc:native"
]}
def javaProjectsWithNativeDeps = allprojects.findAll {it.path in [
":lucene:misc"
]}
// Set up defaults for projects with native dependencies.
configure(javaProjectsWithNativeDeps, {
configurations {
nativeDeps {
attributes {
attributes.attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage.class, Usage.NATIVE_RUNTIME))
attributes.attribute(CppBinary.OPTIMIZED_ATTRIBUTE, false)
}
}
}
plugins.withType(JavaPlugin) {
ext {
testOptions += [
[propName: 'tests.native', value: buildNative, description: "Enable tests that require native extensions."]
]
nativeDepsDir = file("${buildDir}/nativeDeps")
}
// Only copy and attach native deps if native build is enabled.
if (buildNative && Os.isFamily(Os.FAMILY_WINDOWS)) {
task copyNativeDeps(type: Sync) {
from configurations.nativeDeps
into nativeDepsDir
}
tasks.withType(Test) {
dependsOn copyNativeDeps
systemProperty "java.library.path", nativeDepsDir
}
}
}
})
// If native build is disabled we just disable all tasks in the active task set that
// originate from "native" projects.
//
// Perhaps there is a cleaner way to do it but removing their references from
// settings.gradle would remove them from IDE detection, dependency resolution, etc.
// This way seems better.
if (!buildNative) {
gradle.taskGraph.whenReady { taskGraph ->
def tasks = taskGraph.getAllTasks()
tasks.findAll { task -> task.project in nativeProjects }.each { task ->
task.enabled = false
}
}
}

3
lucene/CHANGES.txt
View File

@ -90,6 +90,9 @@ API Changes
* GITHUB#11742: MatchingFacetSetsCounts#getTopChildren now properly returns "top" children instead
of all children. (Greg Miller)
* GITHUB#11772: Removed native subproject and WindowsDirectory implementation from lucene.misc. Recommendation:
use MMapDirectory implementation on Windows. (Robert Muir, Uwe Schindler, Dawid Weiss)
Bug Fixes
---------------------
* GITHUB#11726: Indexing term vectors on large documents could fail due to

2
lucene/misc/build.gradle
View File

@ -22,6 +22,4 @@ description = 'Index tools and other miscellaneous code'
dependencies {
moduleApi project(':lucene:core')
moduleTestImplementation project(':lucene:test-framework')
nativeDeps project(":lucene:misc:native")
}

60
lucene/misc/native/build.gradle
View File

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* This gets separated out from misc module into a native module due to incompatibility between cpp-library and java-library plugins.
* For details, please see https://github.com/gradle/gradle-native/issues/352#issuecomment-461724948
*/
import org.apache.tools.ant.taskdefs.condition.Os
description = 'Module for native code'
apply plugin: 'cpp-library'
library {
baseName = 'LuceneNativeIO'
// Native build for Windows platform will be added in later stage
targetMachines = [
machines.windows.x86_64
]
// Point at platform-specific sources. Other platforms will be ignored
// (plugin won't find the toolchain).
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
source.from file("${projectDir}/src/main/windows")
}
}
tasks.withType(CppCompile).configureEach {
def javaHome = rootProject.ext.runtimeJavaHome
// Assume standard openjdk layout. This means only one architecture-specific include folder is present.
systemIncludes.from file("${javaHome}/include")
for (def path : [file("${javaHome}/include/win32")]) {
if (path.exists()) {
systemIncludes.from path
}
}
compilerArgs.add '-fPIC'
}
tasks.withType(LinkSharedLibrary).configureEach {
linkerArgs.add '-lstdc++'
}

175
lucene/misc/native/src/main/windows/WindowsDirectory.cpp
View File

@ -1,175 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
#include <jni.h>
#include "windows.h"
/**
* Windows Native IO methods.
*/
extern "C" {
/**
* Utility to format a Windows system error code into an exception.
*/
void throwIOException(JNIEnv *env, DWORD error)
{
jclass ioex;
char *msg;
ioex = env->FindClass("java/io/IOException");
if (ioex != NULL) {
FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR) &msg, 0, NULL );
env->ThrowNew(ioex, msg);
LocalFree(msg);
}
}
/**
* Utility to throw Exceptions on bad input
*/
void throwException(JNIEnv *env, const char *clazz, const char *msg)
{
jclass exc = env->FindClass(clazz);
if (exc != NULL) {
env->ThrowNew(exc, msg);
}
}
/**
* Opens a handle to a file.
*
* Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: open
* Signature: (Ljava/lang/String;)J
*/
JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_open
(JNIEnv *env, jclass ignored, jstring filename)
{
char *fname;
HANDLE handle;
if (filename == NULL) {
throwException(env, "java/lang/NullPointerException", "filename must not be null");
return -1;
}
fname = (char *) env->GetStringUTFChars(filename, NULL);
if (fname == NULL) {
throwException(env, "java/lang/IllegalArgumentException", "invalid filename");
return -1;
}
handle = CreateFile(fname, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE,
NULL, OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, NULL);
env->ReleaseStringUTFChars(filename, fname);
if (handle == INVALID_HANDLE_VALUE) {
throwIOException(env, GetLastError());
return -1;
}
return (jlong) handle;
}
/**
* Reads data into the byte array, starting at offset, for length characters.
* The read is positioned at pos.
*
* Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: read
* Signature: (J[BIIJ)I
*/
JNIEXPORT jint JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_read
(JNIEnv *env, jclass ignored, jlong fd, jbyteArray bytes, jint offset, jint length, jlong pos)
{
OVERLAPPED io = { 0 };
DWORD numRead = -1;
io.Offset = (DWORD) (pos & 0xFFFFFFFF);
io.OffsetHigh = (DWORD) ((pos >> 0x20) & 0x7FFFFFFF);
if (bytes == NULL) {
throwException(env, "java/lang/NullPointerException", "bytes must not be null");
return -1;
}
if (length <= 4096) { /* For small buffers, avoid GetByteArrayElements' copy */
char buffer[length];
if (ReadFile((HANDLE) fd, &buffer, length, &numRead, &io)) {
env->SetByteArrayRegion(bytes, offset, numRead, (const jbyte *) buffer);
} else {
throwIOException(env, GetLastError());
numRead = -1;
}
} else {
jbyte *buffer = env->GetByteArrayElements (bytes, NULL);
if (!ReadFile((HANDLE) fd, (void *)(buffer+offset), length, &numRead, &io)) {
throwIOException(env, GetLastError());
numRead = -1;
}
env->ReleaseByteArrayElements(bytes, buffer, numRead == 0 || numRead == -1 ? JNI_ABORT : 0);
}
return numRead;
}
/**
* Closes a handle to a file
*
* Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: close
* Signature: (J)V
*/
JNIEXPORT void JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_close
(JNIEnv *env, jclass ignored, jlong fd)
{
if (!CloseHandle((HANDLE) fd)) {
throwIOException(env, GetLastError());
}
}
/**
* Returns the length in bytes of a file.
*
* Class: org_apache_lucene_misc_store_WindowsDirectory
* Method: length
* Signature: (J)J
*/
JNIEXPORT jlong JNICALL Java_org_apache_lucene_misc_store_WindowsDirectory_length
(JNIEnv *env, jclass ignored, jlong fd)
{
BY_HANDLE_FILE_INFORMATION info;
if (GetFileInformationByHandle((HANDLE) fd, (LPBY_HANDLE_FILE_INFORMATION) &info)) {
return (jlong) (((DWORDLONG) info.nFileSizeHigh << 0x20) + info.nFileSizeLow);
} else {
throwIOException(env, GetLastError());
return -1;
}
}
} /* extern "C" */

147
lucene/misc/src/java/org/apache/lucene/misc/store/WindowsDirectory.java
View File

@ -1,147 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.misc.store;
import java.io.EOFException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FSLockFactory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.LockFactory;
/**
* Native {@link Directory} implementation for Microsoft Windows.
*
* <p>Steps:
*
* <ol>
* <li>Compile the source code to create libLuceneNativeIO.dll: <code>./gradlew build</code>
* <li>Put the resulting <code>libLuceneNativeIO.dll</code> (from <code>
* lucene/misc/native/build/lib/release/platform/</code>) into some directory in your windows
* PATH
* <li>Open indexes with WindowsDirectory and use it.
* </ol>
*
* @lucene.experimental
*/
public class WindowsDirectory extends FSDirectory {
private static final int DEFAULT_BUFFERSIZE = 4096; /* default pgsize on ia32/amd64 */
static {
System.loadLibrary("LuceneNativeIO");
}
/**
* Create a new WindowsDirectory for the named location.
*
* @param path the path of the directory
* @param lockFactory the lock factory to use
* @throws IOException If there is a low-level I/O error
*/
public WindowsDirectory(Path path, LockFactory lockFactory) throws IOException {
super(path, lockFactory);
}
/**
* Create a new WindowsDirectory for the named location and {@link FSLockFactory#getDefault()}.
*
* @param path the path of the directory
* @throws IOException If there is a low-level I/O error
*/
public WindowsDirectory(Path path) throws IOException {
this(path, FSLockFactory.getDefault());
}
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
return new WindowsIndexInput(
getDirectory().resolve(name),
Math.max(BufferedIndexInput.bufferSize(context), DEFAULT_BUFFERSIZE));
}
static class WindowsIndexInput extends BufferedIndexInput {
private final long fd;
private final long length;
boolean isClone;
boolean isOpen;
public WindowsIndexInput(Path file, int bufferSize) throws IOException {
super("WindowsIndexInput(path=\"" + file + "\")", bufferSize);
fd = WindowsDirectory.open(file.toString());
length = WindowsDirectory.length(fd);
isOpen = true;
}
@Override
protected void readInternal(ByteBuffer b) throws IOException {
int bytesRead;
try {
bytesRead =
WindowsDirectory.read(fd, b.array(), b.position(), b.remaining(), getFilePointer());
} catch (IOException ioe) {
throw new IOException(ioe.getMessage() + ": " + this, ioe);
}
if (bytesRead != length) {
throw new EOFException("read past EOF: " + this);
}
}
@Override
protected void seekInternal(long pos) throws IOException {}
@Override
public synchronized void close() throws IOException {
// NOTE: we synchronize and track "isOpen" because Lucene sometimes closes IIs twice!
if (!isClone && isOpen) {
WindowsDirectory.close(fd);
isOpen = false;
}
}
@Override
public long length() {
return length;
}
@Override
public WindowsIndexInput clone() {
WindowsIndexInput clone = (WindowsIndexInput) super.clone();
clone.isClone = true;
return clone;
}
}
/** Opens a handle to a file. */
private static native long open(String filename) throws IOException;
/** Reads data from a file at pos into bytes */
private static native int read(long fd, byte[] bytes, int offset, int length, long pos)
throws IOException;
/** Closes a handle to a file */
private static native void close(long fd) throws IOException;
/** Returns the length of a file */
private static native long length(long fd) throws IOException;
}

55
lucene/misc/src/test/org/apache/lucene/misc/store/NativeLibEnableRule.java
View File

@ -1,55 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.misc.store;
import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
import java.util.Set;
import org.apache.lucene.util.Constants;
import org.junit.Assume;
public class NativeLibEnableRule extends TestRuleAdapter {
enum OperatingSystem {
LINUX(Constants.LINUX),
WINDOWS(Constants.WINDOWS),
SUN_OS(Constants.SUN_OS),
MAC(Constants.MAC_OS_X),
FREE_BSD(Constants.FREE_BSD);
public final boolean enabled;
OperatingSystem(boolean enabled) {
this.enabled = enabled;
}
}
private final Set<OperatingSystem> runOn;
public NativeLibEnableRule(Set<OperatingSystem> runOn) {
this.runOn = runOn;
}
@Override
protected void before() {
Assume.assumeTrue(
"Test ignored (tests.native is false)",
Boolean.parseBoolean(System.getProperty("tests.native", "false")));
Assume.assumeTrue(
"Test ignored, only applies to architectures: " + runOn,
runOn.stream().anyMatch(os -> os.enabled));
}
}

39
lucene/misc/src/test/org/apache/lucene/misc/store/TestWindowsDirectory.java
View File

@ -1,39 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.misc.store;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import java.io.IOException;
import java.util.EnumSet;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.junit.Rule;
import org.junit.rules.TestRule;
public class TestWindowsDirectory extends LuceneTestCase {
@Rule
public static TestRule requiresNative =
new NativeLibEnableRule(EnumSet.of(NativeLibEnableRule.OperatingSystem.WINDOWS));
public void testLibraryLoaded() throws IOException {
try (Directory dir = new WindowsDirectory(RandomizedTest.newTempDir(LifecycleScope.TEST))) {
dir.createOutput("test", IOContext.DEFAULT).close();
}
}
}

1
settings.gradle
View File

@ -55,7 +55,6 @@ include "lucene:join"
include "lucene:luke"
include "lucene:memory"
include "lucene:misc"
include "lucene:misc:native"
include "lucene:monitor"
include "lucene:queries"
include "lucene:queryparser"