Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

An eye-gouging way to limit suppressAccessChecks to just the three JARs that need them. (#13164)

This commit is contained in:
Dawid Weiss 2024-03-08 08:10:49 +01:00 committed by GitHub
parent 49db29e59f
commit 1c77e2315c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 72 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
gradle/testing/randomization.gradle
View File

@ -119,6 +119,18 @@ allprojects {
// Resolve test option values after all evaluation is complete.
allprojects {
plugins.withType(JavaPlugin) {
configurations {
secManagerExclusions
}
dependencies {
secManagerExclusions ( "com.carrotsearch.randomizedtesting:randomizedtesting-runner", {
exclude group: "junit"
})
secManagerExclusions ( "junit:junit", {
exclude group: "org.hamcrest"
})
}
afterEvaluate {
ext.testOptionsResolved = testOptions.findAll { opt ->
propertyOrDefault(opt.propName, opt.value) != null
@ -155,7 +167,7 @@ allprojects {
}
// Append resolved test properties to the test task.
tasks.withType(Test) { task ->
tasks.withType(Test) { Test task ->
// TODO: we could remove some options that are only relevant to the build environment
// and not the test JVM itself.
systemProperties testOptionsResolved
@ -172,13 +184,13 @@ allprojects {
// Enable security manager, if requested. We could move the selection of security manager and security policy
// to each project's build/ configuration but it seems compact enough to keep it here for now.
def securityArgumentProvider = new SecurityArgumentProvider(commonDir: project(":lucene").layout.projectDirectory)
def securityArgumentProvider = new SecurityArgumentProvider()
securityArgumentProvider.commonDir = project(":lucene").layout.projectDirectory
securityArgumentProvider.otherProperties = project.objects.mapProperty(String, String)
if (Boolean.parseBoolean(testOptionsResolved["tests.useSecurityManager"])) {
if (project.path.endsWith(".tests")) {
// LUCENE-10301: for now, do not use the security manager for modular tests (test framework is not available).
} else if (project.path == ":lucene:replicator") {
systemProperty 'java.security.manager', "org.apache.lucene.tests.util.TestSecurityManager"
securityArgumentProvider.javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/replicator-tests.policy")
} else if (project.path.startsWith(":lucene")) {
systemProperty 'java.security.manager', "org.apache.lucene.tests.util.TestSecurityManager"
securityArgumentProvider.javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/tests.policy")
@ -189,6 +201,14 @@ allprojects {
systemProperty 'gradle.lib.dir', Paths.get(project.class.location.toURI()).parent.toAbsolutePath().toString().replace('\\', '/')
systemProperty 'gradle.worker.jar', Paths.get("${gradleUserHome}/caches/${gradle.gradleVersion}/workerMain/gradle-worker.jar").toAbsolutePath().toString()
systemProperty 'gradle.user.home', gradleUserHome.toPath().toAbsolutePath().toString()
securityArgumentProvider.otherProperties.put("randomizedtesting.jar", project.provider { ->
return configurations.secManagerExclusions.resolve().find { it.name.startsWith("randomizedtesting-runner-") }.absolutePath
})
securityArgumentProvider.otherProperties.put("junit.jar", project.provider { ->
return configurations.secManagerExclusions.resolve().find { it.name.startsWith("junit-") }.absolutePath
})
securityArgumentProvider.otherProperties.put("lucene.test.framework", project(":lucene:test-framework").layout.buildDirectory.get().asFile.absolutePath)
}
doFirst {
@ -242,12 +262,20 @@ class SecurityArgumentProvider implements CommandLineArgumentProvider {
@PathSensitive(PathSensitivity.RELATIVE)
RegularFile javaSecurityPolicy
@Input
MapProperty<String, String> otherProperties
@Override
Iterable<String> asArguments() {
def args = ["-Dcommon.dir=${commonDir.getAsFile()}"]
if (javaSecurityPolicy) {
args.add("-Djava.security.policy=${javaSecurityPolicy.getAsFile()}")
}
otherProperties.getOrElse(Map.of()).forEach { key, value ->
args.add("-D${key}=${value}")
}
return args
}
}

94
gradle/testing/randomization/policies/replicator-tests.policy
View File

@ -1,94 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Policy file for :lucene:replicator tests. Please keep minimal and avoid wildcards.
// this differs from the standard lucene policy in that it must allow read-write access
// to all system properties, because of jetty calls to System.getProperties()
grant {
// jetty-specific:
permission java.lang.RuntimePermission "getenv.JETTY_AVAILABLE_PROCESSORS";
permission java.lang.RuntimePermission "getenv.JETTY_WORKER_INSTANCE";
// allow TestNRTReplication fork its jvm
permission java.io.FilePermission "${java.home}${/}-", "read,execute";
// read/write access to all system properties (required by jetty in these tests)
permission java.util.PropertyPermission "*", "read,write";
// all possibilities of accepting/binding/connecting on localhost with ports >= 1024:
permission java.net.SocketPermission "localhost:1024-", "accept,connect,resolve";
permission java.net.SocketPermission "127.0.0.1:1024-", "accept,connect,resolve";
permission java.net.SocketPermission "[::1]:1024-", "accept,connect,resolve";
// test-files/ resources
permission java.io.FilePermission "${common.dir}${/}-", "read";
// write only to sandbox
permission java.io.FilePermission "${tests.linedocsfile}", "read";
permission java.io.FilePermission "${java.io.tmpdir}", "read,write";
permission java.io.FilePermission "${java.io.tmpdir}${/}-", "read,write,delete";
// needed by randomizedtesting runner to identify test methods.
permission java.lang.RuntimePermission "accessDeclaredMembers";
// needed by certain tests to redirect sysout/syserr:
permission java.lang.RuntimePermission "setIO";
// needed by randomized runner to catch failures from other threads:
permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler";
// needed by randomized runner getTopThreadGroup:
permission java.lang.RuntimePermission "modifyThreadGroup";
// needed by tests e.g. shutting down executors:
permission java.lang.RuntimePermission "modifyThread";
// needed for tons of test hacks etc
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
// test runner seems to require supression of access checks:
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};
// Permissions to support ant build
grant {
permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read";
permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";
permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";
};
// Permissions for jacoco code coverage
grant {
// permission to write the per-jvm code coverage output
permission java.io.FilePermission "${user.dir}${/}jacoco.exec", "write";
// needed by jacoco to dump coverage on shutdown
permission java.lang.RuntimePermission "shutdownHooks";
// needed by jacoco to instrument classes
permission java.lang.RuntimePermission "defineClass";
// needed by jacoco for God knows what.
permission java.lang.RuntimePermission "createClassLoader";
};
// Grant all permissions to Gradle test runner classes.
grant codeBase "file:${gradle.lib.dir}${/}-" {
permission java.security.AllPermission;
};
grant codeBase "file:${gradle.worker.jar}" {
permission java.security.AllPermission;
};
grant {
// Allow reading gradle worker JAR.
permission java.io.FilePermission "${gradle.worker.jar}", "read";
// Allow reading from classpath JARs (resources).
permission java.io.FilePermission "${gradle.user.home}${/}-", "read";
};

33
gradle/testing/randomization/policies/tests.policy
View File

@ -33,34 +33,29 @@ grant {
// misc HardlinkCopyDirectoryWrapper needs this to test if hardlinks can be created
permission java.nio.file.LinkPermission "hard";
// needed by SSD detection tests in TestIOUtils (creates symlinks)
permission java.nio.file.LinkPermission "symbolic";
// needed by randomizedtesting runner to identify test methods.
// needed by RamUsageEstimator, and randomizedtesting runner to identify test methods.
permission java.lang.RuntimePermission "accessDeclaredMembers";
// needed by certain tests to capture sysout/syserr:
permission java.lang.RuntimePermission "setIO";
// needed by randomized runner to catch failures from other threads:
// needed by luke and randomized runner to catch failures from other threads:
permission java.lang.RuntimePermission "setDefaultUncaughtExceptionHandler";
// needed by randomized runner getTopThreadGroup:
permission java.lang.RuntimePermission "modifyThreadGroup";
// needed by tests e.g. shutting down executors:
permission java.lang.RuntimePermission "modifyThread";
// needed for tons of test hacks etc
permission java.lang.RuntimePermission "getStackTrace";
// needed for mock filesystems in tests
permission java.lang.RuntimePermission "fileSystemProvider";
// test runner seems to require supression of access checks:
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
// needed by cyberneko usage by benchmarks on J9
permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util";
// Needed for loading native library (lucene:misc:native) in lucene:misc
// Needed for DirectIODirectory to retrieve block size
permission java.lang.RuntimePermission "getFileStoreAttributes";
permission java.lang.RuntimePermission "writeFileDescriptor";
// TestLockFactoriesMultiJVM opens a random port on 127.0.0.1 (port 0 = ephemeral port range):
permission java.net.SocketPermission "127.0.0.1:0", "accept,listen,resolve";
// Replicator tests connect to ephemeral ports
permission java.net.SocketPermission "127.0.0.1:1024-", "connect,resolve";
// read access to all system properties:
permission java.util.PropertyPermission "*", "read";
@ -87,13 +82,6 @@ grant {
permission java.io.FilePermission "${hunspell.dictionaries}${/}-", "read";
};
// Permissions to support ant build
grant {
permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read";
permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";
permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";
};
// Permissions for jacoco code coverage
grant {
// permission to write the per-jvm code coverage output
@ -122,3 +110,14 @@ grant {
permission java.io.FilePermission "${gradle.user.home}${/}-", "read";
};
// Grant permissions to certain test-related JARs (https://github.com/apache/lucene/pull/13146)
grant codeBase "file:${randomizedtesting.jar}" {
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
permission java.lang.RuntimePermission "modifyThreadGroup";
};
grant codeBase "file:${junit.jar}" {
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};
grant codeBase "file:${lucene.test.framework}${/}-" {
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};

84
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
View File

@ -22,8 +22,6 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.lang.ProcessBuilder.Redirect;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
@ -31,10 +29,10 @@ import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.tests.store.BaseDirectoryWrapper;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.SuppressForbidden;
/**
@ -57,7 +55,8 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {
if (System.getProperty("tests.crashmode") == null) {
// try up to 10 times to create an index
for (int i = 0; i < 10; i++) {
forkTest();
final int crashDelayMs = TestUtil.nextInt(random(), 3000, 4000);
forkTest(crashDelayMs);
// if we succeeded in finding an index, we are done.
if (checkIndexes(tempDir)) return;
}
@ -68,34 +67,17 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {
// assumeFalse("does not support PreFlex, see LUCENE-3992",
// Codec.getDefault().getName().equals("Lucene4x"));
// we are the fork, setup a crashing thread
final int crashTime = TestUtil.nextInt(random(), 3000, 4000);
Thread t =
new Thread() {
@SuppressForbidden(reason = "Thread sleep")
@Override
public void run() {
try {
Thread.sleep(crashTime);
} catch (
@SuppressWarnings("unused")
InterruptedException e) {
}
crashJRE();
}
};
t.setPriority(Thread.MAX_PRIORITY);
t.start();
// run the test until we crash.
for (int i = 0; i < 1000; i++) {
// Effectively an endless loop until the process is killed from the outside.
for (int i = 0; i < Integer.MAX_VALUE; i++) {
super.testNRTThreads();
}
}
}
/** fork ourselves in a new jvm. sets -Dtests.crashmode=true */
@SuppressForbidden(reason = "ProcessBuilder requires java.io.File for CWD")
public void forkTest() throws Exception {
@SuppressForbidden(
reason = "ProcessBuilder requires java.io.File for setting ProcessBuilder.directory")
public void forkTest(int crashDelayMs) throws Exception {
List<String> cmd = new ArrayList<>();
cmd.add(Paths.get(System.getProperty("java.home"), "bin", "java").toString());
cmd.add("-Xmx512m");
@ -114,12 +96,21 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {
.redirectInput(Redirect.INHERIT)
.redirectErrorStream(true);
Process p = pb.start();
// We pump everything to stderr.
PrintStream childOut = System.err;
Thread stdoutPumper = ThreadPumper.start(p.getInputStream(), childOut);
if (VERBOSE) childOut.println(">>> Begin subprocess output");
p.waitFor();
try {
if (VERBOSE) childOut.println(">>> Begin subprocess output");
if (p.waitFor(crashDelayMs, TimeUnit.MILLISECONDS)) {
// This means the process has exited before the timeout. This is odd because it should
// run in an endless loop?
throw new AssertionError("Subprocess has exited unexpectedly.");
}
} finally {
// Try to kill the process and wait until it terminates. destroyForcibly is a no-op in case
// the process is not alive, so no need for additional checks.
p.destroyForcibly().waitFor();
}
stdoutPumper.join();
if (VERBOSE) childOut.println("<<< End subprocess output");
}
@ -140,7 +131,7 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {
}
}
} catch (IOException e) {
System.err.println("Couldn't pipe from the forked process: " + e.toString());
System.err.println("Couldn't pipe from the forked process: " + e);
}
}
};
@ -186,37 +177,4 @@ public class TestIndexWriterOnJRECrash extends TestNRTThreads {
});
return found.get();
}
/** currently, this only works/tested on Sun and IBM. */
@SuppressForbidden(reason = "We need Unsafe to actually crush :-)")
public void crashJRE() {
final String vendor = Constants.JAVA_VENDOR;
final boolean supportsUnsafeNpeDereference =
vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple");
try {
if (supportsUnsafeNpeDereference) {
try {
Class<?> clazz = Class.forName("sun.misc.Unsafe");
Field field = clazz.getDeclaredField("theUnsafe");
field.setAccessible(true);
Object o = field.get(null);
Method m = clazz.getMethod("putAddress", long.class, long.class);
m.invoke(o, 0L, 0L);
} catch (Throwable e) {
System.out.println("Couldn't kill the JVM via Unsafe.");
e.printStackTrace(System.out);
}
}
// Fallback attempt to Runtime.halt();
Runtime.getRuntime().halt(-1);
} catch (Exception e) {
System.out.println("Couldn't kill the JVM.");
e.printStackTrace(System.out);
}
// We couldn't get the JVM to crash for some reason.
fail();
}
}

34
lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java
View File

@ -21,7 +21,6 @@ import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Method;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
@ -45,7 +44,6 @@ import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks;
import org.apache.lucene.tests.util.TestRuleIgnoreTestSuites;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.SuppressForbidden;
import org.junit.Assume;
@ -58,7 +56,6 @@ import org.junit.BeforeClass;
*/
@SuppressCodecs({"MockRandom", "Direct", "SimpleText"})
@SuppressSysoutChecks(bugUrl = "Stuff gets printed, important stuff for debugging a failure")
@SuppressForbidden(reason = "We need Unsafe to actually crush :-)")
public class TestSimpleServer extends LuceneTestCase {
static final Set<Thread> clientThreads = Collections.synchronizedSet(new HashSet<>());
@ -135,39 +132,12 @@ public class TestSimpleServer extends LuceneTestCase {
}
}
/** currently, this only works/tested on Sun and IBM. */
/** forcibly halt the JVM: similar to crashing */
// poached from TestIndexWriterOnJRECrash ... should we factor out to TestUtil? seems dangerous
// to give it such "publicity"?
private static void crashJRE() {
final String vendor = Constants.JAVA_VENDOR;
final boolean supportsUnsafeNpeDereference =
vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple");
try {
if (supportsUnsafeNpeDereference) {
try {
Class<?> clazz = Class.forName("sun.misc.Unsafe");
java.lang.reflect.Field field = clazz.getDeclaredField("theUnsafe");
field.setAccessible(true);
Object o = field.get(null);
Method m = clazz.getMethod("putAddress", long.class, long.class);
m.invoke(o, 0L, 0L);
} catch (Throwable e) {
System.out.println("Couldn't kill the JVM via Unsafe.");
e.printStackTrace(System.out);
}
}
// Fallback attempt to Runtime.halt();
Runtime.getRuntime().halt(-1);
} catch (Exception e) {
System.out.println("Couldn't kill the JVM.");
e.printStackTrace(System.out);
}
// We couldn't get the JVM to crash for some reason.
throw new RuntimeException("JVM refuses to die!");
Runtime.getRuntime().halt(1);
}
static void writeFilesMetaData(DataOutput out, Map<String, FileMetaData> files)