Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-9512: Move LockFactory stress test to be a unit/integration test (#1842) 

Co-authored-by: Dawid Weiss <dawid.weiss@carrotsearch.com>
This commit is contained in:
Uwe Schindler 2020-09-09 18:42:30 +02:00 committed by GitHub
parent 2e4fc14e62
commit f0d3bab321
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 164 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
gradle/testing/randomization/policies/tests.policy
View File

@ -64,6 +64,9 @@ grant {
permission java.lang.RuntimePermission "getClassLoader";
permission java.lang.RuntimePermission "setContextClassLoader";
// TestLockFactoriesMultiJVM opens a random port on 127.0.0.1 (port 0 = ephemeral port range):
permission java.net.SocketPermission "127.0.0.1:0", "accept,listen,resolve";
// read access to all system properties:
permission java.util.PropertyPermission "*", "read";
// write access to only these:

4
lucene/CHANGES.txt
View File

@ -252,6 +252,10 @@ Other
* LUCENE-9470: Make TestXYMultiPolygonShapeQueries more resilient for CONTAINS queries. (Ignacio Vera)
* LUCENE-9512: Move LockFactory stress test to be a unit/integration
test. (Uwe Schindler, Dawid Weiss, Robert Muir)
======================= Lucene 8.6.2 =======================
Bug Fixes

29
lucene/core/src/java/org/apache/lucene/store/LockStressTest.java
View File

@ -30,7 +30,7 @@ import org.apache.lucene.util.SuppressForbidden;
/**
* Simple standalone tool that forever acquires and releases a
* lock using a specific LockFactory. Run without any args
* lock using a specific {@link LockFactory}. Run without any args
* to see usage.
*
* @see VerifyingLockFactory
@ -38,11 +38,9 @@ import org.apache.lucene.util.SuppressForbidden;
*/
public class LockStressTest {
static final String LOCK_FILE_NAME = "test.lock";
@SuppressForbidden(reason = "System.out required: command line tool")
@SuppressWarnings("try")
public static void main(String[] args) throws Exception {
if (args.length != 7) {
System.out.println("Usage: java org.apache.lucene.store.LockStressTest myID verifierHost verifierPort lockFactoryClassName lockDirName sleepTimeMS count\n" +
@ -65,12 +63,6 @@ public class LockStressTest {
int arg = 0;
final int myID = Integer.parseInt(args[arg++]);
if (myID < 0 || myID > 255) {
System.out.println("myID must be a unique int 0..255");
System.exit(1);
}
final String verifierHost = args[arg++];
final int verifierPort = Integer.parseInt(args[arg++]);
final String lockFactoryClassName = args[arg++];
@ -78,8 +70,21 @@ public class LockStressTest {
final int sleepTimeMS = Integer.parseInt(args[arg++]);
final int count = Integer.parseInt(args[arg++]);
int exitCode = run(myID, verifierHost, verifierPort, lockFactoryClassName, lockDirPath, sleepTimeMS, count);
System.exit(exitCode);
}
@SuppressForbidden(reason = "System.out required: command line tool")
@SuppressWarnings("try")
private static int run(int myID, String verifierHost, int verifierPort, String lockFactoryClassName,
Path lockDirPath, int sleepTimeMS, int count) throws IOException, InterruptedException {
if (myID < 0 || myID > 255) {
System.out.println("myID must be a unique int 0..255");
return 1;
}
final LockFactory lockFactory = getNewLockFactory(lockFactoryClassName);
// we test the lock factory directly, so we don't need it on the directory itsself (the directory is just for testing)
// we test the lock factory directly, so we don't need it on the directory itself (the directory is just for testing)
final FSDirectory lockDir = new NIOFSDirectory(lockDirPath, NoLockFactory.INSTANCE);
final InetSocketAddress addr = new InetSocketAddress(verifierHost, verifierPort);
System.out.println("Connecting to server " + addr +
@ -96,7 +101,7 @@ public class LockStressTest {
final Random rnd = new Random();
// wait for starting gun
if (in.read() != 43) {
if (in.read() != LockVerifyServer.START_GUN_SIGNAL) {
throw new IOException("Protocol violation");
}
@ -126,9 +131,9 @@ public class LockStressTest {
}
System.out.println("Finished " + count + " tries.");
return 0;
}
private static FSLockFactory getNewLockFactory(String lockFactoryClassName) throws IOException {
// try to get static INSTANCE field of class
try {

44
lucene/core/src/java/org/apache/lucene/store/LockVerifyServer.java
View File

@ -24,6 +24,7 @@ import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.concurrent.CountDownLatch;
import java.util.function.Consumer;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.SuppressForbidden;
@ -38,20 +39,12 @@ import org.apache.lucene.util.SuppressForbidden;
* @see LockStressTest
*/
@SuppressForbidden(reason = "System.out required: command line tool")
public class LockVerifyServer {
public static final int START_GUN_SIGNAL = 43;
@SuppressForbidden(reason = "System.out required: command line tool")
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.out.println("Usage: java org.apache.lucene.store.LockVerifyServer bindToIp clients\n");
System.exit(1);
}
int arg = 0;
final String hostname = args[arg++];
final int maxClients = Integer.parseInt(args[arg++]);
// method pkg-private for tests
static void run(String hostname, int maxClients, Consumer<InetSocketAddress> startClients) throws Exception {
try (final ServerSocket s = new ServerSocket()) {
s.setReuseAddress(true);
s.setSoTimeout(30000); // initially 30 secs to give clients enough time to startup
@ -59,8 +52,8 @@ public class LockVerifyServer {
final InetSocketAddress localAddr = (InetSocketAddress) s.getLocalSocketAddress();
System.out.println("Listening on " + localAddr + "...");
// we set the port as a sysprop, so the ANT task can read it. For that to work, this server must run in-process:
System.setProperty("lockverifyserver.port", Integer.toString(localAddr.getPort()));
// callback only for the test to start the clients:
startClients.accept(localAddr);
final Object localLock = new Object();
final int[] lockedID = new int[1];
@ -80,22 +73,22 @@ public class LockVerifyServer {
}
startingGun.await();
os.write(43);
os.write(START_GUN_SIGNAL);
os.flush();
while(true) {
while (true) {
final int command = in.read();
if (command < 0) {
return; // closed
}
synchronized(localLock) {
synchronized (localLock) {
final int currentLock = lockedID[0];
if (currentLock == -2) {
return; // another thread got error, so we exit, too!
}
switch (command) {
case 1:
case VerifyingLockFactory.MSG_LOCK_ACQUIRED:
// Locked
if (currentLock != -1) {
lockedID[0] = -2;
@ -103,7 +96,7 @@ public class LockVerifyServer {
}
lockedID[0] = id;
break;
case 0:
case VerifyingLockFactory.MSG_LOCK_RELEASED:
// Unlocked
if (currentLock != id) {
lockedID[0] = -2;
@ -139,10 +132,17 @@ public class LockVerifyServer {
t.join();
}
// cleanup sysprop
System.clearProperty("lockverifyserver.port");
System.out.println("Server terminated.");
}
}
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.out.println("Usage: java org.apache.lucene.store.LockVerifyServer bindToIp clients\n");
System.exit(1);
}
run(args[0], Integer.parseInt(args[1]), addr -> {});
}
}

6
lucene/core/src/java/org/apache/lucene/store/VerifyingLockFactory.java
View File

@ -36,6 +36,8 @@ import java.io.OutputStream;
*/
public final class VerifyingLockFactory extends LockFactory {
public static final int MSG_LOCK_RELEASED = 0;
public static final int MSG_LOCK_ACQUIRED = 1;
final LockFactory lf;
final InputStream in;
@ -46,7 +48,7 @@ public final class VerifyingLockFactory extends LockFactory {
public CheckedLock(Lock lock) throws IOException {
this.lock = lock;
verify((byte) 1);
verify((byte) MSG_LOCK_ACQUIRED);
}
@Override
@ -58,7 +60,7 @@ public final class VerifyingLockFactory extends LockFactory {
public void close() throws IOException {
try (Lock l = lock) {
l.ensureValid();
verify((byte) 0);
verify((byte) MSG_LOCK_RELEASED);
}
}

104
lucene/core/src/test/org/apache/lucene/store/TestStressLockFactories.java Normal file
View File

@ -0,0 +1,104 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;
import java.io.IOException;
import java.lang.ProcessBuilder.Redirect;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.SuppressForbidden;
@LuceneTestCase.SuppressFileSystems("*")
public class TestStressLockFactories extends LuceneTestCase {
@SuppressForbidden(reason = "ProcessBuilder only allows to redirect to java.io.File")
private static final ProcessBuilder applyRedirection(ProcessBuilder pb, int client, Path dir) {
if (VERBOSE) {
return pb.inheritIO();
} else {
return pb
.redirectError(dir.resolve("err-" + client + ".txt").toFile())
.redirectOutput(dir.resolve("out-" + client + ".txt").toFile())
.redirectInput(Redirect.INHERIT);
}
}
private void runImpl(Class<? extends LockFactory> impl) throws Exception {
final int clients = TEST_NIGHTLY ? 5 : 2;
final String host = "127.0.0.1";
final int delay = 1;
final int rounds = (TEST_NIGHTLY ? 30000 : 500) * RANDOM_MULTIPLIER;
final Path dir = createTempDir(impl.getSimpleName());
final List<Process> processes = new ArrayList<>(clients);
LockVerifyServer.run(host, clients, addr -> {
// spawn clients as separate Java processes
for (int i = 0; i < clients; i++) {
try {
processes.add(applyRedirection(new ProcessBuilder(
Paths.get(System.getProperty("java.home"), "bin", "java").toString(),
"-Xmx32M",
"-cp",
System.getProperty("java.class.path"),
LockStressTest.class.getName(),
Integer.toString(i),
addr.getHostString(),
Integer.toString(addr.getPort()),
impl.getName(),
dir.toString(),
Integer.toString(delay),
Integer.toString(rounds)
), i, dir).start());
} catch (IOException ioe) {
throw new AssertionError("Failed to start child process.", ioe);
}
}
});
// wait for all processes to exit...
try {
for (Process p : processes) {
if (p.waitFor(15, TimeUnit.SECONDS)) {
assertEquals("Process " + p.pid() + " died abnormally?", 0, p.waitFor());
}
}
} finally {
// kill all processes, which are still alive.
for (Process p : processes) {
if (p.isAlive()) {
p.destroyForcibly().waitFor();
}
}
}
}
public void testNativeFSLockFactory() throws Exception {
runImpl(NativeFSLockFactory.class);
}
public void testSimpleFSLockFactory() throws Exception {
runImpl(SimpleFSLockFactory.class);
}
}