LUCENE-9512: Move LockFactory stress test to be a unit/integration test (#1842)

Co-authored-by: Dawid Weiss <dawid.weiss@carrotsearch.com>

gradle/testing/randomization/policies/tests.policy

@@ -64,6 +64,9 @@ grant {
   permission java.lang.RuntimePermission "getClassLoader";
   permission java.lang.RuntimePermission "setContextClassLoader";
 
+  // TestLockFactoriesMultiJVM opens a random port on 127.0.0.1 (port 0 = ephemeral port range):
+  permission java.net.SocketPermission "127.0.0.1:0", "accept,listen,resolve";
+
   // read access to all system properties:
   permission java.util.PropertyPermission "*", "read";
   // write access to only these:

lucene/CHANGES.txt

@@ -252,6 +252,10 @@ Other
 
 * LUCENE-9470: Make TestXYMultiPolygonShapeQueries more resilient for CONTAINS queries. (Ignacio Vera)
 
+* LUCENE-9512: Move LockFactory stress test to be a unit/integration
+  test. (Uwe Schindler, Dawid Weiss, Robert Muir)
+
+
 ======================= Lucene 8.6.2 =======================
 
 Bug Fixes

lucene/core/src/java/org/apache/lucene/store/LockStressTest.java

@@ -30,7 +30,7 @@ import org.apache.lucene.util.SuppressForbidden;
 
 /**
  * Simple standalone tool that forever acquires and releases a
- * lock using a specific LockFactory.  Run without any args
+ * lock using a specific {@link LockFactory}.  Run without any args
  * to see usage.
  *
  * @see VerifyingLockFactory
@@ -38,11 +38,9 @@ import org.apache.lucene.util.SuppressForbidden;
  */ 
 
 public class LockStressTest {
-  
   static final String LOCK_FILE_NAME = "test.lock";
   
   @SuppressForbidden(reason = "System.out required: command line tool")
-  @SuppressWarnings("try")
   public static void main(String[] args) throws Exception {
     if (args.length != 7) {
       System.out.println("Usage: java org.apache.lucene.store.LockStressTest myID verifierHost verifierPort lockFactoryClassName lockDirName sleepTimeMS count\n" +
@@ -65,12 +63,6 @@ public class LockStressTest {
 
     int arg = 0;
     final int myID = Integer.parseInt(args[arg++]);
-
-    if (myID < 0 || myID > 255) {
-      System.out.println("myID must be a unique int 0..255");
-      System.exit(1);
-    }
-
     final String verifierHost = args[arg++];
     final int verifierPort = Integer.parseInt(args[arg++]);
     final String lockFactoryClassName = args[arg++];
@@ -78,8 +70,21 @@ public class LockStressTest {
     final int sleepTimeMS = Integer.parseInt(args[arg++]);
     final int count = Integer.parseInt(args[arg++]);
 
+    int exitCode = run(myID, verifierHost, verifierPort, lockFactoryClassName, lockDirPath, sleepTimeMS, count);
+    System.exit(exitCode);
+  }
+
+  @SuppressForbidden(reason = "System.out required: command line tool")
+  @SuppressWarnings("try")
+  private static int run(int myID, String verifierHost, int verifierPort, String lockFactoryClassName,
+                        Path lockDirPath, int sleepTimeMS, int count) throws IOException, InterruptedException {
+    if (myID < 0 || myID > 255) {
+      System.out.println("myID must be a unique int 0..255");
+      return 1;
+    }
+
     final LockFactory lockFactory = getNewLockFactory(lockFactoryClassName);
-    // we test the lock factory directly, so we don't need it on the directory itsself (the directory is just for testing)
+    // we test the lock factory directly, so we don't need it on the directory itself (the directory is just for testing)
     final FSDirectory lockDir = new NIOFSDirectory(lockDirPath, NoLockFactory.INSTANCE);
     final InetSocketAddress addr = new InetSocketAddress(verifierHost, verifierPort);
     System.out.println("Connecting to server " + addr +
@@ -96,7 +101,7 @@ public class LockStressTest {
       final Random rnd = new Random();
 
       // wait for starting gun
-      if (in.read() != 43) {
+      if (in.read() != LockVerifyServer.START_GUN_SIGNAL) {
         throw new IOException("Protocol violation");
       }
 
@@ -126,9 +131,9 @@ public class LockStressTest {
     }
 
     System.out.println("Finished " + count + " tries.");
+    return 0;
   }
 
-
   private static FSLockFactory getNewLockFactory(String lockFactoryClassName) throws IOException {
     // try to get static INSTANCE field of class
     try {

lucene/core/src/java/org/apache/lucene/store/LockVerifyServer.java

@@ -24,6 +24,7 @@ import java.net.InetSocketAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.util.concurrent.CountDownLatch;
+import java.util.function.Consumer;
 
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.SuppressForbidden;
@@ -38,20 +39,12 @@ import org.apache.lucene.util.SuppressForbidden;
  * @see LockStressTest
  */
 
+@SuppressForbidden(reason = "System.out required: command line tool")
 public class LockVerifyServer {
+  public static final int START_GUN_SIGNAL = 43;
 
-  @SuppressForbidden(reason = "System.out required: command line tool")
+  // method pkg-private for tests
-  public static void main(String[] args) throws Exception {
+  static void run(String hostname, int maxClients, Consumer<InetSocketAddress> startClients) throws Exception {
-
-    if (args.length != 2) {
-      System.out.println("Usage: java org.apache.lucene.store.LockVerifyServer bindToIp clients\n");
-      System.exit(1);
-    }
-
-    int arg = 0;
-    final String hostname = args[arg++];
-    final int maxClients = Integer.parseInt(args[arg++]);
-
     try (final ServerSocket s = new ServerSocket()) {
       s.setReuseAddress(true);
       s.setSoTimeout(30000); // initially 30 secs to give clients enough time to startup
@@ -59,8 +52,8 @@ public class LockVerifyServer {
       final InetSocketAddress localAddr = (InetSocketAddress) s.getLocalSocketAddress();
       System.out.println("Listening on " + localAddr + "...");
       
-      // we set the port as a sysprop, so the ANT task can read it. For that to work, this server must run in-process:
+      // callback only for the test to start the clients:
-      System.setProperty("lockverifyserver.port", Integer.toString(localAddr.getPort()));
+      startClients.accept(localAddr);
       
       final Object localLock = new Object();
       final int[] lockedID = new int[1];
@@ -80,22 +73,22 @@ public class LockVerifyServer {
               }
               
               startingGun.await();
-              os.write(43);
+              os.write(START_GUN_SIGNAL);
               os.flush();
               
-              while(true) {
+              while (true) {
                 final int command = in.read();
                 if (command < 0) {
                   return; // closed
                 }
                 
-                synchronized(localLock) {
+                synchronized (localLock) {
                   final int currentLock = lockedID[0];
                   if (currentLock == -2) {
                     return; // another thread got error, so we exit, too!
                   }
                   switch (command) {
-                    case 1:
+                    case VerifyingLockFactory.MSG_LOCK_ACQUIRED:
                       // Locked
                       if (currentLock != -1) {
                         lockedID[0] = -2;
@@ -103,7 +96,7 @@ public class LockVerifyServer {
                       }
                       lockedID[0] = id;
                       break;
-                    case 0:
+                    case VerifyingLockFactory.MSG_LOCK_RELEASED:
                       // Unlocked
                       if (currentLock != id) {
                         lockedID[0] = -2;
@@ -139,10 +132,17 @@ public class LockVerifyServer {
         t.join();
       }
 
-      // cleanup sysprop
-      System.clearProperty("lockverifyserver.port");
-
       System.out.println("Server terminated.");
     }
   }
+
+  public static void main(String[] args) throws Exception {
+    if (args.length != 2) {
+      System.out.println("Usage: java org.apache.lucene.store.LockVerifyServer bindToIp clients\n");
+      System.exit(1);
+    }
+
+    run(args[0], Integer.parseInt(args[1]), addr -> {});
+  }
+  
 }

lucene/core/src/java/org/apache/lucene/store/VerifyingLockFactory.java

@@ -36,6 +36,8 @@ import java.io.OutputStream;
  */
 
 public final class VerifyingLockFactory extends LockFactory {
+  public static final int MSG_LOCK_RELEASED = 0;
+  public static final int MSG_LOCK_ACQUIRED = 1;
 
   final LockFactory lf;
   final InputStream in;
@@ -46,7 +48,7 @@ public final class VerifyingLockFactory extends LockFactory {
 
     public CheckedLock(Lock lock) throws IOException {
       this.lock = lock;
-      verify((byte) 1);
+      verify((byte) MSG_LOCK_ACQUIRED);
     }
 
     @Override
@@ -58,7 +60,7 @@ public final class VerifyingLockFactory extends LockFactory {
     public void close() throws IOException {
       try (Lock l = lock) {
         l.ensureValid();
-        verify((byte) 0);
+        verify((byte) MSG_LOCK_RELEASED);
       }
     }
 

lucene/core/src/test/org/apache/lucene/store/TestStressLockFactories.java

@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.store;
+
+import java.io.IOException;
+import java.lang.ProcessBuilder.Redirect;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.SuppressForbidden;
+
+@LuceneTestCase.SuppressFileSystems("*")
+public class TestStressLockFactories extends LuceneTestCase {
+  
+  @SuppressForbidden(reason = "ProcessBuilder only allows to redirect to java.io.File")
+  private static final ProcessBuilder applyRedirection(ProcessBuilder pb, int client, Path dir) {
+    if (VERBOSE) {
+      return pb.inheritIO();
+    } else {
+      return pb
+        .redirectError(dir.resolve("err-" + client + ".txt").toFile())
+        .redirectOutput(dir.resolve("out-" + client + ".txt").toFile())
+        .redirectInput(Redirect.INHERIT);
+    }
+  }
+  
+  private void runImpl(Class<? extends LockFactory> impl) throws Exception {
+    final int clients = TEST_NIGHTLY ? 5 : 2;
+    final String host = "127.0.0.1";
+    final int delay = 1;
+    final int rounds = (TEST_NIGHTLY ? 30000 : 500) * RANDOM_MULTIPLIER;
+    
+    final Path dir = createTempDir(impl.getSimpleName());
+    
+    final List<Process> processes = new ArrayList<>(clients);
+
+    LockVerifyServer.run(host, clients, addr -> {
+      // spawn clients as separate Java processes
+      for (int i = 0; i < clients; i++) {
+        try {
+          processes.add(applyRedirection(new ProcessBuilder(
+              Paths.get(System.getProperty("java.home"), "bin", "java").toString(),
+              "-Xmx32M",
+              "-cp",
+              System.getProperty("java.class.path"),
+              LockStressTest.class.getName(),
+              Integer.toString(i),
+              addr.getHostString(),
+              Integer.toString(addr.getPort()),
+              impl.getName(),
+              dir.toString(),
+              Integer.toString(delay),
+              Integer.toString(rounds)
+            ), i, dir).start());
+        } catch (IOException ioe) {
+          throw new AssertionError("Failed to start child process.", ioe);
+        }
+      }
+    });
+     
+    // wait for all processes to exit...
+    try {
+      for (Process p : processes) {
+        if (p.waitFor(15, TimeUnit.SECONDS)) {
+          assertEquals("Process " + p.pid() + " died abnormally?", 0, p.waitFor());
+        }
+      }
+    } finally {
+      // kill all processes, which are still alive.
+      for (Process p : processes) {
+        if (p.isAlive()) {
+          p.destroyForcibly().waitFor();
+        }
+      }
+    }
+  }
+  
+  public void testNativeFSLockFactory() throws Exception {
+    runImpl(NativeFSLockFactory.class);
+  }
+
+  public void testSimpleFSLockFactory() throws Exception {
+    runImpl(SimpleFSLockFactory.class);
+  }
+
+}