SOLR-7359: Add additional testing for FSHDFSUtils#recoverFileLease.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1676531 13f79535-47bb-0310-9956-ffa450edef68

solr/core/src/java/org/apache/solr/util/FSHDFSUtils.java

@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.lang.reflect.Method;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -38,6 +39,9 @@ import org.slf4j.LoggerFactory;
 
 public class FSHDFSUtils {
   public static Logger log = LoggerFactory.getLogger(FSHDFSUtils.class);
+  
+  // internal, for tests
+  public static AtomicLong RECOVER_LEASE_SUCCESS_COUNT = new AtomicLong();
 
   public interface CallerInfo {
     boolean isCallerClosed();
@@ -136,6 +140,9 @@ public class FSHDFSUtils {
         throw iioe;
       }
     }
+    if (recovered) {
+      RECOVER_LEASE_SUCCESS_COUNT.incrementAndGet();
+    }
     return recovered;
   }
 

solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java

@@ -0,0 +1,248 @@
+package org.apache.solr.cloud.hdfs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.util.BadHdfsThreadsFilter;
+import org.apache.solr.util.FSHDFSUtils;
+import org.apache.solr.util.FSHDFSUtils.CallerInfo;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {
+    BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+})
+public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 {
+  
+  private static MiniDFSCluster dfsCluster;
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
+  }
+
+  @AfterClass
+  public static void afterClass() throws Exception {
+    HdfsTestUtil.teardownClass(dfsCluster);
+    dfsCluster = null;
+  }
+  
+  @Before
+  public void setUp() throws Exception {
+    super.setUp();
+  }
+  
+  @After
+  public void tearDown() throws Exception {
+    super.tearDown();
+  }
+  
+  @Test
+  public void testBasic() throws IOException {
+    long startRecoverLeaseSuccessCount = FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get();
+    
+    URI uri = dfsCluster.getURI();
+    Path path = new Path(uri);
+    Configuration conf = new Configuration();
+    conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+    FileSystem fs1 = FileSystem.get(path.toUri(), conf);
+    Path testFile = new Path(uri.toString() + "/testfile");
+    FSDataOutputStream out = fs1.create(testFile);
+    
+    out.write(5);
+    out.hflush();
+    out.close();
+
+    FSHDFSUtils.recoverFileLease(fs1, testFile, conf, new CallerInfo() {
+      
+      @Override
+      public boolean isCallerClosed() {
+        return false;
+      }
+    });
+    assertEquals(0, FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get() - startRecoverLeaseSuccessCount);
+    
+    fs1.close();
+
+    
+    FileSystem fs2 = FileSystem.get(path.toUri(), conf);
+    Path testFile2 = new Path(uri.toString() + "/testfile2");
+    FSDataOutputStream out2 = fs2.create(testFile2);
+    
+    if (random().nextBoolean()) {
+      int cnt = random().nextInt(100);
+      for (int i = 0; i < cnt; i++) {
+        out2.write(random().nextInt(20000));
+      }
+      out2.hflush();
+    }
+
+    
+    // closing the fs will close the file it seems
+    // fs2.close();
+    
+    FileSystem fs3 = FileSystem.get(path.toUri(), conf);
+
+    FSHDFSUtils.recoverFileLease(fs3, testFile2, conf, new CallerInfo() {
+      
+      @Override
+      public boolean isCallerClosed() {
+        return false;
+      }
+    });
+    assertEquals(1, FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get() - startRecoverLeaseSuccessCount);
+    
+    fs3.close();
+    fs2.close();
+  }
+  
+  @Test
+  public void testMultiThreaded() throws Exception {
+    long startRecoverLeaseSuccessCount = FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get();
+    
+    final URI uri = dfsCluster.getURI();
+    final Path path = new Path(uri);
+    final Configuration conf = new Configuration();
+    conf.setBoolean("fs.hdfs.impl.disable.cache", true);
+    
+    // n threads create files
+    class WriterThread extends Thread {
+      private FileSystem fs;
+      private int id;
+      
+      public WriterThread(int id) {
+        this.id = id;
+        try {
+          fs = FileSystem.get(path.toUri(), conf);
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+      
+      @Override
+      public void run() {
+        Path testFile = new Path(uri.toString() + "/file-" + id);
+        FSDataOutputStream out;
+        try {
+          out = fs.create(testFile);
+          
+          if (random().nextBoolean()) {
+            int cnt = random().nextInt(100);
+            for (int i = 0; i < cnt; i++) {
+              out.write(random().nextInt(20000));
+            }
+            out.hflush();
+          }
+        } catch (IOException e) {
+          throw new RuntimeException();
+        }
+      }
+      
+      public void close() throws IOException {
+        fs.close();
+      }
+      
+      public int getFileId() {
+        return id;
+      }
+    }
+    
+    class RecoverThread extends Thread {
+      private FileSystem fs;
+      private int id;
+      
+      public RecoverThread(int id) {
+        this.id = id;
+        try {
+          fs = FileSystem.get(path.toUri(), conf);
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+      
+      @Override
+      public void run() {
+        Path testFile = new Path(uri.toString() + "/file-" + id);
+        try {
+          FSHDFSUtils.recoverFileLease(fs, testFile, conf, new CallerInfo() {
+            
+            @Override
+            public boolean isCallerClosed() {
+              return false;
+            }
+          });
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+      }
+      
+      public void close() throws IOException {
+        fs.close();
+      }
+    }
+    
+    Set<WriterThread> writerThreads = new HashSet<WriterThread>();
+    Set<RecoverThread> recoverThreads = new HashSet<RecoverThread>();
+    
+    int threadCount = 3;
+    for (int i = 0; i < threadCount; i++) {
+      WriterThread wt = new WriterThread(i);
+      writerThreads.add(wt);
+      wt.run();
+    }
+    
+    for (WriterThread wt : writerThreads) {
+      wt.join();
+    }
+    
+    Thread.sleep(2000);
+    
+    for (WriterThread wt : writerThreads) {
+      RecoverThread rt = new RecoverThread(wt.getFileId());
+      recoverThreads.add(rt);
+      rt.run();
+    }
+    
+    for (WriterThread wt : writerThreads) {
+      wt.close();
+    }
+    
+    for (RecoverThread rt : recoverThreads) {
+      rt.close();
+    }
+
+    assertEquals(threadCount, FSHDFSUtils.RECOVER_LEASE_SUCCESS_COUNT.get() - startRecoverLeaseSuccessCount);
+    
+  }
+
+}