From cf828163bdfa010c87f1171b6919e444bd0ff01c Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Tue, 12 Mar 2019 13:06:19 -0400 Subject: [PATCH] SOLR-13330: Improve HDFS tests Related JIRAs: * SOLR-11010 * SOLR-11381 * SOLR-12040 * SOLR-13297 Changes: * Consolidate hdfs configuration into HdfsTestUtil * Ensure socketTimeout long enough for HDFS tests * Ensure HdfsTestUtil.getClientConfiguration used in tests * Replace deprecated HDFS calls * Use try-with-resources to ensure closing of HDFS resources Signed-off-by: Kevin Risden --- solr/CHANGES.txt | 2 + .../solr/core/HdfsDirectoryFactory.java | 2 +- .../collection1/conf/solrconfig-minhash.xml | 4 +- .../solr/collection1/conf/solrconfig.xml | 4 +- .../cloud/MoveReplicaHDFSFailoverTest.java | 5 - .../solr/cloud/MoveReplicaHDFSTest.java | 4 - .../solr/cloud/UnloadDistributedZkTest.java | 40 +++---- .../HdfsCollectionsAPIDistributedZkTest.java | 4 - .../TestHdfsCloudBackupRestore.java | 1 - .../HdfsAutoAddReplicasIntegrationTest.java | 6 - .../cloud/hdfs/HDFSCollectionsAPITest.java | 5 - .../hdfs/HdfsBasicDistributedZkTest.java | 2 - .../solr/cloud/hdfs/HdfsRecoverLeaseTest.java | 6 +- .../solr/cloud/hdfs/HdfsRecoveryZkTest.java | 16 +-- .../hdfs/HdfsRestartWhileUpdatingTest.java | 10 +- .../apache/solr/cloud/hdfs/HdfsTestUtil.java | 106 +++++++++++------- .../solr/cloud/hdfs/HdfsThreadLeakTest.java | 43 +++---- ...fsTlogReplayBufferedWhileIndexingTest.java | 6 +- .../hdfs/HdfsUnloadDistributedZkTest.java | 4 +- .../HdfsWriteToMultipleCollectionsTest.java | 7 +- .../solr/cloud/hdfs/StressHdfsTest.java | 1 - .../solr/core/HdfsDirectoryFactoryTest.java | 68 ++++++----- .../handler/TestHdfsBackupRestoreCore.java | 22 ++-- .../solr/index/hdfs/CheckHdfsIndexTest.java | 24 ++-- .../apache/solr/search/TestRecoveryHdfs.java | 62 +++++----- .../apache/solr/update/TestHdfsUpdateLog.java | 7 +- 26 files changed, 208 insertions(+), 253 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 1c6ee04b8ea..bdb73509f04 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -158,6 +158,8 @@ Other Changes * SOLR-13307: Ensure HDFS tests clear System properties they set (Kevin Risden) +* SOLR-13330: Improve HDFS tests (Kevin Risden) + ================== 8.0.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java index 13e1de188e5..a0806ed52bc 100644 --- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java @@ -531,7 +531,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol boolean accept = false; String pathName = path.getName(); try { - accept = fs.isDirectory(path) && !path.equals(currentIndexDirPath) && + accept = fs.getFileStatus(path).isDirectory() && !path.equals(currentIndexDirPath) && (pathName.equals("index") || pathName.matches(INDEX_W_TIMESTAMP_REGEX)); } catch (IOException e) { log.error("Error checking if path {} is an old index directory, caused by: {}", path, e); diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml index 333c1db1aa7..3df6f2bc13b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml @@ -49,8 +49,8 @@ ${solr.hdfs.blockcache.enabled:true} ${solr.hdfs.blockcache.global:true} ${solr.hdfs.blockcache.write.enabled:false} - 10 - 1 + ${solr.hdfs.blockcache.blocksperbank:10} + ${solr.hdfs.blockcache.slab.count:1} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml index 9e7e5058f05..c68e7e04d05 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml @@ -49,8 +49,8 @@ ${solr.hdfs.blockcache.enabled:true} ${solr.hdfs.blockcache.global:true} ${solr.hdfs.blockcache.write.enabled:false} - 10 - 1 + ${solr.hdfs.blockcache.blocksperbank:10} + ${solr.hdfs.blockcache.slab.count:1} diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java index 2a26956e655..f66029b34e9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSFailoverTest.java @@ -52,13 +52,10 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase { .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) .configure(); - System.setProperty("solr.hdfs.blockcache.enabled", "false"); dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); ZkConfigManager configManager = new ZkConfigManager(zkClient()); configManager.uploadConfigDir(configset("cloud-hdfs"), "conf1"); - - System.setProperty("solr.hdfs.home", HdfsTestUtil.getDataDir(dfsCluster, "data")); } @AfterClass @@ -70,8 +67,6 @@ public class MoveReplicaHDFSFailoverTest extends SolrCloudTestCase { HdfsTestUtil.teardownClass(dfsCluster); } finally { dfsCluster = null; - System.clearProperty("solr.hdfs.home"); - System.clearProperty("solr.hdfs.blockcache.enabled"); } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java index 9678c2e0afb..82040276669 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java @@ -39,8 +39,6 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest { @BeforeClass public static void setupClass() throws Exception { - System.setProperty("solr.hdfs.blockcache.blocksperbank", "512"); - System.setProperty("tests.hdfs.numdatanodes", "1"); dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); } @@ -50,8 +48,6 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest { HdfsTestUtil.teardownClass(dfsCluster); } finally { dfsCluster = null; - System.clearProperty("solr.hdfs.blockcache.blocksperbank"); - System.clearProperty("tests.hdfs.numdatanodes"); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java index 943774828ea..003e933047c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java @@ -31,7 +31,6 @@ import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.TimeSource; import org.apache.solr.core.SolrCore; @@ -42,6 +41,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.Random; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -54,23 +54,19 @@ import java.util.concurrent.TimeUnit; @Slow @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") public class UnloadDistributedZkTest extends BasicDistributedZkTest { - - protected String getSolrXml() { - return "solr.xml"; - } - public UnloadDistributedZkTest() { super(); } + protected String getSolrXml() { + return "solr.xml"; + } + @Test - //28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018 - // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 public void test() throws Exception { - testCoreUnloadAndLeaders(); // long testUnloadLotsOfCores(); // long - + testUnloadShardAndCollection(); } @@ -78,9 +74,9 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { boolean shouldBePresent, int expectedSliceCount) throws Exception { final TimeOut timeout = new TimeOut(45, TimeUnit.SECONDS, TimeSource.NANO_TIME); Boolean isPresent = null; // null meaning "don't know" - while (null == isPresent || shouldBePresent != isPresent.booleanValue()) { + while (null == isPresent || shouldBePresent != isPresent) { final DocCollection docCollection = getCommonCloudSolrClient().getZkStateReader().getClusterState().getCollectionOrNull(collectionName); - final Collection slices = (docCollection != null) ? docCollection.getSlices() : null; + final Collection slices = (docCollection != null) ? docCollection.getSlices() : Collections.emptyList(); if (timeout.hasTimedOut()) { printLayout(); fail("checkCoreNamePresenceAndSliceCount failed:" @@ -88,14 +84,12 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { +" shouldBePresent="+shouldBePresent+" isPresent="+isPresent +" expectedSliceCount="+expectedSliceCount+" actualSliceCount="+slices.size()); } - if (expectedSliceCount == (slices == null ? 0 : slices.size())) { + if (expectedSliceCount == slices.size()) { isPresent = false; - if (slices != null) { - for (Slice slice : slices) { - for (Replica replica : slice.getReplicas()) { - if (coreName.equals(replica.get("core"))) { - isPresent = true; - } + for (Slice slice : slices) { + for (Replica replica : slice.getReplicas()) { + if (coreName.equals(replica.get("core"))) { + isPresent = true; } } } @@ -105,7 +99,6 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { } private void testUnloadShardAndCollection() throws Exception{ - final int numShards = 2; final String collection = "test_unload_shard_and_collection"; @@ -285,7 +278,6 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { Unload unloadCmd = new Unload(false); unloadCmd.setCoreName(leaderProps.getCoreName()); - SolrParams p = (ModifiableSolrParams) unloadCmd.getParams(); collectionClient.request(unloadCmd); } tries = 50; @@ -335,7 +327,6 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { assertEquals(found1, found3); assertEquals(found3, found4); } - } private void testUnloadLotsOfCores() throws Exception { @@ -343,7 +334,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { try (final HttpSolrClient adminClient = (HttpSolrClient) jetty.newClient(15000, 60000)) { int numReplicas = atLeast(3); ThreadPoolExecutor executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0, Integer.MAX_VALUE, - 5, TimeUnit.SECONDS, new SynchronousQueue(), + 5, TimeUnit.SECONDS, new SynchronousQueue<>(), new DefaultSolrThreadFactory("testExecutor")); try { // create the cores @@ -353,7 +344,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { } executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0, Integer.MAX_VALUE, 5, - TimeUnit.SECONDS, new SynchronousQueue(), + TimeUnit.SECONDS, new SynchronousQueue<>(), new DefaultSolrThreadFactory("testExecutor")); try { for (int j = 0; j < numReplicas; j++) { @@ -374,5 +365,4 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest { } } } - } diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java index 700165baac7..94b3102f0c8 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCollectionsAPIDistributedZkTest.java @@ -38,8 +38,6 @@ public class HdfsCollectionsAPIDistributedZkTest extends CollectionsAPIDistribut @BeforeClass public static void setupClass() throws Exception { - System.setProperty("solr.hdfs.blockcache.blocksperbank", "512"); - System.setProperty("tests.hdfs.numdatanodes", "1"); dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); } @@ -49,8 +47,6 @@ public class HdfsCollectionsAPIDistributedZkTest extends CollectionsAPIDistribut HdfsTestUtil.teardownClass(dfsCluster); } finally { dfsCluster = null; - System.clearProperty("solr.hdfs.blockcache.blocksperbank"); - System.clearProperty("tests.hdfs.numdatanodes"); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java index a9901e80509..0d876db5813 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java @@ -112,7 +112,6 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa try { URI uri = new URI(hdfsUri); Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); fs = FileSystem.get(uri, conf); if (fs instanceof DistributedFileSystem) { diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java index 37ce805e829..8d8ef9c76c5 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java @@ -38,9 +38,6 @@ public class HdfsAutoAddReplicasIntegrationTest extends AutoAddReplicasIntegrati @BeforeClass public static void setupClass() throws Exception { - System.setProperty("solr.hdfs.blockcache.global", "true"); - System.setProperty("solr.hdfs.blockcache.blocksperbank", "512"); - System.setProperty("tests.hdfs.numdatanodes", "1"); dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); } @@ -50,9 +47,6 @@ public class HdfsAutoAddReplicasIntegrationTest extends AutoAddReplicasIntegrati HdfsTestUtil.teardownClass(dfsCluster); } finally { dfsCluster = null; - System.clearProperty("solr.hdfs.blockcache.global"); - System.clearProperty("solr.hdfs.blockcache.blocksperbank"); - System.clearProperty("tests.hdfs.numdatanodes"); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java index c06b28a08d1..6a93042e960 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java @@ -43,13 +43,10 @@ public class HDFSCollectionsAPITest extends SolrCloudTestCase { configureCluster(2) .configure(); - System.setProperty("solr.hdfs.blockcache.enabled", "false"); dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); ZkConfigManager configManager = new ZkConfigManager(zkClient()); configManager.uploadConfigDir(configset("cloud-hdfs"), "conf1"); - - System.setProperty("solr.hdfs.home", HdfsTestUtil.getDataDir(dfsCluster, "data")); } @@ -62,8 +59,6 @@ public class HDFSCollectionsAPITest extends SolrCloudTestCase { HdfsTestUtil.teardownClass(dfsCluster); } finally { dfsCluster = null; - System.clearProperty("solr.hdfs.blockcache.enabled"); - System.clearProperty("solr.hdfs.home"); } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java index 59293c6597f..192e7ee13d0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java @@ -38,7 +38,6 @@ public class HdfsBasicDistributedZkTest extends BasicDistributedZkTest { @BeforeClass public static void setupClass() throws Exception { - System.setProperty("tests.hdfs.numdatanodes", "1"); dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); } @@ -53,7 +52,6 @@ public class HdfsBasicDistributedZkTest extends BasicDistributedZkTest { HdfsTestUtil.teardownClass(dfsCluster); } finally { dfsCluster = null; - System.clearProperty("tests.hdfs.numdatanodes"); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java index fcacb8b5d58..ad8556cf0ca 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoverLeaseTest.java @@ -72,8 +72,7 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 { URI uri = dfsCluster.getURI(); Path path = new Path(uri); - Configuration conf = new Configuration(); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); + Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); FileSystem fs1 = FileSystem.get(path.toUri(), conf); Path testFile = new Path(uri.toString() + "/testfile"); FSDataOutputStream out = fs1.create(testFile); @@ -131,8 +130,7 @@ public class HdfsRecoverLeaseTest extends SolrTestCaseJ4 { final URI uri = dfsCluster.getURI(); final Path path = new Path(uri); - final Configuration conf = new Configuration(); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); + final Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); // n threads create files class WriterThread extends Thread { diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java index b881292d1f2..6e6bd3df633 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java @@ -18,35 +18,30 @@ package org.apache.solr.cloud.hdfs; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneTestCase.BadApple; +import org.apache.lucene.util.LuceneTestCase.Nightly; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.cloud.RecoveryZkTest; import org.apache.solr.common.cloud.ZkConfigManager; import org.apache.solr.util.BadHdfsThreadsFilter; -import org.apache.solr.util.LogLevel; import org.junit.AfterClass; import org.junit.BeforeClass; @Slow -@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") -//@Nightly +@BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") +@Nightly @ThreadLeakFilters(defaultFilters = true, filters = { BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) -@LogLevel("org.apache.solr.update.HdfsTransactionLog=DEBUG") public class HdfsRecoveryZkTest extends RecoveryZkTest { - private static MiniDFSCluster dfsCluster; @BeforeClass public static void setupClass() throws Exception { dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); - System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048"); ZkConfigManager configManager = new ZkConfigManager(zkClient()); configManager.uploadConfigDir(configset("cloud-hdfs"), "conf"); - - System.setProperty("solr.hdfs.home", HdfsTestUtil.getDataDir(dfsCluster, "data")); } @AfterClass @@ -58,10 +53,7 @@ public class HdfsRecoveryZkTest extends RecoveryZkTest { HdfsTestUtil.teardownClass(dfsCluster); } finally { dfsCluster = null; - System.clearProperty("solr.hdfs.blockcache.blocksperbank"); - System.clearProperty("solr.hdfs.home"); } } } - } diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java index 02a79434023..64af1091996 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRestartWhileUpdatingTest.java @@ -16,8 +16,6 @@ */ package org.apache.solr.cloud.hdfs; -import java.io.IOException; - import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.cloud.RestartWhileUpdatingTest; @@ -34,16 +32,15 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) public class HdfsRestartWhileUpdatingTest extends RestartWhileUpdatingTest { + private static MiniDFSCluster dfsCluster; + public HdfsRestartWhileUpdatingTest() throws Exception { super(); } - private static MiniDFSCluster dfsCluster; - @BeforeClass public static void setupClass() throws Exception { dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); - System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048"); } @AfterClass @@ -52,12 +49,11 @@ public class HdfsRestartWhileUpdatingTest extends RestartWhileUpdatingTest { HdfsTestUtil.teardownClass(dfsCluster); } finally { dfsCluster = null; - System.clearProperty("solr.hdfs.blockcache.blocksperbank"); } } @Override - protected String getDataDir(String dataDir) throws IOException { + protected String getDataDir(String dataDir) { return HdfsTestUtil.getDataDir(dfsCluster, dataDir); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java index d601565a5fa..6987a6a49f1 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java @@ -17,9 +17,9 @@ package org.apache.solr.cloud.hdfs; import java.io.File; -import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.URI; +import java.util.Enumeration; import java.util.Locale; import java.util.Map; import java.util.Timer; @@ -47,6 +47,8 @@ import org.apache.solr.util.HdfsUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.lucene.util.LuceneTestCase.random; + public class HdfsTestUtil { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -99,49 +101,49 @@ public class HdfsTestUtil { if (!HA_TESTING_ENABLED) haTesting = false; - int dataNodes = Integer.getInteger("tests.hdfs.numdatanodes", 2); - - Configuration conf = new Configuration(); - conf.set("dfs.block.access.token.enable", "false"); - conf.set("dfs.permissions.enabled", "false"); - conf.set("hadoop.security.authentication", "simple"); + Configuration conf = getBasicConfiguration(new Configuration()); conf.set("hdfs.minidfs.basedir", dir + File.separator + "hdfsBaseDir"); conf.set("dfs.namenode.name.dir", dir + File.separator + "nameNodeNameDir"); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); + // Disable metrics logging for HDFS + conf.setInt("dfs.namenode.metrics.logger.period.seconds", 0); + conf.setInt("dfs.datanode.metrics.logger.period.seconds", 0); System.setProperty("test.build.data", dir + File.separator + "hdfs" + File.separator + "build"); System.setProperty("test.cache.data", dir + File.separator + "hdfs" + File.separator + "cache"); System.setProperty("solr.lock.type", DirectoryFactory.LOCK_TYPE_HDFS); - System.setProperty("solr.hdfs.blockcache.global", - System.getProperty("solr.hdfs.blockcache.global", Boolean.toString(LuceneTestCase.random().nextBoolean()))); + // test-files/solr/solr.xml sets this to be 15000. This isn't long enough for HDFS in some cases. + System.setProperty("socketTimeout", "90000"); - final MiniDFSCluster dfsCluster; - - if (!haTesting) { - dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(dataNodes).format(true).build(); - - System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home")); + String blockcacheGlobal = System.getProperty("solr.hdfs.blockcache.global", Boolean.toString(random().nextBoolean())); + System.setProperty("solr.hdfs.blockcache.global", blockcacheGlobal); + // Limit memory usage for HDFS tests + if(Boolean.parseBoolean(blockcacheGlobal)) { + System.setProperty("solr.hdfs.blockcache.blocksperbank", "4096"); } else { - dfsCluster = new MiniDFSCluster.Builder(conf) - .nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(dataNodes) - .build(); - - Configuration haConfig = getClientConfiguration(dfsCluster); - - HdfsUtil.TEST_CONF = haConfig; - System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home")); + System.setProperty("solr.hdfs.blockcache.blocksperbank", "512"); + System.setProperty("tests.hdfs.numdatanodes", "1"); } + int dataNodes = Integer.getInteger("tests.hdfs.numdatanodes", 2); + final MiniDFSCluster.Builder dfsClusterBuilder = new MiniDFSCluster.Builder(conf) + .numDataNodes(dataNodes).format(true); + if (haTesting) { + dfsClusterBuilder.nnTopology(MiniDFSNNTopology.simpleHATopology()); + } + MiniDFSCluster dfsCluster = dfsClusterBuilder.build(); + HdfsUtil.TEST_CONF = getClientConfiguration(dfsCluster); + System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home")); + dfsCluster.waitActive(); if (haTesting) dfsCluster.transitionToActive(0); - int rndMode = LuceneTestCase.random().nextInt(3); + int rndMode = random().nextInt(3); if (safeModeTesting && rndMode == 1) { NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false); - int rnd = LuceneTestCase.random().nextInt(10000); + int rnd = random().nextInt(10000); Timer timer = new Timer(); timers.put(dfsCluster, timer); timer.schedule(new TimerTask() { @@ -151,9 +153,8 @@ public class HdfsTestUtil { NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode()); } }, rnd); - } else if (haTesting && rndMode == 2) { - int rnd = LuceneTestCase.random().nextInt(30000); + int rnd = random().nextInt(30000); Timer timer = new Timer(); timers.put(dfsCluster, timer); timer.schedule(new TimerTask() { @@ -176,7 +177,7 @@ public class HdfsTestUtil { URI uri = dfsCluster.getURI(); Path hdfsDirPath = new Path(uri.toString() + "/solr/collection1/core_node1/data/tlog/tlog.0000000000000000000"); // tran log already being created testing - badTlogOutStreamFs = FileSystem.get(hdfsDirPath.toUri(), conf); + badTlogOutStreamFs = FileSystem.get(hdfsDirPath.toUri(), getClientConfiguration(dfsCluster)); badTlogOutStream = badTlogOutStreamFs.create(hdfsDirPath); } @@ -185,18 +186,23 @@ public class HdfsTestUtil { return dfsCluster; } + private static Configuration getBasicConfiguration(Configuration conf) { + conf.setBoolean("dfs.block.access.token.enable", false); + conf.setBoolean("dfs.permissions.enabled", false); + conf.set("hadoop.security.authentication", "simple"); + conf.setBoolean("fs.hdfs.impl.disable.cache", true); + return conf; + } + public static Configuration getClientConfiguration(MiniDFSCluster dfsCluster) { + Configuration conf = getBasicConfiguration(dfsCluster.getConfiguration(0)); if (dfsCluster.getNameNodeInfos().length > 1) { - Configuration conf = new Configuration(); HATestUtil.setFailoverConfigurations(dfsCluster, conf); - return conf; - } else { - return new Configuration(); } + return conf; } public static void teardownClass(MiniDFSCluster dfsCluster) throws Exception { - if (badTlogOutStream != null) { IOUtils.closeQuietly(badTlogOutStream); } @@ -205,16 +211,19 @@ public class HdfsTestUtil { IOUtils.closeQuietly(badTlogOutStreamFs); } - SolrTestCaseJ4.resetFactory(); - try { + try { + SolrTestCaseJ4.resetFactory(); + } catch (Exception e) { + log.error("Exception trying to reset solr.directoryFactory", e); + } if (dfsCluster != null) { Timer timer = timers.remove(dfsCluster); if (timer != null) { timer.cancel(); } try { - dfsCluster.shutdown(); + dfsCluster.shutdown(true); } catch (Error e) { // Added in SOLR-7134 // Rarely, this can fail to either a NullPointerException @@ -224,16 +233,27 @@ public class HdfsTestUtil { } } } finally { - System.clearProperty("solr.lock.type"); System.clearProperty("test.build.data"); System.clearProperty("test.cache.data"); - System.clearProperty("solr.hdfs.home"); - System.clearProperty("solr.hdfs.blockcache.global"); + + System.clearProperty("socketTimeout"); + + System.clearProperty("tests.hdfs.numdatanodes"); + + System.clearProperty("solr.lock.type"); + + // Clear "solr.hdfs." system properties + Enumeration propertyNames = System.getProperties().propertyNames(); + while(propertyNames.hasMoreElements()) { + String propertyName = String.valueOf(propertyNames.nextElement()); + if(propertyName.startsWith("solr.hdfs.")) { + System.clearProperty(propertyName); + } + } } } - public static String getDataDir(MiniDFSCluster dfsCluster, String dataDir) - throws IOException { + public static String getDataDir(MiniDFSCluster dfsCluster, String dataDir) { if (dataDir == null) { return null; } @@ -250,7 +270,7 @@ public class HdfsTestUtil { return "hdfs://" + logicalName; } else { URI uri = dfsCluster.getURI(0); - return uri.toString() ; + return uri.toString(); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java index 1cd68190ba1..f250d53f46a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsThreadLeakTest.java @@ -25,9 +25,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.util.BadHdfsThreadsFilter; -import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -37,7 +35,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) public class HdfsThreadLeakTest extends SolrTestCaseJ4 { - private static MiniDFSCluster dfsCluster; @BeforeClass @@ -47,38 +44,26 @@ public class HdfsThreadLeakTest extends SolrTestCaseJ4 { @AfterClass public static void afterClass() throws Exception { - HdfsTestUtil.teardownClass(dfsCluster); - dfsCluster = null; - } - - @Before - public void setUp() throws Exception { - super.setUp(); - } - - @After - public void tearDown() throws Exception { - super.tearDown(); + try { + HdfsTestUtil.teardownClass(dfsCluster); + } finally { + dfsCluster = null; + } } @Test public void testBasic() throws IOException { String uri = HdfsTestUtil.getURI(dfsCluster); Path path = new Path(uri); - Configuration conf = new Configuration(); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); - FileSystem fs = FileSystem.get(path.toUri(), conf); - Path testFile = new Path(uri.toString() + "/testfile"); - FSDataOutputStream out = fs.create(testFile); - - out.write(5); - out.hflush(); - out.close(); + Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); + try(FileSystem fs = FileSystem.get(path.toUri(), conf)) { + Path testFile = new Path(uri + "/testfile"); + try(FSDataOutputStream out = fs.create(testFile)) { + out.write(5); + out.hflush(); + } - ((DistributedFileSystem) fs).recoverLease(testFile); - - - fs.close(); + ((DistributedFileSystem) fs).recoverLease(testFile); + } } - } diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java index 52b3a82f522..ff19b090d16 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTlogReplayBufferedWhileIndexingTest.java @@ -16,8 +16,6 @@ */ package org.apache.solr.cloud.hdfs; -import java.io.IOException; - import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest; @@ -42,7 +40,6 @@ public class HdfsTlogReplayBufferedWhileIndexingTest extends TlogReplayBufferedW @BeforeClass public static void setupClass() throws Exception { - System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048"); dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); } @@ -52,12 +49,11 @@ public class HdfsTlogReplayBufferedWhileIndexingTest extends TlogReplayBufferedW HdfsTestUtil.teardownClass(dfsCluster); } finally { dfsCluster = null; - System.clearProperty("solr.hdfs.blockcache.blocksperbank"); } } @Override - protected String getDataDir(String dataDir) throws IOException { + protected String getDataDir(String dataDir) { return HdfsTestUtil.getDataDir(dfsCluster, dataDir); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java index c15b8c36532..0d3d23ea646 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java @@ -16,8 +16,6 @@ */ package org.apache.solr.cloud.hdfs; -import java.io.IOException; - import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.cloud.UnloadDistributedZkTest; @@ -51,7 +49,7 @@ public class HdfsUnloadDistributedZkTest extends UnloadDistributedZkTest { } @Override - protected String getDataDir(String dataDir) throws IOException { + protected String getDataDir(String dataDir) { return HdfsTestUtil.getDataDir(dfsCluster, dataDir); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java index 6f48b989f79..1f316033785 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsWriteToMultipleCollectionsTest.java @@ -141,10 +141,9 @@ public class HdfsWriteToMultipleCollectionsTest extends BasicDistributedZkTest { Directory dir = factory.get(core.getDataDir(), null, null); try { long dataDirSize = factory.size(dir); - FileSystem fileSystem = null; - - fileSystem = FileSystem.newInstance( - new Path(core.getDataDir()).toUri(), new Configuration()); + Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); + FileSystem fileSystem = FileSystem.newInstance( + new Path(core.getDataDir()).toUri(), conf); long size = fileSystem.getContentSummary( new Path(core.getDataDir())).getLength(); assertEquals(size, dataDirSize); diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java index b3d5e2750b6..04f93875f9b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java @@ -229,7 +229,6 @@ public class StressHdfsTest extends BasicDistributedZkTest { // check that all dirs are gone for (String dataDir : dataDirs) { Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); try(FileSystem fs = FileSystem.get(new URI(HdfsTestUtil.getURI(dfsCluster)), conf)) { assertFalse( "Data directory exists after collection removal : " + dataDir, diff --git a/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java index c599fb7233a..e8c16a3535d 100644 --- a/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/core/HdfsDirectoryFactoryTest.java @@ -16,6 +16,8 @@ */ package org.apache.solr.core; +import java.io.FileNotFoundException; +import java.io.IOException; import java.nio.file.Path; import java.text.SimpleDateFormat; import java.util.Date; @@ -24,7 +26,6 @@ import java.util.Locale; import java.util.Map; import java.util.Random; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.lucene.store.Directory; @@ -55,7 +56,6 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 { @BeforeClass public static void setupClass() throws Exception { - System.setProperty("solr.hdfs.blockcache.blocksperbank", "1024"); dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false); } @@ -65,41 +65,42 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 { HdfsTestUtil.teardownClass(dfsCluster); } finally { dfsCluster = null; - System.clearProperty("solr.hdfs.home"); + System.clearProperty(HdfsDirectoryFactory.HDFS_HOME); + System.clearProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY); + System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED); System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB); - System.clearProperty("solr.hdfs.blockcache.blocksperbank"); + System.clearProperty(HdfsDirectoryFactory.LOCALITYMETRICS_ENABLED); } } @Test public void testInitArgsOrSysPropConfig() throws Exception { try(HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) { - // test sys prop config - System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1"); + System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1"); hdfsFactory.init(new NamedList<>()); String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor()); assertTrue(dataHome.endsWith("/solr1/mock/data")); - System.clearProperty("solr.hdfs.home"); + System.clearProperty(HdfsDirectoryFactory.HDFS_HOME); // test init args config NamedList nl = new NamedList<>(); - nl.add("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr2"); + nl.add(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr2"); hdfsFactory.init(nl); dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor()); assertTrue(dataHome.endsWith("/solr2/mock/data")); // test sys prop and init args config - init args wins - System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1"); + System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1"); hdfsFactory.init(nl); dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor()); assertTrue(dataHome.endsWith("/solr2/mock/data")); - System.clearProperty("solr.hdfs.home"); + System.clearProperty(HdfsDirectoryFactory.HDFS_HOME); // set conf dir by sys prop Path confDir = createTempDir(); @@ -141,44 +142,49 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 { @Test public void testCleanupOldIndexDirectories() throws Exception { try (HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory()) { - System.setProperty("solr.hdfs.home", HdfsTestUtil.getURI(dfsCluster) + "/solr1"); + System.setProperty(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr1"); hdfsFactory.init(new NamedList<>()); String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor()); assertTrue(dataHome.endsWith("/solr1/mock/data")); - System.clearProperty("solr.hdfs.home"); + System.clearProperty(HdfsDirectoryFactory.HDFS_HOME); - FileSystem hdfs = dfsCluster.getFileSystem(); + try(FileSystem hdfs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) { + org.apache.hadoop.fs.Path dataHomePath = new org.apache.hadoop.fs.Path(dataHome); + org.apache.hadoop.fs.Path currentIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index"); + assertFalse(checkHdfsDirectory(hdfs,currentIndexDirPath)); + hdfs.mkdirs(currentIndexDirPath); + assertTrue(checkHdfsDirectory(hdfs, currentIndexDirPath)); - org.apache.hadoop.fs.Path dataHomePath = new org.apache.hadoop.fs.Path(dataHome); - org.apache.hadoop.fs.Path currentIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index"); - assertTrue(!hdfs.isDirectory(currentIndexDirPath)); - hdfs.mkdirs(currentIndexDirPath); - assertTrue(hdfs.isDirectory(currentIndexDirPath)); + String timestamp1 = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date()); + org.apache.hadoop.fs.Path oldIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index." + timestamp1); + assertFalse(checkHdfsDirectory(hdfs,oldIndexDirPath)); + hdfs.mkdirs(oldIndexDirPath); + assertTrue(checkHdfsDirectory(hdfs, oldIndexDirPath)); - String timestamp1 = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date()); - org.apache.hadoop.fs.Path oldIndexDirPath = new org.apache.hadoop.fs.Path(dataHomePath, "index." + timestamp1); - assertTrue(!hdfs.isDirectory(oldIndexDirPath)); - hdfs.mkdirs(oldIndexDirPath); - assertTrue(hdfs.isDirectory(oldIndexDirPath)); + hdfsFactory.cleanupOldIndexDirectories(dataHomePath.toString(), currentIndexDirPath.toString(), false); - hdfsFactory.cleanupOldIndexDirectories(dataHomePath.toString(), currentIndexDirPath.toString(), false); + assertTrue(checkHdfsDirectory(hdfs, currentIndexDirPath)); + assertFalse(checkHdfsDirectory(hdfs, oldIndexDirPath)); + } + } + } - assertTrue(hdfs.isDirectory(currentIndexDirPath)); - assertTrue(!hdfs.isDirectory(oldIndexDirPath)); + private boolean checkHdfsDirectory(FileSystem hdfs, org.apache.hadoop.fs.Path path) throws IOException { + try { + return hdfs.getFileStatus(path).isDirectory(); + } catch (FileNotFoundException e) { + return false; } } @Test public void testLocalityReporter() throws Exception { - Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); - conf.set("dfs.permissions.enabled", "false"); - Random r = random(); try(HdfsDirectoryFactory factory = new HdfsDirectoryFactory()) { SolrMetricManager metricManager = new SolrMetricManager(); String registry = TestUtil.randomSimpleString(r, 2, 10); String scope = TestUtil.randomSimpleString(r, 2, 10); - Map props = new HashMap(); + Map props = new HashMap<>(); props.put(HdfsDirectoryFactory.HDFS_HOME, HdfsTestUtil.getURI(dfsCluster) + "/solr"); props.put(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false"); props.put(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_ENABLE, "false"); @@ -190,7 +196,7 @@ public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 { MetricsMap metrics = (MetricsMap) ((SolrMetricManager.GaugeWrapper) metricManager.registry(registry).getMetrics().get("OTHER." + scope + ".hdfsLocality")).getGauge(); // We haven't done anything, so there should be no data Map statistics = metrics.getValue(); - assertEquals("Saw bytes that were not written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL), 0l, + assertEquals("Saw bytes that were not written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL), 0L, statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_TOTAL)); assertEquals( "Counted bytes as local when none written: " + statistics.get(HdfsLocalityReporter.LOCALITY_BYTES_RATIO), 0, diff --git a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java index 60f77b4cd62..f51c4acfd3c 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java +++ b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.lucene.util.TestUtil; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -109,7 +108,6 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase { try { URI uri = new URI(hdfsUri); Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); fs = FileSystem.get(uri, conf); if (fs instanceof DistributedFileSystem) { @@ -147,13 +145,17 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase { IOUtils.closeQuietly(fs); fs = null; try { - HdfsTestUtil.teardownClass(dfsCluster); + SolrTestCaseJ4.resetFactory(); } finally { - dfsCluster = null; - System.clearProperty("solr.hdfs.home"); - System.clearProperty("solr.hdfs.default.backup.path"); - System.clearProperty("test.build.data"); - System.clearProperty("test.cache.data"); + try { + HdfsTestUtil.teardownClass(dfsCluster); + } finally { + dfsCluster = null; + System.clearProperty("solr.hdfs.home"); + System.clearProperty("solr.hdfs.default.backup.path"); + System.clearProperty("test.build.data"); + System.clearProperty("test.cache.data"); + } } } @@ -180,12 +182,12 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase { boolean testViaReplicationHandler = random().nextBoolean(); String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString(); - try (SolrClient masterClient = getHttpSolrClient(replicaBaseUrl)) { + try (HttpSolrClient masterClient = getHttpSolrClient(replicaBaseUrl)) { // Create a backup. if (testViaReplicationHandler) { log.info("Running Backup via replication handler"); BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName); - CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, coreName, null); + CheckBackupStatus checkBackupStatus = new CheckBackupStatus(masterClient, coreName, null); while (!checkBackupStatus.success) { checkBackupStatus.fetchStatus(); Thread.sleep(1000); diff --git a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java index d1d3f722b5f..7a7ec7800fb 100644 --- a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java +++ b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java @@ -19,6 +19,7 @@ package org.apache.solr.index.hdfs; import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.lucene.index.BaseTestCheckIndex; @@ -66,8 +67,11 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase { @AfterClass public static void teardownClass() throws Exception { - HdfsTestUtil.teardownClass(dfsCluster); - dfsCluster = null; + try { + HdfsTestUtil.teardownClass(dfsCluster); + } finally { + dfsCluster = null; + } } @Override @@ -76,17 +80,21 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase { super.setUp(); Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); - directory = new HdfsDirectory(path, conf); } @Override @After public void tearDown() throws Exception { - directory.close(); - dfsCluster.getFileSystem().delete(path, true); - super.tearDown(); + try { + directory.close(); + } finally { + try(FileSystem fs = FileSystem.get(HdfsTestUtil.getClientConfiguration(dfsCluster))) { + fs.delete(path, true); + } finally { + super.tearDown(); + } + } } @Override @@ -108,7 +116,7 @@ public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase { SolrClient client = clients.get(0); NamedList response = client.query(new SolrQuery().setRequestHandler("/admin/system")).getResponse(); NamedList coreInfo = (NamedList) response.get("core"); - String indexDir = (String) ((NamedList) coreInfo.get("directory")).get("data") + "/index"; + String indexDir = ((NamedList) coreInfo.get("directory")).get("data") + "/index"; args = new String[] {indexDir}; } diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java index 33caf74b071..9c97c03baba 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java +++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java @@ -82,7 +82,6 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 { try { URI uri = new URI(hdfsUri); Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); fs = FileSystem.get(uri, conf); } catch (IOException | URISyntaxException e) { throw new RuntimeException(e); @@ -257,7 +256,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 { ulog.bufferUpdates(); assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); Future rinfoFuture = ulog.applyBufferedUpdates(); - assertTrue(rinfoFuture == null); + assertNull(rinfoFuture); assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); ulog.bufferUpdates(); @@ -295,7 +294,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 { rinfoFuture = ulog.applyBufferedUpdates(); - assertTrue(rinfoFuture != null); + assertNotNull(rinfoFuture); assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState()); @@ -341,7 +340,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 { logReplay.drainPermits(); rinfoFuture = ulog.applyBufferedUpdates(); - assertTrue(rinfoFuture != null); + assertNotNull(rinfoFuture); assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState()); // apply a single update @@ -409,7 +408,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 { ulog.bufferUpdates(); assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); Future rinfoFuture = ulog.applyBufferedUpdates(); - assertTrue(rinfoFuture == null); + assertNull(rinfoFuture); assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); ulog.bufferUpdates(); @@ -737,7 +736,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 { // addDocs(1, start, new LinkedList()); // don't add this to the versions list because we are going to lose it... h.close(); - files = HdfsUpdateLog.getLogList(fs, new Path(logDir));; + files = HdfsUpdateLog.getLogList(fs, new Path(logDir)); Arrays.sort(files); FSDataOutputStream dos = fs.create(new Path(new Path(logDir), files[files.length-1]), (short)1); @@ -850,7 +849,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 { Arrays.sort(files); FSDataOutputStream dos = fs.create(new Path(logDir, files[files.length-1]), (short)1); - dos.write(new byte[(int)800]); // zero out file + dos.write(new byte[800]); // zero out file dos.close(); @@ -920,34 +919,31 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 { Arrays.sort(files); String fname = files[files.length-1]; - FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length-1])); - dos.writeLong(0xffffffffffffffffL); - dos.writeChars("This should be appended to a good log file, representing a bad partially written record."); - dos.close(); - - FSDataInputStream dis = fs.open(new Path(logDir, files[files.length-1])); - byte[] content = new byte[(int)dis.available()]; + try(FSDataOutputStream dos = fs.append(new Path(logDir, files[files.length-1]))) { + dos.writeLong(0xffffffffffffffffL); + dos.writeChars("This should be appended to a good log file, representing a bad partially written record."); + } - dis.readFully(content); + try(FSDataInputStream dis = fs.open(new Path(logDir, files[files.length-1]))) { + byte[] content = new byte[dis.available()]; + dis.readFully(content); - dis.close(); + // Now make a newer log file with just the IDs changed. NOTE: this may not work if log format changes too much! + findReplace("AAAAAA".getBytes(StandardCharsets.UTF_8), "aaaaaa".getBytes(StandardCharsets.UTF_8), content); + findReplace("BBBBBB".getBytes(StandardCharsets.UTF_8), "bbbbbb".getBytes(StandardCharsets.UTF_8), content); + findReplace("CCCCCC".getBytes(StandardCharsets.UTF_8), "cccccc".getBytes(StandardCharsets.UTF_8), content); - // Now make a newer log file with just the IDs changed. NOTE: this may not work if log format changes too much! - findReplace("AAAAAA".getBytes(StandardCharsets.UTF_8), "aaaaaa".getBytes(StandardCharsets.UTF_8), content); - findReplace("BBBBBB".getBytes(StandardCharsets.UTF_8), "bbbbbb".getBytes(StandardCharsets.UTF_8), content); - findReplace("CCCCCC".getBytes(StandardCharsets.UTF_8), "cccccc".getBytes(StandardCharsets.UTF_8), content); + // WARNING... assumes format of .00000n where n is less than 9 + long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf(".") + 1)); + String fname2 = String.format(Locale.ROOT, + UpdateLog.LOG_FILENAME_PATTERN, + UpdateLog.TLOG_NAME, + logNumber + 1); - // WARNING... assumes format of .00000n where n is less than 9 - long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf(".") + 1)); - String fname2 = String.format(Locale.ROOT, - UpdateLog.LOG_FILENAME_PATTERN, - UpdateLog.TLOG_NAME, - logNumber + 1); - - dos = fs.create(new Path(logDir, fname2), (short)1); - dos.write(content); - dos.close(); - + try(FSDataOutputStream dos = fs.create(new Path(logDir, fname2), (short)1)) { + dos.write(content); + } + } logReplay.release(1000); logReplayFinish.drainPermits(); @@ -971,9 +967,7 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 { for(;;) { idx = indexOf(from, data, idx + from.length); // skip over previous match if (idx < 0) break; - for (int i=0; i