From cdf254415b0494b23d468235333aacbff0243284 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Thu, 6 Mar 2014 14:06:35 +0000 Subject: [PATCH 1/4] SOLR-5820: OverseerCollectionProcessor#lookupReplicas has a timeout that is too short and a bad error message on timeout. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1574883 13f79535-47bb-0310-9956-ffa450edef68 --- .../cloud/OverseerCollectionProcessor.java | 45 ++++++++++--------- .../CollectionsAPIDistributedZkTest.java | 2 +- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java index ee806a5c856..bee9e1848bc 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java @@ -1627,7 +1627,7 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread { if (!created) throw new SolrException(ErrorCode.SERVER_ERROR, "Could not fully createcollection: " + message.getStr("name")); - log.info("going to create cores replicas shardNames {} , repFactor : {}", shardNames, repFactor); + log.info("Creating SolrCores for new collection, shardNames {} , replicationFactor : {}", shardNames, repFactor); Map coresToCreate = new LinkedHashMap(); for (int i = 1; i <= shardNames.size(); i++) { String sliceName = shardNames.get(i-1); @@ -1671,14 +1671,17 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread { sreq.actualShards = sreq.shards; sreq.params = params; - if(isLegacyCloud) shardHandler.submit(sreq, sreq.shards[0], sreq.params); - else coresToCreate.put(coreName, sreq); + if(isLegacyCloud) { + shardHandler.submit(sreq, sreq.shards[0], sreq.params); + } else { + coresToCreate.put(coreName, sreq); + } } } if(!isLegacyCloud) { - //wait for all replica entries to be created - Map replicas = lookupReplicas(collectionName, coresToCreate.keySet()); + // wait for all replica entries to be created + Map replicas = waitToSeeReplicasInState(collectionName, coresToCreate.keySet()); for (Map.Entry e : coresToCreate.entrySet()) { ShardRequest sreq = e.getValue(); sreq.params.set(CoreAdminParams.CORE_NODE_NAME, replicas.get(e.getKey()).getName()); @@ -1704,37 +1707,35 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread { } } - private Map lookupReplicas(String collectionName, Collection coreNames) throws InterruptedException { + private Map waitToSeeReplicasInState(String collectionName, Collection coreNames) throws InterruptedException { Map result = new HashMap(); - long endTime = System.nanoTime() + TimeUnit.NANOSECONDS.convert(3, TimeUnit.SECONDS); - for(;;) { - DocCollection coll = zkStateReader.getClusterState().getCollection(collectionName); - for (String coreName : coreNames) { - if(result.containsKey(coreName)) continue; + long endTime = System.nanoTime() + TimeUnit.NANOSECONDS.convert(30, TimeUnit.SECONDS); + while (true) { + DocCollection coll = zkStateReader.getClusterState().getCollection( + collectionName); + for (String coreName : coreNames) { + if (result.containsKey(coreName)) continue; for (Slice slice : coll.getSlices()) { for (Replica replica : slice.getReplicas()) { - if(coreName.equals(replica.getStr(ZkStateReader.CORE_NAME_PROP))) { - result.put(coreName,replica); + if (coreName.equals(replica.getStr(ZkStateReader.CORE_NAME_PROP))) { + result.put(coreName, replica); break; } } } } - - if(result.size() == coreNames.size()) { + + if (result.size() == coreNames.size()) { return result; } - if( System.nanoTime() > endTime) { - //time up . throw exception and go out - throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to create replica entries in ZK"); + if (System.nanoTime() > endTime) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Timed out waiting to see all replicas in cluster state."); } - + Thread.sleep(100); } - } - private void addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException { String collection = message.getStr(COLLECTION_PROP); String node = message.getStr("node"); @@ -1789,7 +1790,7 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread { ZkStateReader.STATE_PROP, ZkStateReader.DOWN, ZkStateReader.BASE_URL_PROP,zkStateReader.getBaseUrlForNodeName(node)); Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(props)); - params.set(CoreAdminParams.CORE_NODE_NAME, lookupReplicas(collection, Collections.singletonList(coreName)).get(coreName).getName()); + params.set(CoreAdminParams.CORE_NODE_NAME, waitToSeeReplicasInState(collection, Collections.singletonList(coreName)).get(coreName).getName()); } diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java index e257b5e66ca..5a4c45a04a3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java @@ -665,7 +665,7 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa boolean disableLegacy = random().nextBoolean(); CloudSolrServer client1 = null; - if(disableLegacy) { + if (disableLegacy) { log.info("legacyCloud=false"); client1 = createCloudClient(null); setClusterProp(client1, ZkStateReader.LEGACY_CLOUD, "false"); From 692f577d435b4a7deabc2ec3af193e3969e99dba Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Thu, 6 Mar 2014 15:25:28 +0000 Subject: [PATCH 2/4] fix test failure: limit max token length git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1574917 13f79535-47bb-0310-9956-ffa450edef68 --- .../lucene/index/ThreadedIndexingAndSearchingTestCase.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java index 739d45a631c..bbf00a544f6 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java @@ -440,8 +440,10 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas if (dir instanceof BaseDirectoryWrapper) { ((BaseDirectoryWrapper) dir).setCheckIndexOnClose(false); // don't double-checkIndex, we do it ourselves. } + MockAnalyzer analyzer = new MockAnalyzer(random()); + analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH)); final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random())).setInfoStream(new FailOnNonBulkMergesInfoStream()); + analyzer).setInfoStream(new FailOnNonBulkMergesInfoStream()); if (LuceneTestCase.TEST_NIGHTLY) { // newIWConfig makes smallish max seg size, which From b221115ebc471de39f462abef1a817a6d642d58f Mon Sep 17 00:00:00 2001 From: "Chris M. Hostetter" Date: Thu, 6 Mar 2014 16:13:20 +0000 Subject: [PATCH 3/4] SOLR-5628: work arround for this test to avoid whatever bug is in the cloud test framework git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1574941 13f79535-47bb-0310-9956-ffa450edef68 --- .../test/org/apache/solr/cloud/TestDistribDocBasedVersion.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solr/core/src/test/org/apache/solr/cloud/TestDistribDocBasedVersion.java b/solr/core/src/test/org/apache/solr/cloud/TestDistribDocBasedVersion.java index 2f6f796c69c..86043b0f011 100755 --- a/solr/core/src/test/org/apache/solr/cloud/TestDistribDocBasedVersion.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestDistribDocBasedVersion.java @@ -103,6 +103,8 @@ public class TestDistribDocBasedVersion extends AbstractFullDistribZkTestBase { doTestDocVersions(); doTestHardFail(); + commit(); // work arround SOLR-5628 + testFinished = true; } finally { if (!testFinished) { From 31dacf44948d26cdd2d1330f86fac2c388de75b6 Mon Sep 17 00:00:00 2001 From: Steven Rowe Date: Thu, 6 Mar 2014 16:33:32 +0000 Subject: [PATCH 4/4] SOLR-3854: IntelliJ config: add solr example lib test dependency to map-reduce and dataimporthandler contribs git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1574951 13f79535-47bb-0310-9956-ffa450edef68 --- .../idea/solr/contrib/dataimporthandler/dataimporthandler.iml | 1 + dev-tools/idea/solr/contrib/map-reduce/map-reduce.iml | 1 + 2 files changed, 2 insertions(+) diff --git a/dev-tools/idea/solr/contrib/dataimporthandler/dataimporthandler.iml b/dev-tools/idea/solr/contrib/dataimporthandler/dataimporthandler.iml index 17eed225e48..380e99a7a3c 100644 --- a/dev-tools/idea/solr/contrib/dataimporthandler/dataimporthandler.iml +++ b/dev-tools/idea/solr/contrib/dataimporthandler/dataimporthandler.iml @@ -16,6 +16,7 @@ + diff --git a/dev-tools/idea/solr/contrib/map-reduce/map-reduce.iml b/dev-tools/idea/solr/contrib/map-reduce/map-reduce.iml index 676d4f64a9a..572e1d838ac 100644 --- a/dev-tools/idea/solr/contrib/map-reduce/map-reduce.iml +++ b/dev-tools/idea/solr/contrib/map-reduce/map-reduce.iml @@ -18,6 +18,7 @@ +