Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-8959: Refactored TestSegmentSorting out of TestMiniSolrCloudCluster

This commit is contained in:
Chris Hostetter 2016-12-14 13:18:56 -07:00
parent 512374384a
commit 6525bb56f0
4 changed files with 145 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -290,6 +290,9 @@ Other Changes
* SOLR-9846: OverseerAutoReplicaFailoverThread can take too long to stop and leak out of unit tests. (Mark Miller)
* SOLR-8959: Refactored TestSegmentSorting out of TestMiniSolrCloudCluster (hossman)
================== 6.3.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

12
solr/core/src/test/org/apache/solr/cloud/SegmentTerminateEarlyTestState.java
View File

@ -22,6 +22,7 @@ import java.time.ZonedDateTime;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.Random;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@ -47,7 +48,12 @@ class SegmentTerminateEarlyTestState {
Integer maxTimestampMM = null;
int numDocs = 0;
final Random rand;
public SegmentTerminateEarlyTestState(Random rand) {
this.rand = rand;
}
void addDocuments(CloudSolrClient cloudSolrClient,
int numCommits, int numDocsPerCommit, boolean optimize) throws Exception {
for (int cc = 1; cc <= numCommits; ++cc) {
@ -56,7 +62,7 @@ class SegmentTerminateEarlyTestState {
final Integer docKey = new Integer(numDocs);
SolrInputDocument doc = new SolrInputDocument();
doc.setField(keyField, ""+docKey);
final int MM = TestMiniSolrCloudCluster.random().nextInt(60); // minutes
final int MM = rand.nextInt(60); // minutes
if (minTimestampMM == null || MM <= minTimestampMM.intValue()) {
if (minTimestampMM != null && MM < minTimestampMM.intValue()) {
minTimestampDocKeys.clear();
@ -116,7 +122,7 @@ class SegmentTerminateEarlyTestState {
query.setFields(keyField, oddField, timestampField);
final int rowsWanted = 1;
query.setRows(rowsWanted);
final Boolean shardsInfoWanted = (TestMiniSolrCloudCluster.random().nextBoolean() ? null : new Boolean(TestMiniSolrCloudCluster.random().nextBoolean()));
final Boolean shardsInfoWanted = (rand.nextBoolean() ? null : new Boolean(rand.nextBoolean()));
if (shardsInfoWanted != null) {
query.set(ShardParams.SHARDS_INFO, shardsInfoWanted.booleanValue());
}
@ -163,7 +169,7 @@ class SegmentTerminateEarlyTestState {
query.setSort(timestampField, SolrQuery.ORDER.desc);
query.setFields(keyField, oddField, timestampField);
query.setRows(1);
final Boolean shardsInfoWanted = (TestMiniSolrCloudCluster.random().nextBoolean() ? null : new Boolean(TestMiniSolrCloudCluster.random().nextBoolean()));
final Boolean shardsInfoWanted = (rand.nextBoolean() ? null : new Boolean(rand.nextBoolean()));
if (shardsInfoWanted != null) {
query.set(ShardParams.SHARDS_INFO, shardsInfoWanted.booleanValue());
}

50
solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
View File

@ -16,7 +16,6 @@
*/
package org.apache.solr.cloud;
import java.io.File;
import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.util.ArrayList;
@ -384,53 +383,4 @@ public class TestMiniSolrCloudCluster extends LuceneTestCase {
}
}
@Test
public void testSegmentTerminateEarly() throws Exception {
final String collectionName = "testSegmentTerminateEarlyCollection";
final SegmentTerminateEarlyTestState tstes = new SegmentTerminateEarlyTestState();
File solrXml = new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml");
Builder jettyConfig = JettyConfig.builder();
jettyConfig.waitForLoadingCoresToFinish(null);
final MiniSolrCloudCluster miniCluster = createMiniSolrCloudCluster();
final CloudSolrClient cloudSolrClient = miniCluster.getSolrClient();
cloudSolrClient.setDefaultCollection(collectionName);
try {
// create collection
{
final String asyncId = (random().nextBoolean() ? null : "asyncId("+collectionName+".create)="+random().nextInt());
final Map<String, String> collectionProperties = new HashMap<>();
collectionProperties.put(CoreDescriptor.CORE_CONFIG, "solrconfig-sortingmergepolicyfactory.xml");
createCollection(miniCluster, collectionName, null, asyncId, Boolean.TRUE, collectionProperties);
}
ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330);
// add some documents, then optimize to get merged-sorted segments
tstes.addDocuments(cloudSolrClient, 10, 10, true);
// CommonParams.SEGMENT_TERMINATE_EARLY parameter intentionally absent
tstes.queryTimestampDescending(cloudSolrClient);
// add a few more documents, but don't optimize to have some not-merge-sorted segments
tstes.addDocuments(cloudSolrClient, 2, 10, false);
// CommonParams.SEGMENT_TERMINATE_EARLY parameter now present
tstes.queryTimestampDescendingSegmentTerminateEarlyYes(cloudSolrClient);
tstes.queryTimestampDescendingSegmentTerminateEarlyNo(cloudSolrClient);
// CommonParams.SEGMENT_TERMINATE_EARLY parameter present but it won't be used
tstes.queryTimestampDescendingSegmentTerminateEarlyYesGrouped(cloudSolrClient);
tstes.queryTimestampAscendingSegmentTerminateEarlyYes(cloudSolrClient); // uses a sort order that is _not_ compatible with the merge sort order
}
finally {
miniCluster.shutdown();
}
}
}

133
solr/core/src/test/org/apache/solr/cloud/TestSegmentSorting.java Normal file
View File

@ -0,0 +1,133 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.index.TieredMergePolicyFactory;
import org.junit.After;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestSegmentSorting extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final int NUM_SERVERS = 5;
private static final int NUM_SHARDS = 2;
private static final int REPLICATION_FACTOR = 2;
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(NUM_SERVERS).configure();
}
@After
public void ensureClusterEmpty() throws Exception {
cluster.deleteAllCollections();
cluster.getSolrClient().setDefaultCollection(null);
}
private void createCollection(MiniSolrCloudCluster miniCluster, String collectionName, String createNodeSet, String asyncId,
Boolean indexToPersist, Map<String,String> collectionProperties) throws Exception {
String configName = "solrCloudCollectionConfig";
miniCluster.uploadConfigSet(SolrTestCaseJ4.TEST_PATH().resolve("collection1").resolve("conf"), configName);
final boolean persistIndex = (indexToPersist != null ? indexToPersist.booleanValue() : random().nextBoolean());
if (collectionProperties == null) {
collectionProperties = new HashMap<>();
}
collectionProperties.putIfAbsent(CoreDescriptor.CORE_CONFIG, "solrconfig-tlog.xml");
collectionProperties.putIfAbsent("solr.tests.maxBufferedDocs", "100000");
collectionProperties.putIfAbsent("solr.tests.ramBufferSizeMB", "100");
// use non-test classes so RandomizedRunner isn't necessary
if (random().nextBoolean()) {
collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_MERGEPOLICY, TieredMergePolicy.class.getName());
collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_USEMERGEPOLICY, "true");
collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_USEMERGEPOLICYFACTORY, "false");
} else {
collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_MERGEPOLICYFACTORY, TieredMergePolicyFactory.class.getName());
collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_USEMERGEPOLICYFACTORY, "true");
collectionProperties.putIfAbsent(SolrTestCaseJ4.SYSTEM_PROPERTY_SOLR_TESTS_USEMERGEPOLICY, "false");
}
collectionProperties.putIfAbsent("solr.tests.mergeScheduler", "org.apache.lucene.index.ConcurrentMergeScheduler");
collectionProperties.putIfAbsent("solr.directoryFactory", (persistIndex ? "solr.StandardDirectoryFactory" : "solr.RAMDirectoryFactory"));
if (asyncId == null) {
CollectionAdminRequest.createCollection(collectionName, configName, NUM_SHARDS, REPLICATION_FACTOR)
.setCreateNodeSet(createNodeSet)
.setProperties(collectionProperties)
.process(miniCluster.getSolrClient());
}
else {
CollectionAdminRequest.createCollection(collectionName, configName, NUM_SHARDS, REPLICATION_FACTOR)
.setCreateNodeSet(createNodeSet)
.setProperties(collectionProperties)
.processAndWait(miniCluster.getSolrClient(), 30);
}
}
public void testSegmentTerminateEarly() throws Exception {
final String collectionName = "testSegmentTerminateEarlyCollection";
final SegmentTerminateEarlyTestState tstes = new SegmentTerminateEarlyTestState(random());
final CloudSolrClient cloudSolrClient = cluster.getSolrClient();
cloudSolrClient.setDefaultCollection(collectionName);
// create collection
{
final String asyncId = (random().nextBoolean() ? null : "asyncId("+collectionName+".create)="+random().nextInt());
final Map<String, String> collectionProperties = new HashMap<>();
collectionProperties.put(CoreDescriptor.CORE_CONFIG, "solrconfig-sortingmergepolicyfactory.xml");
createCollection(cluster, collectionName, null, asyncId, Boolean.TRUE, collectionProperties);
}
ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330);
// add some documents, then optimize to get merged-sorted segments
tstes.addDocuments(cloudSolrClient, 10, 10, true);
// CommonParams.SEGMENT_TERMINATE_EARLY parameter intentionally absent
tstes.queryTimestampDescending(cloudSolrClient);
// add a few more documents, but don't optimize to have some not-merge-sorted segments
tstes.addDocuments(cloudSolrClient, 2, 10, false);
// CommonParams.SEGMENT_TERMINATE_EARLY parameter now present
tstes.queryTimestampDescendingSegmentTerminateEarlyYes(cloudSolrClient);
tstes.queryTimestampDescendingSegmentTerminateEarlyNo(cloudSolrClient);
// CommonParams.SEGMENT_TERMINATE_EARLY parameter present but it won't be used
tstes.queryTimestampDescendingSegmentTerminateEarlyYesGrouped(cloudSolrClient);
tstes.queryTimestampAscendingSegmentTerminateEarlyYes(cloudSolrClient); // uses a sort order that is _not_ compatible with the merge sort order
}
}