SOLR-13370 - Trying reduction of cluster size, but not clear that

should help from local tests/profile, but maybe it helps in more memory
constrained build servers, this and prior version both beasted success
10 rounds of 5 on a machine with lots of memory.
This commit is contained in:
Gus Heck 2019-04-09 09:37:48 -04:00
parent 3e628b562c
commit 4a93199803
1 changed files with 30 additions and 19 deletions

View File

@ -64,10 +64,14 @@ public class CategoryRoutedAliasUpdateProcessorTest extends RoutedAliasUpdatePro
private int lastDocId = 0; private int lastDocId = 0;
private static CloudSolrClient solrClient; private static CloudSolrClient solrClient;
private int numDocsDeletedOrFailed = 0; private int numDocsDeletedOrFailed = 0;
// uncomment to create pause for attaching profiler.
// static {
// JOptionPane.showMessageDialog(null,"Ready?");
// }
@Before @Before
public void doBefore() throws Exception { public void doBefore() throws Exception {
configureCluster(4).configure(); configureCluster(1).configure();
solrClient = getCloudSolrClient(cluster); solrClient = getCloudSolrClient(cluster);
//log this to help debug potential causes of problems //log this to help debug potential causes of problems
log.info("SolrClient: {}", solrClient); log.info("SolrClient: {}", solrClient);
@ -85,6 +89,8 @@ public class CategoryRoutedAliasUpdateProcessorTest extends RoutedAliasUpdatePro
IOUtils.close(solrClient); IOUtils.close(solrClient);
} }
@Test
@Slow
public void testNonEnglish() throws Exception { public void testNonEnglish() throws Exception {
// test to document the expected behavior with non-english text for categories // test to document the expected behavior with non-english text for categories
// the present expectation is that non-latin text and many accented latin characters // the present expectation is that non-latin text and many accented latin characters
@ -128,7 +134,7 @@ public class CategoryRoutedAliasUpdateProcessorTest extends RoutedAliasUpdatePro
CollectionAdminRequest.createCategoryRoutedAlias(getAlias(), categoryField, 20, CollectionAdminRequest.createCategoryRoutedAlias(getAlias(), categoryField, 20,
CollectionAdminRequest.createCollection("_unused_", configName, 1, 1) CollectionAdminRequest.createCollection("_unused_", configName, 1, 1)
.setMaxShardsPerNode(2)) .setMaxShardsPerNode(12))
.process(solrClient); .process(solrClient);
addDocsAndCommit(true, addDocsAndCommit(true,
newDoc(somethingInChinese), newDoc(somethingInChinese),
@ -188,7 +194,6 @@ public class CategoryRoutedAliasUpdateProcessorTest extends RoutedAliasUpdatePro
// now we index a document // now we index a document
addDocsAndCommit(true, newDoc(SHIPS[0])); addDocsAndCommit(true, newDoc(SHIPS[0]));
//assertDocRoutedToCol(lastDocId, col23rd);
String uninitialized = getAlias() + "__CRA__" + CategoryRoutedAlias.UNINITIALIZED; String uninitialized = getAlias() + "__CRA__" + CategoryRoutedAlias.UNINITIALIZED;
assertInvariants(colVogon, uninitialized); assertInvariants(colVogon, uninitialized);
@ -357,7 +362,7 @@ public class CategoryRoutedAliasUpdateProcessorTest extends RoutedAliasUpdatePro
final int numReplicas = 1 + random().nextInt(3); final int numReplicas = 1 + random().nextInt(3);
CollectionAdminRequest.createCategoryRoutedAlias(getAlias(), categoryField, 20, CollectionAdminRequest.createCategoryRoutedAlias(getAlias(), categoryField, 20,
CollectionAdminRequest.createCollection("_unused_", configName, numShards, numReplicas) CollectionAdminRequest.createCollection("_unused_", configName, numShards, numReplicas)
.setMaxShardsPerNode(numReplicas)) .setMaxShardsPerNode(numReplicas*numShards))
.process(solrClient); .process(solrClient);
// cause some collections to be created // cause some collections to be created
@ -397,36 +402,42 @@ public class CategoryRoutedAliasUpdateProcessorTest extends RoutedAliasUpdatePro
} }
/*
* We expect the following invariants:
* 1.) to see all the supplied collections
* 2.) Independently Querying all collections we can find to yield the same number of docs as querying the alias
* 3.) find as many docs as have been added but not deleted/failed
*/
private void assertInvariants(String... expectedColls) throws IOException, SolrServerException { private void assertInvariants(String... expectedColls) throws IOException, SolrServerException {
final int expectNumFound = lastDocId - numDocsDeletedOrFailed; //lastDocId is effectively # generated docs final int expectNumFound = lastDocId - numDocsDeletedOrFailed; //lastDocId is effectively # generated docs
List<String> cols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(getAlias()); List<String> observedCols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(getAlias());
cols = new ArrayList<>(cols); observedCols = new ArrayList<>(observedCols);
cols.sort(String::compareTo); // don't really care about the order here. observedCols.sort(String::compareTo); // don't really care about the order here.
assert !cols.isEmpty(); assert !observedCols.isEmpty();
int totalNumFound = 0; int numFoundViaCollections = 0;
for (String col : cols) { for (String col : observedCols) {
final QueryResponse colResponse = solrClient.query(col, params( final QueryResponse colResponse = solrClient.query(col, params(
"q", "*:*", "q", "*:*",
"rows", "0")); "rows", "0"));
long numFound = colResponse.getResults().getNumFound(); long numFound = colResponse.getResults().getNumFound();
if (numFound > 0) { if (numFound > 0) {
totalNumFound += numFound; numFoundViaCollections += numFound;
} }
} }
final QueryResponse colResponse = solrClient.query(getAlias(), params( final QueryResponse colResponse = solrClient.query(getAlias(), params(
"q", "*:*", "q", "*:*",
"rows", "0")); "rows", "0"));
long aliasNumFound = colResponse.getResults().getNumFound(); long numFoundViaAlias = colResponse.getResults().getNumFound();
List<String> actual = Arrays.asList(expectedColls); List<String> expectedList = Arrays.asList(expectedColls);
actual.sort(String::compareTo); expectedList.sort(String::compareTo);
assertArrayEquals("Expected " + expectedColls.length + " collections, found " + cols.size() + ":\n" + assertArrayEquals("Expected " + expectedColls.length + " collections, found " + observedCols.size() + ":\n" +
cols + " vs \n" + actual, expectedColls, cols.toArray()); observedCols + " vs \n" + expectedList, expectedColls, observedCols.toArray());
assertEquals("Expected collections and alias to have same number of documents", assertEquals("Expected collections and alias to have same number of documents",
aliasNumFound, totalNumFound); numFoundViaAlias, numFoundViaCollections);
assertEquals("Expected to find " + expectNumFound + " docs but found " + aliasNumFound, assertEquals("Expected to find " + expectNumFound + " docs but found " + numFoundViaAlias,
expectNumFound, aliasNumFound); expectNumFound, numFoundViaAlias);
} }
private SolrInputDocument newDoc(String routedValue) { private SolrInputDocument newDoc(String routedValue) {