honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Changed p/c benchmarks to use random generated parent ids, index parent and its children in a non deterministic order and let query values match with multiple child / parent documents.

This commit is contained in:
Martijn van Groningen 2014-01-28 10:09:10 +01:00
parent 8706cf610a
commit 6aaddb03f1
3 changed files with 196 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchAndIndexingBenchmark.java
View File

@ -20,19 +20,15 @@ package org.elasticsearch.benchmark.search.child;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.StopWatch;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.SizeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.node.Node;
import java.io.IOException;
import java.util.Arrays;
import java.util.Random;
@ -40,7 +36,6 @@ import static org.elasticsearch.client.Requests.createIndexRequest;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.FilterBuilders.hasChildFilter;
import static org.elasticsearch.index.query.QueryBuilders.*;
import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
@ -50,9 +45,9 @@ import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
*/
public class ChildSearchAndIndexingBenchmark {
static long COUNT = SizeValue.parseSizeValue("1m").singles();
static int CHILD_COUNT = 5;
static int BATCH = 100;
static int PARENT_COUNT = (int) SizeValue.parseSizeValue("1m").singles();
static int NUM_CHILDREN_PER_PARENT = 12;
static int QUERY_VALUE_RATIO_PER_PARENT = 3;
static int QUERY_COUNT = 50;
static String indexName = "test";
static Random random = new Random();
@ -79,35 +74,11 @@ public class ChildSearchAndIndexingBenchmark {
.endObject().endObject()).execute().actionGet();
Thread.sleep(5000);
StopWatch stopWatch = new StopWatch().start();
System.out.println("--> Indexing [" + COUNT + "] parent document and [" + (COUNT * CHILD_COUNT) + " child documents");
long ITERS = COUNT / BATCH;
long i = 1;
int counter = 0;
for (; i <= ITERS; i++) {
BulkRequestBuilder request = client.prepareBulk();
for (int j = 0; j < BATCH; j++) {
counter++;
request.add(Requests.indexRequest(indexName).type("parent").id(Integer.toString(counter))
.source(parentSource(Integer.toString(counter), "test" + counter)));
for (int k = 0; k < CHILD_COUNT; k++) {
request.add(Requests.indexRequest(indexName).type("child").id(Integer.toString(counter) + "_" + k)
.parent(Integer.toString(counter))
.source(childSource(Integer.toString(counter), "tag" + k)));
}
}
BulkResponse response = request.execute().actionGet();
if (response.hasFailures()) {
System.err.println("--> failures...");
}
if (((i * BATCH) % 10000) == 0) {
System.out.println("--> Indexed " + (i * BATCH) * (1 + CHILD_COUNT) + " took " + stopWatch.stop().lastTaskTime());
stopWatch.start();
}
}
System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (COUNT * (1 + CHILD_COUNT))) / stopWatch.totalTime().secondsFrac()));
} catch (Exception e) {
long startTime = System.currentTimeMillis();
ParentChildIndexGenerator generator = new ParentChildIndexGenerator(client, PARENT_COUNT, NUM_CHILDREN_PER_PARENT, QUERY_VALUE_RATIO_PER_PARENT);
generator.index();
System.out.println("--> Indexing took " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds.");
} catch (IndexAlreadyExistsException e) {
System.out.println("--> Index already exists, ignoring indexing phase, waiting for green");
ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10m").execute().actionGet();
if (clusterHealthResponse.isTimedOut()) {
@ -130,14 +101,6 @@ public class ChildSearchAndIndexingBenchmark {
node1.close();
}
private static XContentBuilder parentSource(String id, String nameValue) throws IOException {
return jsonBuilder().startObject().field("id", id).field("name", nameValue).endObject();
}
private static XContentBuilder childSource(String id, String tag) throws IOException {
return jsonBuilder().startObject().field("id", id).field("tag", tag).endObject();
}
static class IndexThread implements Runnable {
private final Client client;
@ -150,18 +113,23 @@ public class ChildSearchAndIndexingBenchmark {
@Override
public void run() {
while (run) {
for (int i = 1; run && i < COUNT; i++) {
int childIdLimit = PARENT_COUNT * NUM_CHILDREN_PER_PARENT;
for (int childId = 1; run && childId < childIdLimit;) {
try {
client.prepareIndex(indexName, "parent", Integer.toString(i))
.setSource(parentSource(Integer.toString(i), "test" + i)).execute().actionGet();
for (int j = 0; j < CHILD_COUNT; j++) {
client.prepareIndex(indexName, "child", Integer.toString(i) + "_" + j)
.setParent(Integer.toString(i))
.setSource(childSource(Integer.toString(j), "tag" + j)).execute().actionGet();
for (int j = 0; j < 8; j++) {
GetResponse getResponse = client
.prepareGet(indexName, "child", String.valueOf(++childId))
.setFields("_source", "_parent")
.setRouting("1") // Doesn't matter what value, since there is only one shard
.get();
client.prepareIndex(indexName, "child", Integer.toString(childId) + "_" + j)
.setParent(getResponse.getField("_parent").getValue().toString())
.setSource(getResponse.getSource())
.get();
}
client.admin().indices().prepareRefresh(indexName).execute().actionGet();
Thread.sleep(100);
if (i % 500 == 0) {
Thread.sleep(1000);
if (childId % 500 == 0) {
NodesStatsResponse statsResponse = client.admin().cluster().prepareNodesStats()
.clear().setIndices(true).execute().actionGet();
System.out.println("Deleted docs: " + statsResponse.getAt(0).getIndices().getDocs().getDeleted());
@ -182,10 +150,12 @@ public class ChildSearchAndIndexingBenchmark {
static class SearchThread implements Runnable {
private final Client client;
private final int numValues;
private volatile boolean run = true;
SearchThread(Client client) {
this.client = client;
this.numValues = NUM_CHILDREN_PER_PARENT / NUM_CHILDREN_PER_PARENT;
}
@Override
@ -198,16 +168,13 @@ public class ChildSearchAndIndexingBenchmark {
.setQuery(
filteredQuery(
matchAllQuery(),
hasChildFilter("child", termQuery("tag", "tag" + random.nextInt(CHILD_COUNT)))
hasChildFilter("child", termQuery("field2", "value" + random.nextInt(numValues)))
)
)
.execute().actionGet();
if (searchResponse.getFailedShards() > 0) {
System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures()));
}
if (searchResponse.getHits().totalHits() != COUNT) {
// System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "], expected [" + COUNT + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> has_child filter with term filter Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
@ -225,10 +192,6 @@ public class ChildSearchAndIndexingBenchmark {
if (searchResponse.getFailedShards() > 0) {
System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures()));
}
long expected = (COUNT / BATCH) * BATCH;
if (searchResponse.getHits().totalHits() != expected) {
// System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "], expected [" + expected + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> has_child filter with match_all child query, Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");

218
src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java
View File

@ -20,26 +20,20 @@ package org.elasticsearch.benchmark.search.child;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.StopWatch;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.SizeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.node.Node;
import java.io.IOException;
import java.util.Arrays;
import static org.elasticsearch.client.Requests.createIndexRequest;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.FilterBuilders.*;
import static org.elasticsearch.index.query.QueryBuilders.*;
import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
@ -62,13 +56,14 @@ public class ChildSearchBenchmark {
.settings(settingsBuilder().put(settings).put("name", "node1")).node();
Client client = node1.client();
long COUNT = SizeValue.parseSizeValue("10m").singles();
int CHILD_COUNT = 5;
int BATCH = 100;
int QUERY_WARMUP = 20;
int QUERY_COUNT = 50;
int COUNT = (int) SizeValue.parseSizeValue("2m").singles();
int CHILD_COUNT = 15;
int QUERY_VALUE_RATIO = 3;
int QUERY_WARMUP = 10;
int QUERY_COUNT = 20;
String indexName = "test";
ParentChildIndexGenerator parentChildIndexGenerator = new ParentChildIndexGenerator(client, COUNT, CHILD_COUNT, QUERY_VALUE_RATIO);
client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10s").execute().actionGet();
try {
client.admin().indices().create(createIndexRequest(indexName)).actionGet();
@ -76,36 +71,10 @@ public class ChildSearchBenchmark {
.startObject("_parent").field("type", "parent").endObject()
.endObject().endObject()).execute().actionGet();
Thread.sleep(5000);
StopWatch stopWatch = new StopWatch().start();
System.out.println("--> Indexing [" + COUNT + "] parent document and [" + (COUNT * CHILD_COUNT) + " child documents");
long ITERS = COUNT / BATCH;
long i = 1;
int counter = 0;
for (; i <= ITERS; i++) {
BulkRequestBuilder request = client.prepareBulk();
for (int j = 0; j < BATCH; j++) {
counter++;
request.add(Requests.indexRequest(indexName).type("parent").id(Integer.toString(counter))
.source(parentSource(counter, "test" + counter)));
for (int k = 0; k < CHILD_COUNT; k++) {
request.add(Requests.indexRequest(indexName).type("child").id(Integer.toString(counter) + "_" + k)
.parent(Integer.toString(counter))
.source(childSource(counter, "tag" + k)));
}
}
BulkResponse response = request.execute().actionGet();
if (response.hasFailures()) {
System.err.println("--> failures...");
}
if (((i * BATCH) % 10000) == 0) {
System.out.println("--> Indexed " + (i * BATCH) * (1 + CHILD_COUNT) + " took " + stopWatch.stop().lastTaskTime());
stopWatch.start();
}
}
System.out.println("--> Indexing took " + stopWatch.totalTime() + ", TPS " + (((double) (COUNT * (1 + CHILD_COUNT))) / stopWatch.totalTime().secondsFrac()));
} catch (Exception e) {
long startTime = System.currentTimeMillis();
parentChildIndexGenerator.index();
System.out.println("--> Indexing took " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds.");
} catch (IndexAlreadyExistsException e) {
System.out.println("--> Index already exists, ignoring indexing phase, waiting for green");
ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10m").execute().actionGet();
if (clusterHealthResponse.isTimedOut()) {
@ -118,21 +87,12 @@ public class ChildSearchBenchmark {
System.out.println("--> Running just child query");
// run just the child query, warm up first
for (int j = 0; j < QUERY_WARMUP; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(termQuery("child.tag", "tag1")).execute().actionGet();
if (j == 0) {
System.out.println("--> Warmup took: " + searchResponse.getTook());
}
if (searchResponse.getHits().totalHits() != COUNT) {
System.err.println("--> mismatch on hits");
}
client.prepareSearch(indexName).setQuery(termQuery("child.tag", "tag1")).execute().actionGet();
}
long totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(termQuery("child.tag", "tag1")).execute().actionGet();
if (searchResponse.getHits().totalHits() != COUNT) {
System.err.println("--> mismatch on hits");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> Just Child Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
@ -148,16 +108,13 @@ public class ChildSearchBenchmark {
.setQuery(
filteredQuery(
matchAllQuery(),
hasChildFilter("child", termQuery("tag", "tag1"))
hasChildFilter("child", termQuery("field2", parentChildIndexGenerator.getQueryValue()))
)
)
.execute().actionGet();
if (searchResponse.getFailedShards() > 0) {
System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures()));
}
if (searchResponse.getHits().totalHits() != COUNT) {
System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "], expected [" + COUNT + "]");
}
}
totalQueryTime = 0;
@ -166,15 +123,15 @@ public class ChildSearchBenchmark {
.setQuery(
filteredQuery(
matchAllQuery(),
hasChildFilter("child", termQuery("tag", "tag1"))
hasChildFilter("child", termQuery("field2", parentChildIndexGenerator.getQueryValue()))
)
)
.execute().actionGet();
if (searchResponse.getFailedShards() > 0) {
System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures()));
}
if (searchResponse.getHits().totalHits() != COUNT) {
System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "], expected [" + COUNT + "]");
if (j % 10 == 0) {
System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
@ -194,56 +151,26 @@ public class ChildSearchBenchmark {
if (searchResponse.getFailedShards() > 0) {
System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures()));
}
long expected = (COUNT / BATCH) * BATCH;
if (searchResponse.getHits().totalHits() != expected) {
System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "], expected [" + expected + "]");
if (j % 10 == 0) {
System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> has_child filter with match_all child query, Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(
filteredQuery(matchAllQuery(), hasChildFilter("child", termQuery("id", Integer.toString(j + 1))))
).execute().actionGet();
long expected = 1;
if (searchResponse.getHits().totalHits() != expected) {
System.err.println("mismatch on hits");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> has_child filter with single parent match Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
double expected = Math.pow((j + 1), 3) * CHILD_COUNT;
SearchResponse searchResponse = client.prepareSearch(indexName)
.setQuery(filteredQuery(matchAllQuery(), hasChildFilter("child", constantScoreQuery(rangeFilter("num").lte(expected)))))
.execute().actionGet();
if (searchResponse.getHits().totalHits() != expected) {
System.err.println("mismatch on hits: " + searchResponse.getHits().totalHits() + " != " + expected);
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> has_child filter with exponential parent results Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
// run parent child constant query
for (int j = 0; j < QUERY_WARMUP; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName)
.setQuery(
filteredQuery(
matchAllQuery(),
hasParentFilter("parent", termQuery("name", "test1"))
hasParentFilter("parent", termQuery("field1", parentChildIndexGenerator.getQueryValue()))
)
)
.execute().actionGet();
if (searchResponse.getFailedShards() > 0) {
System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures()));
}
if (searchResponse.getHits().totalHits() != CHILD_COUNT) {
System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "], expected [" + CHILD_COUNT + "]");
}
}
totalQueryTime = 0;
@ -252,15 +179,15 @@ public class ChildSearchBenchmark {
.setQuery(
filteredQuery(
matchAllQuery(),
hasParentFilter("parent", termQuery("name", "test1"))
hasParentFilter("parent", termQuery("field1", parentChildIndexGenerator.getQueryValue()))
)
)
.execute().actionGet();
if (searchResponse.getFailedShards() > 0) {
System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures()));
}
if (searchResponse.getHits().totalHits() != CHILD_COUNT) {
System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "], expected [" + CHILD_COUNT + "]");
if (j % 10 == 0) {
System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
@ -278,8 +205,8 @@ public class ChildSearchBenchmark {
if (searchResponse.getFailedShards() > 0) {
System.err.println("Search Failures " + Arrays.toString(searchResponse.getShardFailures()));
}
if (searchResponse.getHits().totalHits() != 5000000) {
System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "], expected [" + 5000000 + "]");
if (j % 10 == 0) {
System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
@ -287,20 +214,15 @@ public class ChildSearchBenchmark {
System.out.println("--> Running top_children query");
// run parent child score query
for (int j = 0; j < QUERY_WARMUP; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(topChildrenQuery("child", termQuery("tag", "tag1"))).execute().actionGet();
// we expect to have mismatch on hits here
// if (searchResponse.hits().totalHits() != COUNT) {
// System.err.println("mismatch on hits");
// }
client.prepareSearch(indexName).setQuery(topChildrenQuery("child", termQuery("field2", parentChildIndexGenerator.getQueryValue()))).execute().actionGet();
}
totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(topChildrenQuery("child", termQuery("tag", "tag1"))).execute().actionGet();
// we expect to have mismatch on hits here
// if (searchResponse.hits().totalHits() != COUNT) {
// System.err.println("mismatch on hits");
// }
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(topChildrenQuery("child", termQuery("field2", parentChildIndexGenerator.getQueryValue()))).execute().actionGet();
if (j % 10 == 0) {
System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> top_children Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
@ -308,20 +230,15 @@ public class ChildSearchBenchmark {
System.out.println("--> Running top_children query, with match_all as child query");
// run parent child score query
for (int j = 0; j < QUERY_WARMUP; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(topChildrenQuery("child", matchAllQuery())).execute().actionGet();
// we expect to have mismatch on hits here
// if (searchResponse.hits().totalHits() != COUNT) {
// System.err.println("mismatch on hits");
// }
client.prepareSearch(indexName).setQuery(topChildrenQuery("child", matchAllQuery())).execute().actionGet();
}
totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(topChildrenQuery("child", matchAllQuery())).execute().actionGet();
// we expect to have mismatch on hits here
// if (searchResponse.hits().totalHits() != COUNT) {
// System.err.println("mismatch on hits");
// }
if (j % 10 == 0) {
System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> top_children, with match_all Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
@ -335,17 +252,14 @@ public class ChildSearchBenchmark {
System.out.println("--> Running has_child query with score type");
// run parent child score query
for (int j = 0; j < QUERY_WARMUP; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasChildQuery("child", termQuery("tag", "tag1")).scoreType("max")).execute().actionGet();
if (searchResponse.getHits().totalHits() != COUNT) {
System.err.println("mismatch on hits");
}
client.prepareSearch(indexName).setQuery(hasChildQuery("child", termQuery("field2", parentChildIndexGenerator.getQueryValue())).scoreType("max")).execute().actionGet();
}
totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasChildQuery("child", termQuery("tag", "tag1")).scoreType("max")).execute().actionGet();
if (searchResponse.getHits().totalHits() != COUNT) {
System.err.println("mismatch on hits");
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasChildQuery("child", termQuery("field2", parentChildIndexGenerator.getQueryValue())).scoreType("max")).execute().actionGet();
if (j % 10 == 0) {
System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
@ -354,64 +268,38 @@ public class ChildSearchBenchmark {
totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasChildQuery("child", matchAllQuery()).scoreType("max")).execute().actionGet();
long expected = (COUNT / BATCH) * BATCH;
if (searchResponse.getHits().totalHits() != expected) {
System.err.println("mismatch on hits");
if (j % 10 == 0) {
System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> has_child query with match_all Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasChildQuery("child", termQuery("id", Integer.toString(j + 1))).scoreType("max")).execute().actionGet();
long expected = 1;
if (searchResponse.getHits().totalHits() != expected) {
System.err.println("mismatch on hits");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> has_child query with single parent match Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
double expected = Math.pow((j + 1), 3) * CHILD_COUNT;
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasChildQuery("child", constantScoreQuery(rangeFilter("num").lte(expected))).scoreType("max")).execute().actionGet();
if (searchResponse.getHits().totalHits() != expected) {
System.err.println("mismatch on hits: " + searchResponse.getHits().totalHits() + " != " + expected);
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> has_child query with exponential parent results Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
/*System.out.println("--> Running has_parent query with score type");
System.out.println("--> Running has_parent query with score type");
// run parent child score query
for (int j = 0; j < QUERY_WARMUP; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasParentQuery("parent", termQuery("name", "test1")).scoreType("score")).execute().actionGet();
if (searchResponse.getHits().totalHits() != CHILD_COUNT) {
System.err.println("mismatch on hits");
}
client.prepareSearch(indexName).setQuery(hasParentQuery("parent", termQuery("field1", parentChildIndexGenerator.getQueryValue())).scoreType("score")).execute().actionGet();
}
totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasParentQuery("parent", termQuery("name", "test1")).scoreType("score")).execute().actionGet();
if (searchResponse.getHits().totalHits() != CHILD_COUNT) {
System.err.println("mismatch on hits");
for (int j = 1; j < QUERY_COUNT; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasParentQuery("parent", termQuery("field1", parentChildIndexGenerator.getQueryValue())).scoreType("score")).execute().actionGet();
if (j % 10 == 0) {
System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> has_parent Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
totalQueryTime = 0;
for (int j = 0; j < QUERY_COUNT; j++) {
for (int j = 1; j < QUERY_COUNT; j++) {
SearchResponse searchResponse = client.prepareSearch(indexName).setQuery(hasParentQuery("parent", matchAllQuery()).scoreType("score")).execute().actionGet();
if (searchResponse.getHits().totalHits() != 5000000) {
System.err.println("mismatch on hits");
if (j % 10 == 0) {
System.out.println("--> hits [" + j + "], got [" + searchResponse.getHits().totalHits() + "]");
}
totalQueryTime += searchResponse.getTookInMillis();
}
System.out.println("--> has_parent query with match_all Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");*/
System.out.println("--> has_parent query with match_all Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
System.gc();
statsResponse = client.admin().cluster().prepareNodesStats()
@ -423,12 +311,4 @@ public class ChildSearchBenchmark {
client.close();
node1.close();
}
private static XContentBuilder parentSource(int id, String nameValue) throws IOException {
return jsonBuilder().startObject().field("id", Integer.toString(id)).field("num", id).field("name", nameValue).endObject();
}
private static XContentBuilder childSource(int id, String tag) throws IOException {
return jsonBuilder().startObject().field("id", Integer.toString(id)).field("num", id).field("tag", tag).endObject();
}
}

120
src/test/java/org/elasticsearch/benchmark/search/child/ParentChildIndexGenerator.java Normal file
View File

@ -0,0 +1,120 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.benchmark.search.child;
import com.carrotsearch.hppc.ObjectArrayList;
import com.carrotsearch.hppc.ObjectOpenHashSet;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import java.util.Random;
/**
*/
public class ParentChildIndexGenerator {
private final static Random RANDOM = new Random();
private final Client client;
private final int numParents;
private final int numChildrenPerParent;
private final int queryValueRatio;
public ParentChildIndexGenerator(Client client, int numParents, int numChildrenPerParent, int queryValueRatio) {
this.client = client;
this.numParents = numParents;
this.numChildrenPerParent = numChildrenPerParent;
this.queryValueRatio = queryValueRatio;
}
public void index() {
// Memory intensive...
ObjectOpenHashSet<String> usedParentIds = ObjectOpenHashSet.newInstanceWithCapacity(numParents, 0.5f);
ObjectArrayList<ParentDocument> parents = ObjectArrayList.newInstanceWithCapacity(numParents);
for (int i = 0; i < numParents; i++) {
String parentId;
do {
parentId = RandomStrings.randomAsciiOfLength(RANDOM, 10);
} while (!usedParentIds.add(parentId));
String[] queryValues = new String[numChildrenPerParent];
for (int j = 0; j < numChildrenPerParent; j++) {
queryValues[j] = getQueryValue();
}
parents.add(new ParentDocument(parentId, queryValues));
}
int indexCounter = 0;
int childIdCounter = 0;
while (!parents.isEmpty()) {
BulkRequestBuilder request = client.prepareBulk();
for (int i = 0; !parents.isEmpty() && i < 100; i++) {
int index = RANDOM.nextInt(parents.size());
ParentDocument parentDocument = parents.get(index);
if (parentDocument.indexCounter == -1) {
request.add(Requests.indexRequest("test").type("parent")
.id(parentDocument.parentId)
.source("field1", getQueryValue()));
} else {
request.add(Requests.indexRequest("test").type("child")
.parent(parentDocument.parentId)
.id(String.valueOf(++childIdCounter))
.source("field2", parentDocument.queryValues[parentDocument.indexCounter]));
}
if (++parentDocument.indexCounter == parentDocument.queryValues.length) {
parents.remove(index);
}
}
BulkResponse response = request.execute().actionGet();
if (response.hasFailures()) {
System.err.println("--> failures...");
}
indexCounter += response.getItems().length;
if (indexCounter % 100000 == 0) {
System.out.println("--> Indexed " + indexCounter + " documents");
}
}
}
public String getQueryValue() {
return "value" + RANDOM.nextInt(numChildrenPerParent / queryValueRatio);
}
class ParentDocument {
final String parentId;
final String[] queryValues;
int indexCounter;
ParentDocument(String parentId, String[] queryValues) {
this.parentId = parentId;
this.queryValues = queryValues;
this.indexCounter = -1;
}
}
}