honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Facet query crashes the cluster. Wrong serialzation of facets caused for construction of a rough sized array list. closes #28.

This commit is contained in:
kimchy 2010-02-20 15:21:02 +02:00
parent fa54b7c896
commit 008b00f51a
3 changed files with 46 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/elasticsearch/src/main/java/org/elasticsearch/search/facets/Facets.java
View File

@ -101,7 +101,7 @@ public class Facets implements Streamable, ToJson {
} }
@Override public void writeTo(DataOutput out) throws IOException { @Override public void writeTo(DataOutput out) throws IOException {
out.write(facets.size()); out.writeInt(facets.size());
for (Facet facet : facets) { for (Facet facet : facets) {
out.write(facet.type().id()); out.write(facet.type().id());
facet.writeTo(out); facet.writeTo(out);

209
modules/test/integration/src/test/java/org/elasticsearch/test/integration/client/transport/ClientTransportTwoServersSearchTests.java
View File

@ -19,219 +19,22 @@
package org.elasticsearch.test.integration.client.transport; package org.elasticsearch.test.integration.client.transport;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.server.internal.InternalServer; import org.elasticsearch.server.internal.InternalServer;
import org.elasticsearch.test.integration.AbstractServersTests; import org.elasticsearch.test.integration.search.TransportTwoServersSearchTests;
import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportService;
import org.elasticsearch.util.transport.TransportAddress; import org.elasticsearch.util.transport.TransportAddress;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import static org.elasticsearch.action.search.SearchType.*;
import static org.elasticsearch.client.Requests.*;
import static org.elasticsearch.index.query.json.JsonQueryBuilders.*;
import static org.elasticsearch.search.builder.SearchSourceBuilder.*;
import static org.elasticsearch.util.TimeValue.*;
import static org.hamcrest.MatcherAssert.*;
import static org.hamcrest.Matchers.*;
/** /**
* @author kimchy (Shay Banon) * @author kimchy (shay.banon)
*/ */
public class ClientTransportTwoServersSearchTests extends AbstractServersTests { public class ClientTransportTwoServersSearchTests extends TransportTwoServersSearchTests {
private TransportClient client;
@BeforeClass public void createServers() throws Exception {
startServer("server1");
startServer("server2");
@Override protected Client getClient() {
TransportAddress server1Address = ((InternalServer) server("server1")).injector().getInstance(TransportService.class).boundAddress().publishAddress(); TransportAddress server1Address = ((InternalServer) server("server1")).injector().getInstance(TransportService.class).boundAddress().publishAddress();
client = new TransportClient(); TransportClient client = new TransportClient();
client.addTransportAddress(server1Address); client.addTransportAddress(server1Address);
return client;
client.admin().indices().create(createIndexRequest("test")).actionGet();
for (int i = 0; i < 100; i++) {
index(client, Integer.toString(i), "test", i);
}
client.admin().indices().refresh(refreshRequest("test")).actionGet();
}
@AfterClass public void closeServers() {
closeAllServers();
if (client != null) {
client.close();
}
}
@Test public void testDfsQueryThenFetch() throws Exception {
SearchSourceBuilder source = searchSource()
.query(termQuery("multi", "test"))
.from(0).size(60).explain(true);
SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(DFS_QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60));
for (int i = 0; i < 60; i++) {
SearchHit hit = searchResponse.hits().hits()[i];
// System.out.println(hit.target() + ": " + hit.explanation());
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
}
searchResponse = client.searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(40));
for (int i = 0; i < 40; i++) {
SearchHit hit = searchResponse.hits().hits()[i];
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - 60 - 1 - i)));
}
}
//
@Test public void testDfsQueryThenFetchWithSort() throws Exception {
SearchSourceBuilder source = searchSource()
.query(termQuery("multi", "test"))
.from(0).size(60).explain(true).sort("age", false);
SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(DFS_QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60));
for (int i = 0; i < 60; i++) {
SearchHit hit = searchResponse.hits().hits()[i];
// System.out.println(hit.target() + ": " + hit.explanation());
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(i)));
}
searchResponse = client.searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(40));
for (int i = 0; i < 40; i++) {
SearchHit hit = searchResponse.hits().hits()[i];
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(i + 60)));
}
}
@Test public void testQueryThenFetch() throws Exception {
SearchSourceBuilder source = searchSource()
.query(termQuery("multi", "test"))
.from(0).size(60).explain(true);
SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60));
for (int i = 0; i < 60; i++) {
SearchHit hit = searchResponse.hits().hits()[i];
// System.out.println(hit.target() + ": " + hit.explanation());
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
}
searchResponse = client.searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(40));
for (int i = 0; i < 40; i++) {
SearchHit hit = searchResponse.hits().hits()[i];
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - 60 - 1 - i)));
}
}
@Test public void testQueryThenFetchWithSort() throws Exception {
SearchSourceBuilder source = searchSource()
.query(termQuery("multi", "test"))
.from(0).size(60).explain(true).sort("age", false);
SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60));
for (int i = 0; i < 60; i++) {
SearchHit hit = searchResponse.hits().hits()[i];
// System.out.println(hit.target() + ": " + hit.explanation());
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(i)));
}
searchResponse = client.searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(40));
for (int i = 0; i < 40; i++) {
SearchHit hit = searchResponse.hits().hits()[i];
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(i + 60)));
}
}
@Test public void testQueryAndFetch() throws Exception {
SearchSourceBuilder source = searchSource()
.query(termQuery("multi", "test"))
.from(0).size(20).explain(true);
SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(QUERY_AND_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60)); // 20 per shard
for (int i = 0; i < 60; i++) {
SearchHit hit = searchResponse.hits().hits()[i];
// System.out.println(hit.target() + ": " + hit.explanation());
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
}
// TODO support scrolling
// searchResponse = searchScrollAction.submit(new SearchScrollRequest(searchResponse.scrollId())).actionGet();
//
// assertEquals(100, searchResponse.hits().totalHits());
// assertEquals(40, searchResponse.hits().hits().length);
// for (int i = 0; i < 40; i++) {
// SearchHit hit = searchResponse.hits().hits()[i];
// assertEquals("id[" + hit.id() + "]", Integer.toString(100 - 60 - 1 - i), hit.id());
// }
}
@Test public void testDfsQueryAndFetch() throws Exception {
SearchSourceBuilder source = searchSource()
.query(termQuery("multi", "test"))
.from(0).size(20).explain(true);
SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(DFS_QUERY_AND_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60)); // 20 per shard
for (int i = 0; i < 60; i++) {
SearchHit hit = searchResponse.hits().hits()[i];
// System.out.println(hit.target() + ": " + hit.explanation());
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
}
// TODO support scrolling
// searchResponse = searchScrollAction.submit(new SearchScrollRequest(searchResponse.scrollId())).actionGet();
//
// assertEquals(100, searchResponse.hits().totalHits());
// assertEquals(40, searchResponse.hits().hits().length);
// for (int i = 0; i < 40; i++) {
// SearchHit hit = searchResponse.hits().hits()[i];
// assertEquals("id[" + hit.id() + "]", Integer.toString(100 - 60 - 1 - i), hit.id());
// }
}
private void index(Client client, String id, String nameValue, int age) {
client.index(Requests.indexRequest("test").type("type1").id(id).source(source(id, nameValue, age))).actionGet();
}
private String source(String id, String nameValue, int age) {
StringBuilder multi = new StringBuilder().append(nameValue);
for (int i = 0; i < age; i++) {
multi.append(" ").append(nameValue);
}
return "{ type1 : { \"id\" : \"" + id + "\", \"name\" : \"" + (nameValue + id) + "\", age : " + age + ", multi : \"" + multi.toString() + "\", _boost : " + (age * 10) + " } }";
} }
} }

50
modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/TransportTwoServersSearchTests.java
View File

@ -43,28 +43,36 @@ import static org.hamcrest.Matchers.*;
*/ */
public class TransportTwoServersSearchTests extends AbstractServersTests { public class TransportTwoServersSearchTests extends AbstractServersTests {
private Client client;
@BeforeClass public void createServers() throws Exception { @BeforeClass public void createServers() throws Exception {
startServer("server1"); startServer("server1");
startServer("server2"); startServer("server2");
client = getClient();
client("server1").admin().indices().create(createIndexRequest("test")).actionGet(); client.admin().indices().create(createIndexRequest("test")).actionGet();
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
index(client("server1"), Integer.toString(i), "test", i); index(client("server1"), Integer.toString(i), "test", i);
} }
client("server1").admin().indices().refresh(refreshRequest("test")).actionGet(); client.admin().indices().refresh(refreshRequest("test")).actionGet();
} }
@AfterClass public void closeServers() { @AfterClass public void closeServers() {
client.close();
closeAllServers(); closeAllServers();
} }
protected Client getClient() {
return client("server1");
}
@Test public void testDfsQueryThenFetch() throws Exception { @Test public void testDfsQueryThenFetch() throws Exception {
SearchSourceBuilder source = searchSource() SearchSourceBuilder source = searchSource()
.query(termQuery("multi", "test")) .query(termQuery("multi", "test"))
.from(0).size(60).explain(true); .from(0).size(60).explain(true);
SearchResponse searchResponse = client("server1").search(searchRequest("test").source(source).searchType(DFS_QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet(); SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(DFS_QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l)); assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60)); assertThat(searchResponse.hits().hits().length, equalTo(60));
@ -74,7 +82,7 @@ public class TransportTwoServersSearchTests extends AbstractServersTests {
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1))); assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
} }
searchResponse = client("server1").searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet(); searchResponse = client.searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l)); assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(40)); assertThat(searchResponse.hits().hits().length, equalTo(40));
@ -91,7 +99,7 @@ public class TransportTwoServersSearchTests extends AbstractServersTests {
.query(termQuery("multi", "test")) .query(termQuery("multi", "test"))
.from(0).size(60).explain(true).sort("age", false); .from(0).size(60).explain(true).sort("age", false);
SearchResponse searchResponse = client("server1").search(searchRequest("test").source(source).searchType(DFS_QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet(); SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(DFS_QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l)); assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60)); assertThat(searchResponse.hits().hits().length, equalTo(60));
for (int i = 0; i < 60; i++) { for (int i = 0; i < 60; i++) {
@ -100,7 +108,7 @@ public class TransportTwoServersSearchTests extends AbstractServersTests {
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(i))); assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(i)));
} }
searchResponse = client("server1").searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet(); searchResponse = client.searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l)); assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(40)); assertThat(searchResponse.hits().hits().length, equalTo(40));
@ -115,7 +123,7 @@ public class TransportTwoServersSearchTests extends AbstractServersTests {
.query(termQuery("multi", "test")) .query(termQuery("multi", "test"))
.from(0).size(60).explain(true); .from(0).size(60).explain(true);
SearchResponse searchResponse = client("server1").search(searchRequest("test").source(source).searchType(QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet(); SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l)); assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60)); assertThat(searchResponse.hits().hits().length, equalTo(60));
for (int i = 0; i < 60; i++) { for (int i = 0; i < 60; i++) {
@ -124,7 +132,7 @@ public class TransportTwoServersSearchTests extends AbstractServersTests {
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1))); assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
} }
searchResponse = client("server1").searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet(); searchResponse = client.searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l)); assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(40)); assertThat(searchResponse.hits().hits().length, equalTo(40));
@ -139,7 +147,7 @@ public class TransportTwoServersSearchTests extends AbstractServersTests {
.query(termQuery("multi", "test")) .query(termQuery("multi", "test"))
.from(0).size(60).explain(true).sort("age", false); .from(0).size(60).explain(true).sort("age", false);
SearchResponse searchResponse = client("server1").search(searchRequest("test").source(source).searchType(QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet(); SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(QUERY_THEN_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l)); assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60)); assertThat(searchResponse.hits().hits().length, equalTo(60));
for (int i = 0; i < 60; i++) { for (int i = 0; i < 60; i++) {
@ -148,7 +156,7 @@ public class TransportTwoServersSearchTests extends AbstractServersTests {
assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(i))); assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(i)));
} }
searchResponse = client("server1").searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet(); searchResponse = client.searchScroll(searchScrollRequest(searchResponse.scrollId())).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l)); assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(40)); assertThat(searchResponse.hits().hits().length, equalTo(40));
@ -163,7 +171,7 @@ public class TransportTwoServersSearchTests extends AbstractServersTests {
.query(termQuery("multi", "test")) .query(termQuery("multi", "test"))
.from(0).size(20).explain(true); .from(0).size(20).explain(true);
SearchResponse searchResponse = client("server1").search(searchRequest("test").source(source).searchType(QUERY_AND_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet(); SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(QUERY_AND_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l)); assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60)); // 20 per shard assertThat(searchResponse.hits().hits().length, equalTo(60)); // 20 per shard
for (int i = 0; i < 60; i++) { for (int i = 0; i < 60; i++) {
@ -188,7 +196,7 @@ public class TransportTwoServersSearchTests extends AbstractServersTests {
.query(termQuery("multi", "test")) .query(termQuery("multi", "test"))
.from(0).size(20).explain(true); .from(0).size(20).explain(true);
SearchResponse searchResponse = client("server1").search(searchRequest("test").source(source).searchType(DFS_QUERY_AND_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet(); SearchResponse searchResponse = client.search(searchRequest("test").source(source).searchType(DFS_QUERY_AND_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l)); assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.hits().hits().length, equalTo(60)); // 20 per shard assertThat(searchResponse.hits().hits().length, equalTo(60)); // 20 per shard
for (int i = 0; i < 60; i++) { for (int i = 0; i < 60; i++) {
@ -208,6 +216,24 @@ public class TransportTwoServersSearchTests extends AbstractServersTests {
// } // }
} }
@Test public void testSimpleFacets() throws Exception {
SearchSourceBuilder sourceBuilder = searchSource()
.query(termQuery("multi", "test"))
.from(0).size(20).explain(true)
.facets(facets().facet("all", termQuery("multi", "test")).facet("test1", termQuery("name", "test1")));
SearchResponse searchResponse = client.search(searchRequest("test").source(sourceBuilder)).actionGet();
assertThat(searchResponse.hits().totalHits(), equalTo(100l));
assertThat(searchResponse.facets().countFacet("test1").count(), equalTo(1l));
assertThat(searchResponse.facets().countFacet("all").count(), equalTo(100l));
}
@Test public void testSimpleFacetsTwice() throws Exception {
testSimpleFacets();
testSimpleFacets();
}
private void index(Client client, String id, String nameValue, int age) { private void index(Client client, String id, String nameValue, int age) {
client.index(Requests.indexRequest("test").type("type1").id(id).source(source(id, nameValue, age))).actionGet(); client.index(Requests.indexRequest("test").type("type1").id(id).source(source(id, nameValue, age))).actionGet();