From 4decc72da6abc426b97789ad149b31eef9aad583 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sat, 6 Feb 2016 13:40:38 +0100 Subject: [PATCH 01/14] Fix recovery translog stats totals when recovering from store Recovery from store fails to correctly set the translog recovery stats. This fixes it and tightens up the logic bringing it all to IndexShard (previously it was set by the recovery logic). Closes #15974 Closes #16493 --- .../cluster/routing/ShardRouting.java | 2 +- .../index/engine/InternalEngine.java | 17 +------- .../elasticsearch/index/shard/IndexShard.java | 21 ++++++++-- .../index/shard/StoreRecovery.java | 5 --- .../shard/TranslogRecoveryPerformer.java | 21 ++++++++++ .../cluster/routing/ShardRoutingHelper.java | 5 +++ .../index/shard/IndexShardTests.java | 41 +++++++++++++++++-- 7 files changed, 84 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index 7aaf6969948..7535aa1226e 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -451,7 +451,7 @@ public final class ShardRouting implements Streamable, ToXContent { } /** - * Moves the shard from started to initializing and bumps the version + * Moves the shard from started to initializing */ void reinitializeShard() { ensureNotFrozen(); diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index d9ee2f4177a..ff738c0140b 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -43,7 +43,6 @@ import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.InfoStream; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.Nullable; @@ -68,7 +67,6 @@ import org.elasticsearch.index.shard.TranslogRecoveryPerformer; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; import org.elasticsearch.index.translog.TranslogCorruptedException; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -233,20 +231,7 @@ public class InternalEngine extends Engine { final TranslogRecoveryPerformer handler = engineConfig.getTranslogRecoveryPerformer(); try { Translog.Snapshot snapshot = translog.newSnapshot(); - Translog.Operation operation; - while ((operation = snapshot.next()) != null) { - try { - handler.performRecoveryOperation(this, operation, true); - opsRecovered++; - } catch (ElasticsearchException e) { - if (e.status() == RestStatus.BAD_REQUEST) { - // mainly for MapperParsingException and Failure to detect xcontent - logger.info("ignoring recovery of a corrupt translog entry", e); - } else { - throw e; - } - } - } + opsRecovered = handler.recoveryFromSnapshot(this, snapshot); } catch (Throwable e) { throw new EngineException(shardId, "failed to recover from translog", e); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 705d4d5aa5a..bbd00951277 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -55,6 +55,7 @@ import org.elasticsearch.gateway.MetaDataStateFormat; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.NodeServicesProvider; +import org.elasticsearch.index.SearchSlowLog; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.cache.IndexCache; import org.elasticsearch.index.cache.bitset.ShardBitsetFilterCache; @@ -89,13 +90,12 @@ import org.elasticsearch.index.percolator.PercolatorQueriesRegistry; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.recovery.RecoveryStats; import org.elasticsearch.index.refresh.RefreshStats; -import org.elasticsearch.index.SearchSlowLog; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.search.stats.ShardSearchStats; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.snapshots.IndexShardRepository; -import org.elasticsearch.index.store.Store.MetadataSnapshot; import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.Store.MetadataSnapshot; import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.suggest.stats.ShardSuggestMetric; @@ -105,8 +105,8 @@ import org.elasticsearch.index.translog.TranslogConfig; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.index.warmer.ShardIndexWarmerService; import org.elasticsearch.index.warmer.WarmerStats; -import org.elasticsearch.indices.cache.query.IndicesQueryCache; import org.elasticsearch.indices.IndexingMemoryController; +import org.elasticsearch.indices.cache.query.IndicesQueryCache; import org.elasticsearch.indices.recovery.RecoveryFailedException; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.percolator.PercolatorService; @@ -874,6 +874,12 @@ public class IndexShard extends AbstractIndexShardComponent { * After the store has been recovered, we need to start the engine in order to apply operations */ public void performTranslogRecovery(boolean indexExists) { + if (indexExists == false) { + // note: these are set when recovering from the translog + final RecoveryState.Translog translogStats = recoveryState().getTranslog(); + translogStats.totalOperations(0); + translogStats.totalOperationsOnStart(0); + } internalPerformTranslogRecovery(false, indexExists); assert recoveryState.getStage() == RecoveryState.Stage.TRANSLOG : "TRANSLOG stage expected but was: " + recoveryState.getStage(); } @@ -1387,6 +1393,15 @@ public class IndexShard extends AbstractIndexShardComponent { assert recoveryState != null; recoveryState.getTranslog().incrementRecoveredOperations(); } + + @Override + public int recoveryFromSnapshot(Engine engine, Translog.Snapshot snapshot) throws IOException { + assert recoveryState != null; + RecoveryState.Translog translogStats = recoveryState.getTranslog(); + translogStats.totalOperations(snapshot.totalOperations()); + translogStats.totalOperationsOnStart(snapshot.totalOperations()); + return super.recoveryFromSnapshot(engine, snapshot); + } }; return new EngineConfig(shardId, threadPool, indexSettings, warmer, store, deletionPolicy, indexSettings.getMergePolicy(), diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index 5f5aa95a994..aaa30c147c0 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -203,7 +203,6 @@ final class StoreRecovery { logger.trace("cleaning existing shard, shouldn't exists"); IndexWriter writer = new IndexWriter(store.directory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setOpenMode(IndexWriterConfig.OpenMode.CREATE)); writer.close(); - recoveryState.getTranslog().totalOperations(0); } } } catch (Throwable e) { @@ -224,10 +223,6 @@ final class StoreRecovery { } catch (IOException e) { logger.debug("failed to list file details", e); } - if (indexShouldExists == false) { - recoveryState.getTranslog().totalOperations(0); - recoveryState.getTranslog().totalOperationsOnStart(0); - } indexShard.performTranslogRecovery(indexShouldExists); indexShard.finalizeRecovery(); indexShard.postRecovery("post recovery from shard_store"); diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java index 4811ff1a275..f3f8f3c14cf 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java +++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.rest.RestStatus; import java.io.IOException; import java.util.HashMap; @@ -77,6 +78,25 @@ public class TranslogRecoveryPerformer { return numOps; } + public int recoveryFromSnapshot(Engine engine, Translog.Snapshot snapshot) throws IOException { + Translog.Operation operation; + int opsRecovered = 0; + while ((operation = snapshot.next()) != null) { + try { + performRecoveryOperation(engine, operation, true); + opsRecovered++; + } catch (ElasticsearchException e) { + if (e.status() == RestStatus.BAD_REQUEST) { + // mainly for MapperParsingException and Failure to detect xcontent + logger.info("ignoring recovery of a corrupt translog entry", e); + } else { + throw e; + } + } + } + return opsRecovered; + } + public static class BatchOperationException extends ElasticsearchException { private final int completedOperations; @@ -182,6 +202,7 @@ public class TranslogRecoveryPerformer { // noop } + /** * Returns the recovered types modifying the mapping during the recovery */ diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java index fe7938f23b9..5d3466b5e43 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java @@ -48,6 +48,11 @@ public class ShardRoutingHelper { routing.reinitializeShard(); } + public static void reinit(ShardRouting routing, UnassignedInfo.Reason reason) { + routing.reinitializeShard(); + routing.updateUnassignedInfo(new UnassignedInfo(reason, "test_reinit")); + } + public static void moveToUnassigned(ShardRouting routing, UnassignedInfo info) { routing.moveToUnassigned(info); } diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index ec2f8f32a20..778831245d1 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -70,7 +70,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.NodeServicesProvider; @@ -865,10 +864,11 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndicesService indicesService = getInstanceFromNode(IndicesService.class); IndexService test = indicesService.indexService("test"); final IndexShard shard = test.getShardOrNull(0); - + int translogOps = 1; client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get(); if (randomBoolean()) { client().admin().indices().prepareFlush().get(); + translogOps = 0; } ShardRouting routing = new ShardRouting(shard.routingEntry()); test.removeShard(0, "b/c simon says so"); @@ -878,6 +878,10 @@ public class IndexShardTests extends ESSingleNodeTestCase { DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); assertTrue(newShard.recoverFromStore(localNode)); + assertEquals(translogOps, newShard.recoveryState().getTranslog().recoveredOperations()); + assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperations()); + assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperationsOnStart()); + assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f); routing = new ShardRouting(routing); ShardRoutingHelper.moveToStarted(routing); newShard.updateRoutingEntry(routing, true); @@ -885,6 +889,36 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertHitCount(response, 1); } + public void testRecoverFromCleanStore() throws IOException { + createIndex("test"); + ensureGreen(); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService test = indicesService.indexService("test"); + final IndexShard shard = test.getShardOrNull(0); + client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get(); + if (randomBoolean()) { + client().admin().indices().prepareFlush().get(); + } + ShardRouting routing = new ShardRouting(shard.routingEntry()); + test.removeShard(0, "b/c simon says so"); + ShardRoutingHelper.reinit(routing, UnassignedInfo.Reason.INDEX_CREATED); + IndexShard newShard = test.createShard(routing); + newShard.updateRoutingEntry(routing, false); + DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); + newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, + localNode)); + assertTrue(newShard.recoverFromStore(localNode)); + assertEquals(0, newShard.recoveryState().getTranslog().recoveredOperations()); + assertEquals(0, newShard.recoveryState().getTranslog().totalOperations()); + assertEquals(0, newShard.recoveryState().getTranslog().totalOperationsOnStart()); + assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f); + routing = new ShardRouting(routing); + ShardRoutingHelper.moveToStarted(routing); + newShard.updateRoutingEntry(routing, true); + SearchResponse response = client().prepareSearch().get(); + assertHitCount(response, 0); + } + public void testFailIfIndexNotPresentInRecoverFromStore() throws IOException { createIndex("test"); ensureGreen(); @@ -1187,7 +1221,8 @@ public class IndexShardTests extends ESSingleNodeTestCase { List operations = new ArrayList<>(); operations.add(new Translog.Index("testtype", "1", jsonBuilder().startObject().field("foo", "bar").endObject().bytes().toBytes())); newShard.prepareForIndexRecovery(); - newShard.performTranslogRecovery(true); + newShard.recoveryState().getTranslog().totalOperations(operations.size()); + newShard.skipTranslogRecovery(); newShard.performBatchRecovery(operations); assertFalse(newShard.getTranslog().syncNeeded()); } From 0b3cab6da3048d0e6f5aae474d4fe5bce7f3fa9b Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Mon, 8 Feb 2016 13:45:38 +0100 Subject: [PATCH 02/14] Move sorting tests w/o scripting back to core In preparation of #16127 this moves sorting tests back to core that don't actually need scripting or groovy to work. --- .../search/sort/FieldSortIT.java | 1526 +++++++++++++++++ .../messy/tests/SimpleSortTests.java | 1401 +-------------- 2 files changed, 1537 insertions(+), 1390 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java new file mode 100644 index 00000000000..2b3093563a3 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -0,0 +1,1526 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.sort; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Random; +import java.util.Set; +import java.util.TreeMap; +import java.util.Map.Entry; +import java.util.concurrent.ExecutionException; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.hamcrest.Matchers; + +public class FieldSortIT extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return pluginList(InternalSettingsPlugin.class); + } + + @LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elasticsearch/elasticsearch/issues/9421") + public void testIssue8226() { + int numIndices = between(5, 10); + final boolean useMapping = randomBoolean(); + for (int i = 0; i < numIndices; i++) { + if (useMapping) { + assertAcked(prepareCreate("test_" + i).addAlias(new Alias("test")).addMapping("foo", "entry", "type=long")); + } else { + assertAcked(prepareCreate("test_" + i).addAlias(new Alias("test"))); + } + if (i > 0) { + client().prepareIndex("test_" + i, "foo", "" + i).setSource("{\"entry\": " + i + "}").get(); + } + } + ensureYellow(); + refresh(); + // sort DESC + SearchResponse searchResponse = client().prepareSearch() + .addSort(new FieldSortBuilder("entry").order(SortOrder.DESC).unmappedType(useMapping ? null : "long")) + .setSize(10).get(); + logClusterState(); + assertSearchResponse(searchResponse); + + for (int j = 1; j < searchResponse.getHits().hits().length; j++) { + Number current = (Number) searchResponse.getHits().hits()[j].getSource().get("entry"); + Number previous = (Number) searchResponse.getHits().hits()[j-1].getSource().get("entry"); + assertThat(searchResponse.toString(), current.intValue(), lessThan(previous.intValue())); + } + + // sort ASC + searchResponse = client().prepareSearch() + .addSort(new FieldSortBuilder("entry").order(SortOrder.ASC).unmappedType(useMapping ? null : "long")) + .setSize(10).get(); + logClusterState(); + assertSearchResponse(searchResponse); + + for (int j = 1; j < searchResponse.getHits().hits().length; j++) { + Number current = (Number) searchResponse.getHits().hits()[j].getSource().get("entry"); + Number previous = (Number) searchResponse.getHits().hits()[j-1].getSource().get("entry"); + assertThat(searchResponse.toString(), current.intValue(), greaterThan(previous.intValue())); + } + } + + @LuceneTestCase.BadApple(bugUrl = "simon is working on this") + public void testIssue6614() throws ExecutionException, InterruptedException { + List builders = new ArrayList<>(); + boolean strictTimeBasedIndices = randomBoolean(); + final int numIndices = randomIntBetween(2, 25); // at most 25 days in the month + for (int i = 0; i < numIndices; i++) { + final String indexId = strictTimeBasedIndices ? "idx_" + i : "idx"; + if (strictTimeBasedIndices || i == 0) { + createIndex(indexId); + } + final int numDocs = randomIntBetween(1, 23); // hour of the day + for (int j = 0; j < numDocs; j++) { + builders.add( + client().prepareIndex(indexId, "type").setSource( + "foo", "bar", "timeUpdated", "2014/07/" + + String.format(Locale.ROOT, "%02d", i+1)+ + " " + + String.format(Locale.ROOT, "%02d", j+1) + + ":00:00")); + } + } + int docs = builders.size(); + indexRandom(true, builders); + ensureYellow(); + SearchResponse allDocsResponse = client().prepareSearch().setQuery( + QueryBuilders.boolQuery().must(QueryBuilders.termQuery("foo", "bar")).must( + QueryBuilders.rangeQuery("timeUpdated").gte("2014/0" + randomIntBetween(1, 7) + "/01"))) + .addSort(new FieldSortBuilder("timeUpdated").order(SortOrder.ASC).unmappedType("date")) + .setSize(docs).get(); + assertSearchResponse(allDocsResponse); + + final int numiters = randomIntBetween(1, 20); + for (int i = 0; i < numiters; i++) { + SearchResponse searchResponse = client().prepareSearch().setQuery( + QueryBuilders.boolQuery().must(QueryBuilders.termQuery("foo", "bar")).must( + QueryBuilders.rangeQuery("timeUpdated").gte( + "2014/" + String.format(Locale.ROOT, "%02d", randomIntBetween(1, 7)) + "/01"))) + .addSort(new FieldSortBuilder("timeUpdated").order(SortOrder.ASC).unmappedType("date")) + .setSize(scaledRandomIntBetween(1, docs)).get(); + assertSearchResponse(searchResponse); + for (int j = 0; j < searchResponse.getHits().hits().length; j++) { + assertThat(searchResponse.toString() + + "\n vs. \n" + + allDocsResponse.toString(), + searchResponse.getHits().hits()[j].getId(), + equalTo(allDocsResponse.getHits().hits()[j].getId())); + } + } + + } + + public void testIssue6639() throws ExecutionException, InterruptedException { + assertAcked(prepareCreate("$index") + .addMapping( + "$type", + "{\"$type\": " + + " {\"properties\": " + + " {\"grantee\": " + + " {\"index\": " + + " \"not_analyzed\", " + + " \"term_vector\": \"with_positions_offsets\", " + + " \"type\": \"string\", " + + " \"analyzer\": \"snowball\", " + + " \"boost\": 1.0, " + + " \"store\": true}}}}")); + indexRandom(true, + client().prepareIndex( + "$index", + "$type", + "data.activity.5").setSource("{\"django_ct\": \"data.activity\", \"grantee\": \"Grantee 1\"}"), + client().prepareIndex( + "$index", + "$type", + "data.activity.6").setSource("{\"django_ct\": \"data.activity\", \"grantee\": \"Grantee 2\"}")); + ensureYellow(); + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort("grantee", SortOrder.ASC) + .execute().actionGet(); + assertOrderedSearchHits(searchResponse, "data.activity.5", "data.activity.6"); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort("grantee", SortOrder.DESC) + .execute().actionGet(); + assertOrderedSearchHits(searchResponse, "data.activity.6", "data.activity.5"); + } + + public void testTrackScores() throws Exception { + createIndex("test"); + ensureGreen(); + index("test", "type1", jsonBuilder().startObject() + .field("id", "1") + .field("svalue", "aaa") + .field("ivalue", 100) + .field("dvalue", 0.1) + .endObject()); + index("test", "type1", jsonBuilder().startObject() + .field("id", "2") + .field("svalue", "bbb") + .field("ivalue", 200) + .field("dvalue", 0.2) + .endObject()); + refresh(); + + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort("svalue", SortOrder.ASC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getMaxScore(), equalTo(Float.NaN)); + for (SearchHit hit : searchResponse.getHits()) { + assertThat(hit.getScore(), equalTo(Float.NaN)); + } + + // now check with score tracking + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort("svalue", SortOrder.ASC) + .setTrackScores(true) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getMaxScore(), not(equalTo(Float.NaN))); + for (SearchHit hit : searchResponse.getHits()) { + assertThat(hit.getScore(), not(equalTo(Float.NaN))); + } + } + + public void testRandomSorting() throws IOException, InterruptedException, ExecutionException { + Random random = getRandom(); + assertAcked(prepareCreate("test") + .addMapping("type", + XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("properties") + .startObject("sparse_bytes") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .startObject("dense_bytes") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endObject())); + ensureGreen(); + + TreeMap sparseBytes = new TreeMap<>(); + TreeMap denseBytes = new TreeMap<>(); + int numDocs = randomIntBetween(200, 300); + IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; + for (int i = 0; i < numDocs; i++) { + String docId = Integer.toString(i); + BytesRef ref = null; + do { + ref = new BytesRef(TestUtil.randomRealisticUnicodeString(random)); + } while (denseBytes.containsKey(ref)); + denseBytes.put(ref, docId); + XContentBuilder src = jsonBuilder().startObject().field("dense_bytes", ref.utf8ToString()); + if (rarely()) { + src.field("sparse_bytes", ref.utf8ToString()); + sparseBytes.put(ref, docId); + } + src.endObject(); + builders[i] = client().prepareIndex("test", "type", docId).setSource(src); + } + indexRandom(true, builders); + { + int size = between(1, denseBytes.size()); + SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).setSize(size) + .addSort("dense_bytes", SortOrder.ASC).execute().actionGet(); + assertNoFailures(searchResponse); + assertThat(searchResponse.getHits().getTotalHits(), equalTo((long) numDocs)); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + Set> entrySet = denseBytes.entrySet(); + Iterator> iterator = entrySet.iterator(); + for (int i = 0; i < size; i++) { + assertThat(iterator.hasNext(), equalTo(true)); + Entry next = iterator.next(); + assertThat("pos: " + i, searchResponse.getHits().getAt(i).id(), equalTo(next.getValue())); + assertThat(searchResponse.getHits().getAt(i).sortValues()[0].toString(), equalTo(next.getKey().utf8ToString())); + } + } + if (!sparseBytes.isEmpty()) { + int size = between(1, sparseBytes.size()); + SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()) + .setPostFilter(QueryBuilders.existsQuery("sparse_bytes")).setSize(size).addSort("sparse_bytes", SortOrder.ASC).execute() + .actionGet(); + assertNoFailures(searchResponse); + assertThat(searchResponse.getHits().getTotalHits(), equalTo((long) sparseBytes.size())); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + Set> entrySet = sparseBytes.entrySet(); + Iterator> iterator = entrySet.iterator(); + for (int i = 0; i < size; i++) { + assertThat(iterator.hasNext(), equalTo(true)); + Entry next = iterator.next(); + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(next.getValue())); + assertThat(searchResponse.getHits().getAt(i).sortValues()[0].toString(), equalTo(next.getKey().utf8ToString())); + } + } + } + + public void test3078() { + createIndex("test"); + ensureGreen(); + + for (int i = 1; i < 101; i++) { + client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", Integer.toString(i)).execute().actionGet(); + } + refresh(); + SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)).execute().actionGet(); + assertThat(searchResponse.getHits().getAt(0).sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); + assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); + + // reindex and refresh + client().prepareIndex("test", "type", Integer.toString(1)).setSource("field", Integer.toString(1)).execute().actionGet(); + refresh(); + + searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)).execute().actionGet(); + assertThat(searchResponse.getHits().getAt(0).sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); + assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); + + // reindex - no refresh + client().prepareIndex("test", "type", Integer.toString(1)).setSource("field", Integer.toString(1)).execute().actionGet(); + + searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)).execute().actionGet(); + assertThat(searchResponse.getHits().getAt(0).sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); + assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); + + // force merge + forceMerge(); + refresh(); + + client().prepareIndex("test", "type", Integer.toString(1)).setSource("field", Integer.toString(1)).execute().actionGet(); + searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)).execute().actionGet(); + assertThat(searchResponse.getHits().getAt(0).sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); + assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); + + refresh(); + searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)).execute().actionGet(); + assertThat(searchResponse.getHits().getAt(0).sortValues()[0].toString(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); + assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); + } + + public void testScoreSortDirection() throws Exception { + createIndex("test"); + ensureGreen(); + + client().prepareIndex("test", "type", "1").setSource("field", 2).execute().actionGet(); + client().prepareIndex("test", "type", "2").setSource("field", 1).execute().actionGet(); + client().prepareIndex("test", "type", "3").setSource("field", 0).execute().actionGet(); + + refresh(); + + SearchResponse searchResponse = client() + .prepareSearch("test") + .setQuery( + QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field"))) + .execute().actionGet(); + assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score())); + assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(2).score(), Matchers.lessThan(searchResponse.getHits().getAt(1).score())); + assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); + + searchResponse = client() + .prepareSearch("test") + .setQuery( + QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field"))) + .addSort("_score", SortOrder.DESC).execute().actionGet(); + assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score())); + assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(2).score(), Matchers.lessThan(searchResponse.getHits().getAt(1).score())); + assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); + + searchResponse = client() + .prepareSearch("test") + .setQuery( + QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field"))) + .addSort("_score", SortOrder.DESC).execute().actionGet(); + assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); + } + + public void testScoreSortDirectionWithFunctionScore() throws Exception { + createIndex("test"); + ensureGreen(); + + client().prepareIndex("test", "type", "1").setSource("field", 2).execute().actionGet(); + client().prepareIndex("test", "type", "2").setSource("field", 1).execute().actionGet(); + client().prepareIndex("test", "type", "3").setSource("field", 0).execute().actionGet(); + + refresh(); + + SearchResponse searchResponse = client().prepareSearch("test") + .setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field"))).execute().actionGet(); + assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score())); + assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(2).score(), Matchers.lessThan(searchResponse.getHits().getAt(1).score())); + assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); + + searchResponse = client().prepareSearch("test") + .setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field"))) + .addSort("_score", SortOrder.DESC).execute().actionGet(); + assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score())); + assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(2).score(), Matchers.lessThan(searchResponse.getHits().getAt(1).score())); + assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); + + searchResponse = client().prepareSearch("test") + .setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field"))) + .addSort("_score", SortOrder.DESC).execute().actionGet(); + assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); + } + + public void testIssue2986() { + createIndex("test"); + + client().prepareIndex("test", "post", "1").setSource("{\"field1\":\"value1\"}").execute().actionGet(); + client().prepareIndex("test", "post", "2").setSource("{\"field1\":\"value2\"}").execute().actionGet(); + client().prepareIndex("test", "post", "3").setSource("{\"field1\":\"value3\"}").execute().actionGet(); + refresh(); + SearchResponse result = client().prepareSearch("test").setQuery(matchAllQuery()).setTrackScores(true) + .addSort("field1", SortOrder.ASC).execute().actionGet(); + + for (SearchHit hit : result.getHits()) { + assertFalse(Float.isNaN(hit.getScore())); + } + } + + public void testIssue2991() { + for (int i = 1; i < 4; i++) { + try { + client().admin().indices().prepareDelete("test").execute().actionGet(); + } catch (Exception e) { + // ignore + } + createIndex("test"); + ensureGreen(); + client().prepareIndex("test", "type", "1").setSource("tag", "alpha").execute().actionGet(); + refresh(); + + client().prepareIndex("test", "type", "3").setSource("tag", "gamma").execute().actionGet(); + refresh(); + + client().prepareIndex("test", "type", "4").setSource("tag", "delta").execute().actionGet(); + + refresh(); + client().prepareIndex("test", "type", "2").setSource("tag", "beta").execute().actionGet(); + + refresh(); + SearchResponse resp = client().prepareSearch("test").setSize(2).setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("tag").order(SortOrder.ASC)).execute().actionGet(); + assertHitCount(resp, 4); + assertThat(resp.getHits().hits().length, equalTo(2)); + assertFirstHit(resp, hasId("1")); + assertSecondHit(resp, hasId("2")); + + resp = client().prepareSearch("test").setSize(2).setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("tag").order(SortOrder.DESC)).execute().actionGet(); + assertHitCount(resp, 4); + assertThat(resp.getHits().hits().length, equalTo(2)); + assertFirstHit(resp, hasId("3")); + assertSecondHit(resp, hasId("4")); + } + } + + public void testSimpleSorts() throws Exception { + Random random = random(); + assertAcked(prepareCreate("test") +.addMapping("type1", + XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("str_value") + .field("type", "string").field("index", "not_analyzed").startObject("fielddata") + .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("boolean_value") + .field("type", "boolean").endObject().startObject("byte_value").field("type", "byte").startObject("fielddata") + .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("short_value") + .field("type", "short").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null) + .endObject().endObject().startObject("integer_value").field("type", "integer").startObject("fielddata") + .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("long_value") + .field("type", "long").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null) + .endObject().endObject().startObject("float_value").field("type", "float").startObject("fielddata") + .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("double_value") + .field("type", "double").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null) + .endObject().endObject().endObject().endObject().endObject())); + ensureGreen(); + List builders = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + IndexRequestBuilder builder = client().prepareIndex("test", "type1", Integer.toString(i)).setSource(jsonBuilder().startObject() + .field("str_value", new String(new char[]{(char) (97 + i), (char) (97 + i)})) + .field("boolean_value", true) + .field("byte_value", i) + .field("short_value", i) + .field("integer_value", i) + .field("long_value", i) + .field("float_value", 0.1 * i) + .field("double_value", 0.1 * i) + .endObject()); + builders.add(builder); + } + Collections.shuffle(builders, random); + for (IndexRequestBuilder builder : builders) { + builder.execute().actionGet(); + if (random.nextBoolean()) { + if (random.nextInt(5) != 0) { + refresh(); + } else { + client().admin().indices().prepareFlush().execute().actionGet(); + } + } + + } + refresh(); + + // STRING + int size = 1 + random.nextInt(10); + + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(size) + .addSort("str_value", SortOrder.ASC) + .execute().actionGet(); + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); + assertThat(searchResponse.getHits().getAt(i).sortValues()[0].toString(), + equalTo(new String(new char[] { (char) (97 + i), (char) (97 + i) }))); + } + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(size) + .addSort("str_value", SortOrder.DESC) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); + assertThat(searchResponse.getHits().getAt(i).sortValues()[0].toString(), + equalTo(new String(new char[] { (char) (97 + (9 - i)), (char) (97 + (9 - i)) }))); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + + + // BYTE + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("byte_value", SortOrder.ASC).execute() + .actionGet(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).byteValue(), equalTo((byte) i)); + } + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("byte_value", SortOrder.DESC).execute() + .actionGet(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).byteValue(), equalTo((byte) (9 - i))); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + + // SHORT + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("short_value", SortOrder.ASC).execute() + .actionGet(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).shortValue(), equalTo((short) i)); + } + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("short_value", SortOrder.DESC).execute() + .actionGet(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).shortValue(), equalTo((short) (9 - i))); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + + // INTEGER + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("integer_value", SortOrder.ASC).execute() + .actionGet(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).intValue(), equalTo(i)); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("integer_value", SortOrder.DESC) + .execute().actionGet(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).intValue(), equalTo((9 - i))); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + + // LONG + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("long_value", SortOrder.ASC).execute() + .actionGet(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).longValue(), equalTo((long) i)); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("long_value", SortOrder.DESC).execute() + .actionGet(); + assertHitCount(searchResponse, 10L); + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).longValue(), equalTo((long) (9 - i))); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + + // FLOAT + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("float_value", SortOrder.ASC).execute() + .actionGet(); + + assertHitCount(searchResponse, 10L); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).doubleValue(), closeTo(0.1d * i, 0.000001d)); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("float_value", SortOrder.DESC).execute() + .actionGet(); + + assertHitCount(searchResponse, 10); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).doubleValue(), closeTo(0.1d * (9 - i), 0.000001d)); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + + // DOUBLE + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("double_value", SortOrder.ASC).execute() + .actionGet(); + + assertHitCount(searchResponse, 10L); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).doubleValue(), closeTo(0.1d * i, 0.000001d)); + } + + assertThat(searchResponse.toString(), not(containsString("error"))); + size = 1 + random.nextInt(10); + searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("double_value", SortOrder.DESC).execute() + .actionGet(); + + assertHitCount(searchResponse, 10L); + assertThat(searchResponse.getHits().hits().length, equalTo(size)); + for (int i = 0; i < size; i++) { + assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); + assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).doubleValue(), closeTo(0.1d * (9 - i), 0.000001d)); + } + + assertNoFailures(searchResponse); + } + + public void testSortMissingNumbers() throws Exception { + assertAcked(prepareCreate("test").addMapping("type1", + XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("properties") + .startObject("i_value") + .field("type", "integer") + .startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject() + .endObject() + .startObject("d_value") + .field("type", "float") + .startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject() + .endObject() + .endObject() + .endObject() + .endObject())); + ensureGreen(); + client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() + .field("id", "1") + .field("i_value", -1) + .field("d_value", -1.1) + .endObject()).execute().actionGet(); + + client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject() + .field("id", "2") + .endObject()).execute().actionGet(); + + client().prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject() + .field("id", "1") + .field("i_value", 2) + .field("d_value", 2.2) + .endObject()).execute().actionGet(); + + flush(); + refresh(); + + logger.info("--> sort with no missing (same as missing _last)"); + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC)) + .execute().actionGet(); + assertNoFailures(searchResponse); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); + + logger.info("--> sort with missing _last"); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_last")) + .execute().actionGet(); + assertNoFailures(searchResponse); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); + + logger.info("--> sort with missing _first"); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_first")) + .execute().actionGet(); + assertNoFailures(searchResponse); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); + } + + public void testSortMissingStrings() throws IOException { + assertAcked(prepareCreate("test").addMapping("type1", + XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("properties") + .startObject("value") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endObject())); + ensureGreen(); + client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() + .field("id", "1") + .field("value", "a") + .endObject()).execute().actionGet(); + + client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject() + .field("id", "2") + .endObject()).execute().actionGet(); + + client().prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject() + .field("id", "1") + .field("value", "c") + .endObject()).execute().actionGet(); + + flush(); + refresh(); + + // TODO: WTF? + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + throw new RuntimeException(); + } + + logger.info("--> sort with no missing (same as missing _last)"); + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC)) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); + + logger.info("--> sort with missing _last"); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("_last")) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); + + logger.info("--> sort with missing _first"); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("_first")) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); + + logger.info("--> sort with missing b"); + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("b")) + .execute().actionGet(); + assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("2")); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); + } + + public void testIgnoreUnmapped() throws Exception { + createIndex("test"); + ensureYellow(); + + client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() + .field("id", "1") + .field("i_value", -1) + .field("d_value", -1.1) + .endObject()).execute().actionGet(); + + logger.info("--> sort with an unmapped field, verify it fails"); + try { + SearchResponse result = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("kkk")) + .execute().actionGet(); + assertThat("Expected exception but returned with", result, nullValue()); + } catch (SearchPhaseExecutionException e) { + //we check that it's a parse failure rather than a different shard failure + for (ShardSearchFailure shardSearchFailure : e.shardFailures()) { + assertThat(shardSearchFailure.toString(), containsString("[No mapping found for [kkk] in order to sort on]")); + } + } + + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("kkk").unmappedType("string")) + .execute().actionGet(); + assertNoFailures(searchResponse); + } + + public void testSortMVField() throws Exception { + assertAcked(prepareCreate("test") +.addMapping("type1", + XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("long_values") + .field("type", "long").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null) + .endObject().endObject().startObject("int_values").field("type", "integer").startObject("fielddata") + .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("short_values") + .field("type", "short").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null) + .endObject().endObject().startObject("byte_values").field("type", "byte").startObject("fielddata") + .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("float_values") + .field("type", "float").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null) + .endObject().endObject().startObject("double_values").field("type", "double").startObject("fielddata") + .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().startObject("string_values") + .field("type", "string").field("index", "not_analyzed").startObject("fielddata") + .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().endObject().endObject() + .endObject())); + ensureGreen(); + + client().prepareIndex("test", "type1", Integer.toString(1)).setSource(jsonBuilder().startObject() + .array("long_values", 1L, 5L, 10L, 8L) + .array("int_values", 1, 5, 10, 8) + .array("short_values", 1, 5, 10, 8) + .array("byte_values", 1, 5, 10, 8) + .array("float_values", 1f, 5f, 10f, 8f) + .array("double_values", 1d, 5d, 10d, 8d) + .array("string_values", "01", "05", "10", "08") + .endObject()).execute().actionGet(); + client().prepareIndex("test", "type1", Integer.toString(2)).setSource(jsonBuilder().startObject() + .array("long_values", 11L, 15L, 20L, 7L) + .array("int_values", 11, 15, 20, 7) + .array("short_values", 11, 15, 20, 7) + .array("byte_values", 11, 15, 20, 7) + .array("float_values", 11f, 15f, 20f, 7f) + .array("double_values", 11d, 15d, 20d, 7d) + .array("string_values", "11", "15", "20", "07") + .endObject()).execute().actionGet(); + client().prepareIndex("test", "type1", Integer.toString(3)).setSource(jsonBuilder().startObject() + .array("long_values", 2L, 1L, 3L, -4L) + .array("int_values", 2, 1, 3, -4) + .array("short_values", 2, 1, 3, -4) + .array("byte_values", 2, 1, 3, -4) + .array("float_values", 2f, 1f, 3f, -4f) + .array("double_values", 2d, 1d, 3d, -4d) + .array("string_values", "02", "01", "03", "!4") + .endObject()).execute().actionGet(); + + refresh(); + + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("long_values", SortOrder.ASC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(-4L)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(1L)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(7L)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("long_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(20L)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(10L)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(3L)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort(SortBuilders.fieldSort("long_values").order(SortOrder.DESC).sortMode("sum")) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(53L)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(24L)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(2L)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("int_values", SortOrder.ASC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(-4)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(1)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(7)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("int_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(20)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(10)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(3)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("short_values", SortOrder.ASC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(-4)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(1)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(7)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("short_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(20)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(10)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(3)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("byte_values", SortOrder.ASC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(-4)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(1)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(7)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("byte_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(20)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(10)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(3)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("float_values", SortOrder.ASC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).floatValue(), equalTo(-4f)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).floatValue(), equalTo(1f)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).floatValue(), equalTo(7f)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("float_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).floatValue(), equalTo(20f)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).floatValue(), equalTo(10f)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).floatValue(), equalTo(3f)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("double_values", SortOrder.ASC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), equalTo(-4d)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), equalTo(1d)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), equalTo(7d)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("double_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), equalTo(20d)); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), equalTo(10d)); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), equalTo(3d)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("string_values", SortOrder.ASC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); + assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("!4")); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("01")); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); + assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("07")); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(10) + .addSort("string_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20")); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10")); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); + assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03")); + } + + public void testSortOnRareField() throws IOException { + assertAcked(prepareCreate("test").addMapping("type1", + XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("string_values") + .field("type", "string").field("index", "not_analyzed").startObject("fielddata") + .field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject().endObject().endObject() + .endObject())); + ensureGreen(); + client().prepareIndex("test", "type1", Integer.toString(1)).setSource(jsonBuilder().startObject() + .array("string_values", "01", "05", "10", "08") + .endObject()).execute().actionGet(); + + + refresh(); + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(3) + .addSort("string_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().hits().length, equalTo(1)); + + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(1))); + assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("10")); + + client().prepareIndex("test", "type1", Integer.toString(2)).setSource(jsonBuilder().startObject() + .array("string_values", "11", "15", "20", "07") + .endObject()).execute().actionGet(); + for (int i = 0; i < 15; i++) { + client().prepareIndex("test", "type1", Integer.toString(300 + i)).setSource(jsonBuilder().startObject() + .array("some_other_field", "foobar") + .endObject()).execute().actionGet(); + } + refresh(); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(2) + .addSort("string_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().hits().length, equalTo(2)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20")); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10")); + + + client().prepareIndex("test", "type1", Integer.toString(3)).setSource(jsonBuilder().startObject() + .array("string_values", "02", "01", "03", "!4") + .endObject()).execute().actionGet(); + for (int i = 0; i < 15; i++) { + client().prepareIndex("test", "type1", Integer.toString(300 + i)).setSource(jsonBuilder().startObject() + .array("some_other_field", "foobar") + .endObject()).execute().actionGet(); + } + refresh(); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(3) + .addSort("string_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20")); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10")); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); + assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03")); + + for (int i = 0; i < 15; i++) { + client().prepareIndex("test", "type1", Integer.toString(300 + i)).setSource(jsonBuilder().startObject() + .array("some_other_field", "foobar") + .endObject()).execute().actionGet(); + refresh(); + } + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(3) + .addSort("string_values", SortOrder.DESC) + .execute().actionGet(); + + assertThat(searchResponse.getHits().hits().length, equalTo(3)); + + assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); + assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20")); + + assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); + assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10")); + + assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); + assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03")); + } + + public void testSortMetaField() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp").field("enabled", true).endObject() + .endObject().endObject(); + assertAcked(prepareCreate("test") + .addMapping("type", mapping)); + ensureGreen(); + final int numDocs = randomIntBetween(10, 20); + IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; + for (int i = 0; i < numDocs; ++i) { + indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000))) + .setSource(); + } + indexRandom(true, indexReqs); + + SortOrder order = randomFrom(SortOrder.values()); + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(randomIntBetween(1, numDocs + 5)) + .addSort("_uid", order) + .execute().actionGet(); + assertNoFailures(searchResponse); + SearchHit[] hits = searchResponse.getHits().hits(); + BytesRef previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM; + for (int i = 0; i < hits.length; ++i) { + final BytesRef uid = new BytesRef(Uid.createUid(hits[i].type(), hits[i].id())); + assertThat(previous, order == SortOrder.ASC ? lessThan(uid) : greaterThan(uid)); + previous = uid; + } + + /* + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(randomIntBetween(1, numDocs + 5)) + .addSort("_id", order) + .execute().actionGet(); + assertNoFailures(searchResponse); + hits = searchResponse.getHits().hits(); + previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM; + for (int i = 0; i < hits.length; ++i) { + final BytesRef id = new BytesRef(Uid.createUid(hits[i].type(), hits[i].id())); + assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id)); + previous = id; + }*/ + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(randomIntBetween(1, numDocs + 5)) + .addSort("_timestamp", order) + .addField("_timestamp") + .execute().actionGet(); + assertNoFailures(searchResponse); + hits = searchResponse.getHits().hits(); + Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE; + for (int i = 0; i < hits.length; ++i) { + SearchHitField timestampField = hits[i].getFields().get("_timestamp"); + Long timestamp = timestampField.getValue(); + assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp)); + previousTs = timestamp; + } + } + + /** + * Test case for issue 6150: https://github.com/elasticsearch/elasticsearch/issues/6150 + */ + public void testNestedSort() throws IOException, InterruptedException, ExecutionException { + assertAcked(prepareCreate("test") + .addMapping("type", + XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("properties") + .startObject("nested") + .field("type", "nested") + .startObject("properties") + .startObject("foo") + .field("type", "string") + .startObject("fields") + .startObject("sub") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject())); + ensureGreen(); + + client().prepareIndex("test", "type", "1").setSource(jsonBuilder().startObject() + .startObject("nested") + .field("foo", "bar bar") + .endObject() + .endObject()).execute().actionGet(); + refresh(); + + // We sort on nested field + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("nested.foo").setNestedPath("nested").order(SortOrder.DESC)) + .execute().actionGet(); + assertNoFailures(searchResponse); + SearchHit[] hits = searchResponse.getHits().hits(); + for (int i = 0; i < hits.length; ++i) { + assertThat(hits[i].getSortValues().length, is(1)); + Object o = hits[i].getSortValues()[0]; + assertThat(o, notNullValue()); + Text text = (Text) o; + assertThat(text.string(), is("bar")); + } + + + // We sort on nested sub field + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(SortBuilders.fieldSort("nested.foo.sub").setNestedPath("nested").order(SortOrder.DESC)) + .execute().actionGet(); + assertNoFailures(searchResponse); + hits = searchResponse.getHits().hits(); + for (int i = 0; i < hits.length; ++i) { + assertThat(hits[i].getSortValues().length, is(1)); + Object o = hits[i].getSortValues()[0]; + assertThat(o, notNullValue()); + Text text = (Text) o; + assertThat(text.string(), is("bar bar")); + } + } + + public void testSortDuelBetweenSingleShardAndMultiShardIndex() throws Exception { + String sortField = "sortField"; + assertAcked(prepareCreate("test1") + .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(2, maximumNumberOfShards())) + .addMapping("type", sortField, "type=long").get()); + assertAcked(prepareCreate("test2") + .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .addMapping("type", sortField, "type=long").get()); + + for (String index : new String[]{"test1", "test2"}) { + List docs = new ArrayList<>(); + for (int i = 0; i < 256; i++) { + docs.add(client().prepareIndex(index, "type", Integer.toString(i)).setSource(sortField, i)); + } + indexRandom(true, docs); + } + + ensureSearchable("test1", "test2"); + SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC; + int from = between(0, 256); + int size = between(0, 256); + SearchResponse multiShardResponse = client().prepareSearch("test1").setFrom(from).setSize(size).addSort(sortField, order).get(); + assertNoFailures(multiShardResponse); + SearchResponse singleShardResponse = client().prepareSearch("test2").setFrom(from).setSize(size).addSort(sortField, order).get(); + assertNoFailures(singleShardResponse); + + assertThat(multiShardResponse.getHits().totalHits(), equalTo(singleShardResponse.getHits().totalHits())); + assertThat(multiShardResponse.getHits().getHits().length, equalTo(singleShardResponse.getHits().getHits().length)); + for (int i = 0; i < multiShardResponse.getHits().getHits().length; i++) { + assertThat(multiShardResponse.getHits().getAt(i).sortValues()[0], + equalTo(singleShardResponse.getHits().getAt(i).sortValues()[0])); + assertThat(multiShardResponse.getHits().getAt(i).id(), equalTo(singleShardResponse.getHits().getAt(i).id())); + } + } + +} diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java index bae2eb552d1..096e30f40da 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java @@ -20,74 +20,37 @@ package org.elasticsearch.messy.tests; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; -import org.apache.lucene.util.UnicodeUtil; -import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.text.Text; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.mapper.Uid; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.groovy.GroovyPlugin; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHitField; -import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScriptSortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; -import org.hamcrest.Matchers; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; -import java.util.Locale; -import java.util.Map.Entry; import java.util.Random; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ExecutionException; import static org.apache.lucene.util.GeoUtils.TOLERANCE; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertOrderedSearchHits; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; /** * @@ -98,397 +61,6 @@ public class SimpleSortTests extends ESIntegTestCase { return pluginList(GroovyPlugin.class, InternalSettingsPlugin.class); } - @LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elasticsearch/elasticsearch/issues/9421") - public void testIssue8226() { - int numIndices = between(5, 10); - final boolean useMapping = randomBoolean(); - for (int i = 0; i < numIndices; i++) { - if (useMapping) { - assertAcked(prepareCreate("test_" + i).addAlias(new Alias("test")).addMapping("foo", "entry", "type=long")); - } else { - assertAcked(prepareCreate("test_" + i).addAlias(new Alias("test"))); - } - if (i > 0) { - client().prepareIndex("test_" + i, "foo", "" + i).setSource("{\"entry\": " + i + "}").get(); - } - } - ensureYellow(); - refresh(); - // sort DESC - SearchResponse searchResponse = client().prepareSearch() - .addSort(new FieldSortBuilder("entry").order(SortOrder.DESC).unmappedType(useMapping ? null : "long")) - .setSize(10).get(); - logClusterState(); - assertSearchResponse(searchResponse); - - for (int j = 1; j < searchResponse.getHits().hits().length; j++) { - Number current = (Number) searchResponse.getHits().hits()[j].getSource().get("entry"); - Number previous = (Number) searchResponse.getHits().hits()[j-1].getSource().get("entry"); - assertThat(searchResponse.toString(), current.intValue(), lessThan(previous.intValue())); - } - - // sort ASC - searchResponse = client().prepareSearch() - .addSort(new FieldSortBuilder("entry").order(SortOrder.ASC).unmappedType(useMapping ? null : "long")) - .setSize(10).get(); - logClusterState(); - assertSearchResponse(searchResponse); - - for (int j = 1; j < searchResponse.getHits().hits().length; j++) { - Number current = (Number) searchResponse.getHits().hits()[j].getSource().get("entry"); - Number previous = (Number) searchResponse.getHits().hits()[j-1].getSource().get("entry"); - assertThat(searchResponse.toString(), current.intValue(), greaterThan(previous.intValue())); - } - } - - @LuceneTestCase.BadApple(bugUrl = "simon is working on this") - public void testIssue6614() throws ExecutionException, InterruptedException { - List builders = new ArrayList<>(); - boolean strictTimeBasedIndices = randomBoolean(); - final int numIndices = randomIntBetween(2, 25); // at most 25 days in the month - for (int i = 0; i < numIndices; i++) { - final String indexId = strictTimeBasedIndices ? "idx_" + i : "idx"; - if (strictTimeBasedIndices || i == 0) { - createIndex(indexId); - } - final int numDocs = randomIntBetween(1, 23); // hour of the day - for (int j = 0; j < numDocs; j++) { - builders.add(client().prepareIndex(indexId, "type").setSource("foo", "bar", "timeUpdated", "2014/07/" + String.format(Locale.ROOT, "%02d", i+1)+" " + String.format(Locale.ROOT, "%02d", j+1) + ":00:00")); - } - } - int docs = builders.size(); - indexRandom(true, builders); - ensureYellow(); - SearchResponse allDocsResponse = client().prepareSearch().setQuery( - QueryBuilders.boolQuery().must(QueryBuilders.termQuery("foo", "bar")).must( - QueryBuilders.rangeQuery("timeUpdated").gte("2014/0" + randomIntBetween(1, 7) + "/01"))) - .addSort(new FieldSortBuilder("timeUpdated").order(SortOrder.ASC).unmappedType("date")) - .setSize(docs).get(); - assertSearchResponse(allDocsResponse); - - final int numiters = randomIntBetween(1, 20); - for (int i = 0; i < numiters; i++) { - SearchResponse searchResponse = client().prepareSearch().setQuery( - QueryBuilders.boolQuery().must(QueryBuilders.termQuery("foo", "bar")).must( - QueryBuilders.rangeQuery("timeUpdated").gte("2014/" + String.format(Locale.ROOT, "%02d", randomIntBetween(1, 7)) + "/01"))) - .addSort(new FieldSortBuilder("timeUpdated").order(SortOrder.ASC).unmappedType("date")) - .setSize(scaledRandomIntBetween(1, docs)).get(); - assertSearchResponse(searchResponse); - for (int j = 0; j < searchResponse.getHits().hits().length; j++) { - assertThat(searchResponse.toString() + "\n vs. \n" + allDocsResponse.toString(), searchResponse.getHits().hits()[j].getId(), equalTo(allDocsResponse.getHits().hits()[j].getId())); - } - } - - } - - public void testIssue6639() throws ExecutionException, InterruptedException { - assertAcked(prepareCreate("$index") - .addMapping("$type","{\"$type\": {\"properties\": {\"grantee\": {\"index\": \"not_analyzed\", \"term_vector\": \"with_positions_offsets\", \"type\": \"string\", \"analyzer\": \"snowball\", \"boost\": 1.0, \"store\": true}}}}")); - indexRandom(true, - client().prepareIndex("$index", "$type", "data.activity.5").setSource("{\"django_ct\": \"data.activity\", \"grantee\": \"Grantee 1\"}"), - client().prepareIndex("$index", "$type", "data.activity.6").setSource("{\"django_ct\": \"data.activity\", \"grantee\": \"Grantee 2\"}")); - ensureYellow(); - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort("grantee", SortOrder.ASC) - .execute().actionGet(); - assertOrderedSearchHits(searchResponse, "data.activity.5", "data.activity.6"); - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort("grantee", SortOrder.DESC) - .execute().actionGet(); - assertOrderedSearchHits(searchResponse, "data.activity.6", "data.activity.5"); - } - - public void testTrackScores() throws Exception { - createIndex("test"); - ensureGreen(); - index("test", "type1", jsonBuilder().startObject() - .field("id", "1") - .field("svalue", "aaa") - .field("ivalue", 100) - .field("dvalue", 0.1) - .endObject()); - index("test", "type1", jsonBuilder().startObject() - .field("id", "2") - .field("svalue", "bbb") - .field("ivalue", 200) - .field("dvalue", 0.2) - .endObject()); - refresh(); - - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort("svalue", SortOrder.ASC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getMaxScore(), equalTo(Float.NaN)); - for (SearchHit hit : searchResponse.getHits()) { - assertThat(hit.getScore(), equalTo(Float.NaN)); - } - - // now check with score tracking - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort("svalue", SortOrder.ASC) - .setTrackScores(true) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getMaxScore(), not(equalTo(Float.NaN))); - for (SearchHit hit : searchResponse.getHits()) { - assertThat(hit.getScore(), not(equalTo(Float.NaN))); - } - } - - public void testRandomSorting() throws IOException, InterruptedException, ExecutionException { - Random random = getRandom(); - assertAcked(prepareCreate("test") - .addMapping("type", - XContentFactory.jsonBuilder() - .startObject() - .startObject("type") - .startObject("properties") - .startObject("sparse_bytes") - .field("type", "string") - .field("index", "not_analyzed") - .endObject() - .startObject("dense_bytes") - .field("type", "string") - .field("index", "not_analyzed") - .endObject() - .endObject() - .endObject() - .endObject())); - ensureGreen(); - - TreeMap sparseBytes = new TreeMap<>(); - TreeMap denseBytes = new TreeMap<>(); - int numDocs = randomIntBetween(200, 300); - IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; - for (int i = 0; i < numDocs; i++) { - String docId = Integer.toString(i); - BytesRef ref = null; - do { - ref = new BytesRef(TestUtil.randomRealisticUnicodeString(random)); - } while (denseBytes.containsKey(ref)); - denseBytes.put(ref, docId); - XContentBuilder src = jsonBuilder().startObject().field("dense_bytes", ref.utf8ToString()); - if (rarely()) { - src.field("sparse_bytes", ref.utf8ToString()); - sparseBytes.put(ref, docId); - } - src.endObject(); - builders[i] = client().prepareIndex("test", "type", docId).setSource(src); - } - indexRandom(true, builders); - { - int size = between(1, denseBytes.size()); - SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).setSize(size) - .addSort("dense_bytes", SortOrder.ASC).execute().actionGet(); - assertNoFailures(searchResponse); - assertThat(searchResponse.getHits().getTotalHits(), equalTo((long) numDocs)); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - Set> entrySet = denseBytes.entrySet(); - Iterator> iterator = entrySet.iterator(); - for (int i = 0; i < size; i++) { - assertThat(iterator.hasNext(), equalTo(true)); - Entry next = iterator.next(); - assertThat("pos: " + i, searchResponse.getHits().getAt(i).id(), equalTo(next.getValue())); - assertThat(searchResponse.getHits().getAt(i).sortValues()[0].toString(), equalTo(next.getKey().utf8ToString())); - } - } - if (!sparseBytes.isEmpty()) { - int size = between(1, sparseBytes.size()); - SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()) - .setPostFilter(QueryBuilders.existsQuery("sparse_bytes")).setSize(size).addSort("sparse_bytes", SortOrder.ASC).execute() - .actionGet(); - assertNoFailures(searchResponse); - assertThat(searchResponse.getHits().getTotalHits(), equalTo((long) sparseBytes.size())); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - Set> entrySet = sparseBytes.entrySet(); - Iterator> iterator = entrySet.iterator(); - for (int i = 0; i < size; i++) { - assertThat(iterator.hasNext(), equalTo(true)); - Entry next = iterator.next(); - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(next.getValue())); - assertThat(searchResponse.getHits().getAt(i).sortValues()[0].toString(), equalTo(next.getKey().utf8ToString())); - } - } - } - - public void test3078() { - createIndex("test"); - ensureGreen(); - - for (int i = 1; i < 101; i++) { - client().prepareIndex("test", "type", Integer.toString(i)).setSource("field", Integer.toString(i)).execute().actionGet(); - } - refresh(); - SearchResponse searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)).execute().actionGet(); - assertThat(searchResponse.getHits().getAt(0).sortValues()[0].toString(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); - assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); - - // reindex and refresh - client().prepareIndex("test", "type", Integer.toString(1)).setSource("field", Integer.toString(1)).execute().actionGet(); - refresh(); - - searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)).execute().actionGet(); - assertThat(searchResponse.getHits().getAt(0).sortValues()[0].toString(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); - assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); - - // reindex - no refresh - client().prepareIndex("test", "type", Integer.toString(1)).setSource("field", Integer.toString(1)).execute().actionGet(); - - searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)).execute().actionGet(); - assertThat(searchResponse.getHits().getAt(0).sortValues()[0].toString(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); - assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); - - // force merge - forceMerge(); - refresh(); - - client().prepareIndex("test", "type", Integer.toString(1)).setSource("field", Integer.toString(1)).execute().actionGet(); - searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)).execute().actionGet(); - assertThat(searchResponse.getHits().getAt(0).sortValues()[0].toString(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); - assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); - - refresh(); - searchResponse = client().prepareSearch("test").setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("field").order(SortOrder.ASC)).execute().actionGet(); - assertThat(searchResponse.getHits().getAt(0).sortValues()[0].toString(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).sortValues()[0].toString(), equalTo("10")); - assertThat(searchResponse.getHits().getAt(2).sortValues()[0].toString(), equalTo("100")); - } - - public void testScoreSortDirection() throws Exception { - createIndex("test"); - ensureGreen(); - - client().prepareIndex("test", "type", "1").setSource("field", 2).execute().actionGet(); - client().prepareIndex("test", "type", "2").setSource("field", 1).execute().actionGet(); - client().prepareIndex("test", "type", "3").setSource("field", 0).execute().actionGet(); - - refresh(); - - SearchResponse searchResponse = client() - .prepareSearch("test") - .setQuery( - QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field"))) - .execute().actionGet(); - assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score())); - assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); - assertThat(searchResponse.getHits().getAt(2).score(), Matchers.lessThan(searchResponse.getHits().getAt(1).score())); - assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); - - searchResponse = client() - .prepareSearch("test") - .setQuery( - QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field"))) - .addSort("_score", SortOrder.DESC).execute().actionGet(); - assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score())); - assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); - assertThat(searchResponse.getHits().getAt(2).score(), Matchers.lessThan(searchResponse.getHits().getAt(1).score())); - assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); - - searchResponse = client() - .prepareSearch("test") - .setQuery( - QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field"))) - .addSort("_score", SortOrder.DESC).execute().actionGet(); - assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); - assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); - } - - public void testScoreSortDirectionWithFunctionScore() throws Exception { - createIndex("test"); - ensureGreen(); - - client().prepareIndex("test", "type", "1").setSource("field", 2).execute().actionGet(); - client().prepareIndex("test", "type", "2").setSource("field", 1).execute().actionGet(); - client().prepareIndex("test", "type", "3").setSource("field", 0).execute().actionGet(); - - refresh(); - - SearchResponse searchResponse = client().prepareSearch("test") - .setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field"))).execute().actionGet(); - assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score())); - assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); - assertThat(searchResponse.getHits().getAt(2).score(), Matchers.lessThan(searchResponse.getHits().getAt(1).score())); - assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); - - searchResponse = client().prepareSearch("test") - .setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field"))) - .addSort("_score", SortOrder.DESC).execute().actionGet(); - assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).score(), Matchers.lessThan(searchResponse.getHits().getAt(0).score())); - assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); - assertThat(searchResponse.getHits().getAt(2).score(), Matchers.lessThan(searchResponse.getHits().getAt(1).score())); - assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); - - searchResponse = client().prepareSearch("test") - .setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field"))) - .addSort("_score", SortOrder.DESC).execute().actionGet(); - assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(1).getId(), equalTo("2")); - assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1")); - } - - public void testIssue2986() { - createIndex("test"); - - client().prepareIndex("test", "post", "1").setSource("{\"field1\":\"value1\"}").execute().actionGet(); - client().prepareIndex("test", "post", "2").setSource("{\"field1\":\"value2\"}").execute().actionGet(); - client().prepareIndex("test", "post", "3").setSource("{\"field1\":\"value3\"}").execute().actionGet(); - refresh(); - SearchResponse result = client().prepareSearch("test").setQuery(matchAllQuery()).setTrackScores(true).addSort("field1", SortOrder.ASC).execute().actionGet(); - - for (SearchHit hit : result.getHits()) { - assertFalse(Float.isNaN(hit.getScore())); - } - } - - public void testIssue2991() { - for (int i = 1; i < 4; i++) { - try { - client().admin().indices().prepareDelete("test").execute().actionGet(); - } catch (Exception e) { - // ignore - } - createIndex("test"); - ensureGreen(); - client().prepareIndex("test", "type", "1").setSource("tag", "alpha").execute().actionGet(); - refresh(); - - client().prepareIndex("test", "type", "3").setSource("tag", "gamma").execute().actionGet(); - refresh(); - - client().prepareIndex("test", "type", "4").setSource("tag", "delta").execute().actionGet(); - - refresh(); - client().prepareIndex("test", "type", "2").setSource("tag", "beta").execute().actionGet(); - - refresh(); - SearchResponse resp = client().prepareSearch("test").setSize(2).setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("tag").order(SortOrder.ASC)).execute().actionGet(); - assertHitCount(resp, 4); - assertThat(resp.getHits().hits().length, equalTo(2)); - assertFirstHit(resp, hasId("1")); - assertSecondHit(resp, hasId("2")); - - resp = client().prepareSearch("test").setSize(2).setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort("tag").order(SortOrder.DESC)).execute().actionGet(); - assertHitCount(resp, 4); - assertThat(resp.getHits().hits().length, equalTo(2)); - assertFirstHit(resp, hasId("3")); - assertSecondHit(resp, hasId("4")); - } - } - public void testSimpleSorts() throws Exception { Random random = random(); assertAcked(prepareCreate("test") @@ -531,41 +103,10 @@ public class SimpleSortTests extends ESIntegTestCase { } refresh(); - // STRING + // STRING script int size = 1 + random.nextInt(10); SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(size) - .addSort("str_value", SortOrder.ASC) - .execute().actionGet(); - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); - assertThat(searchResponse.getHits().getAt(i).sortValues()[0].toString(), equalTo(new String(new char[]{(char) (97 + i), (char) (97 + i)}))); - } - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(size) - .addSort("str_value", SortOrder.DESC) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); - assertThat(searchResponse.getHits().getAt(i).sortValues()[0].toString(), equalTo(new String(new char[]{(char) (97 + (9 - i)), (char) (97 + (9 - i))}))); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - - - // STRING script - size = 1 + random.nextInt(10); - - searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .setSize(size) .addSort(new ScriptSortBuilder(new Script("doc['str_value'].value"), "string")).execute().actionGet(); @@ -590,172 +131,6 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.toString(), not(containsString("error"))); - // BYTE - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("byte_value", SortOrder.ASC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).byteValue(), equalTo((byte) i)); - } - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("byte_value", SortOrder.DESC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).byteValue(), equalTo((byte) (9 - i))); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - - // SHORT - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("short_value", SortOrder.ASC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).shortValue(), equalTo((short) i)); - } - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("short_value", SortOrder.DESC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).shortValue(), equalTo((short) (9 - i))); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - - // INTEGER - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("integer_value", SortOrder.ASC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).intValue(), equalTo(i)); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("integer_value", SortOrder.DESC) - .execute().actionGet(); - - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).intValue(), equalTo((9 - i))); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - - // LONG - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("long_value", SortOrder.ASC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).longValue(), equalTo((long) i)); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("long_value", SortOrder.DESC).execute() - .actionGet(); - assertHitCount(searchResponse, 10L); - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).longValue(), equalTo((long) (9 - i))); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - - // FLOAT - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("float_value", SortOrder.ASC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10L); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).doubleValue(), closeTo(0.1d * i, 0.000001d)); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("float_value", SortOrder.DESC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).doubleValue(), closeTo(0.1d * (9 - i), 0.000001d)); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - - // DOUBLE - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("double_value", SortOrder.ASC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10L); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).doubleValue(), closeTo(0.1d * i, 0.000001d)); - } - - assertThat(searchResponse.toString(), not(containsString("error"))); - size = 1 + random.nextInt(10); - searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(size).addSort("double_value", SortOrder.DESC).execute() - .actionGet(); - - assertHitCount(searchResponse, 10L); - assertThat(searchResponse.getHits().hits().length, equalTo(size)); - for (int i = 0; i < size; i++) { - assertThat(searchResponse.getHits().getAt(i).id(), equalTo(Integer.toString(9 - i))); - assertThat(((Number) searchResponse.getHits().getAt(i).sortValues()[0]).doubleValue(), closeTo(0.1d * (9 - i), 0.000001d)); - } - - assertNoFailures(searchResponse); - } - - public void test2920() throws IOException { - assertAcked(prepareCreate("test").addMapping( - "test", - jsonBuilder().startObject().startObject("test").startObject("properties").startObject("value").field("type", "string") - .endObject().endObject().endObject().endObject())); - ensureGreen(); - for (int i = 0; i < 10; i++) { - client().prepareIndex("test", "test", Integer.toString(i)) - .setSource(jsonBuilder().startObject().field("value", "" + i).endObject()).execute().actionGet(); - } - refresh(); - SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()) - .addSort(SortBuilders.scriptSort(new Script("\u0027\u0027"), "string")).setSize(10).execute().actionGet(); assertNoFailures(searchResponse); } @@ -939,773 +314,19 @@ public class SimpleSortTests extends ESIntegTestCase { assertThat(searchResponse.getHits().getAt(0).field("id").value(), equalTo("2")); } - public void testSortMissingNumbers() throws Exception { - assertAcked(prepareCreate("test").addMapping("type1", - XContentFactory.jsonBuilder() - .startObject() - .startObject("type1") - .startObject("properties") - .startObject("i_value") - .field("type", "integer") - .startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject() - .endObject() - .startObject("d_value") - .field("type", "float") - .startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject() - .endObject() - .endObject() - .endObject() - .endObject())); + public void test2920() throws IOException { + assertAcked(prepareCreate("test").addMapping( + "test", + jsonBuilder().startObject().startObject("test").startObject("properties").startObject("value").field("type", "string") + .endObject().endObject().endObject().endObject())); ensureGreen(); - client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() - .field("id", "1") - .field("i_value", -1) - .field("d_value", -1.1) - .endObject()).execute().actionGet(); - - client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject() - .field("id", "2") - .endObject()).execute().actionGet(); - - client().prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject() - .field("id", "1") - .field("i_value", 2) - .field("d_value", 2.2) - .endObject()).execute().actionGet(); - - flush(); - refresh(); - - logger.info("--> sort with no missing (same as missing _last)"); - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC)) - .execute().actionGet(); - assertNoFailures(searchResponse); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); - - logger.info("--> sort with missing _last"); - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_last")) - .execute().actionGet(); - assertNoFailures(searchResponse); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); - - logger.info("--> sort with missing _first"); - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_first")) - .execute().actionGet(); - assertNoFailures(searchResponse); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().getAt(0).id(), equalTo("2")); - assertThat(searchResponse.getHits().getAt(1).id(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); - } - - public void testSortMissingStrings() throws IOException { - assertAcked(prepareCreate("test").addMapping("type1", - XContentFactory.jsonBuilder() - .startObject() - .startObject("type1") - .startObject("properties") - .startObject("value") - .field("type", "string") - .field("index", "not_analyzed") - .endObject() - .endObject() - .endObject() - .endObject())); - ensureGreen(); - client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() - .field("id", "1") - .field("value", "a") - .endObject()).execute().actionGet(); - - client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject() - .field("id", "2") - .endObject()).execute().actionGet(); - - client().prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject() - .field("id", "1") - .field("value", "c") - .endObject()).execute().actionGet(); - - flush(); - refresh(); - - // TODO: WTF? - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - throw new RuntimeException(); - } - - logger.info("--> sort with no missing (same as missing _last)"); - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC)) - .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); - - logger.info("--> sort with missing _last"); - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("_last")) - .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); - assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); - - logger.info("--> sort with missing _first"); - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("_first")) - .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().getAt(0).id(), equalTo("2")); - assertThat(searchResponse.getHits().getAt(1).id(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); - - logger.info("--> sort with missing b"); - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("value").order(SortOrder.ASC).missing("b")) - .execute().actionGet(); - assertThat(Arrays.toString(searchResponse.getShardFailures()), searchResponse.getFailedShards(), equalTo(0)); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); - assertThat(searchResponse.getHits().getAt(1).id(), equalTo("2")); - assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); - } - - public void testIgnoreUnmapped() throws Exception { - createIndex("test"); - ensureYellow(); - - client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() - .field("id", "1") - .field("i_value", -1) - .field("d_value", -1.1) - .endObject()).execute().actionGet(); - - logger.info("--> sort with an unmapped field, verify it fails"); - try { - SearchResponse result = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("kkk")) - .execute().actionGet(); - assertThat("Expected exception but returned with", result, nullValue()); - } catch (SearchPhaseExecutionException e) { - //we check that it's a parse failure rather than a different shard failure - for (ShardSearchFailure shardSearchFailure : e.shardFailures()) { - assertThat(shardSearchFailure.toString(), containsString("[No mapping found for [kkk] in order to sort on]")); - } - } - - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("kkk").unmappedType("string")) - .execute().actionGet(); - assertNoFailures(searchResponse); - } - - public void testSortMVField() throws Exception { - assertAcked(prepareCreate("test") - .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("long_values").field("type", "long").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject() - .startObject("int_values").field("type", "integer").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject() - .startObject("short_values").field("type", "short").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject() - .startObject("byte_values").field("type", "byte").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject() - .startObject("float_values").field("type", "float").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject() - .startObject("double_values").field("type", "double").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject() - .startObject("string_values").field("type", "string").field("index", "not_analyzed").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject() - .endObject().endObject().endObject())); - ensureGreen(); - - client().prepareIndex("test", "type1", Integer.toString(1)).setSource(jsonBuilder().startObject() - .array("long_values", 1L, 5L, 10L, 8L) - .array("int_values", 1, 5, 10, 8) - .array("short_values", 1, 5, 10, 8) - .array("byte_values", 1, 5, 10, 8) - .array("float_values", 1f, 5f, 10f, 8f) - .array("double_values", 1d, 5d, 10d, 8d) - .array("string_values", "01", "05", "10", "08") - .endObject()).execute().actionGet(); - client().prepareIndex("test", "type1", Integer.toString(2)).setSource(jsonBuilder().startObject() - .array("long_values", 11L, 15L, 20L, 7L) - .array("int_values", 11, 15, 20, 7) - .array("short_values", 11, 15, 20, 7) - .array("byte_values", 11, 15, 20, 7) - .array("float_values", 11f, 15f, 20f, 7f) - .array("double_values", 11d, 15d, 20d, 7d) - .array("string_values", "11", "15", "20", "07") - .endObject()).execute().actionGet(); - client().prepareIndex("test", "type1", Integer.toString(3)).setSource(jsonBuilder().startObject() - .array("long_values", 2L, 1L, 3L, -4L) - .array("int_values", 2, 1, 3, -4) - .array("short_values", 2, 1, 3, -4) - .array("byte_values", 2, 1, 3, -4) - .array("float_values", 2f, 1f, 3f, -4f) - .array("double_values", 2d, 1d, 3d, -4d) - .array("string_values", "02", "01", "03", "!4") - .endObject()).execute().actionGet(); - - refresh(); - - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("long_values", SortOrder.ASC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(-4L)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(1L)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(7L)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("long_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(20L)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(10L)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(3L)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort(SortBuilders.fieldSort("long_values").order(SortOrder.DESC).sortMode("sum")) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).longValue(), equalTo(53L)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).longValue(), equalTo(24L)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).longValue(), equalTo(2L)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("int_values", SortOrder.ASC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(-4)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(1)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(7)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("int_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(20)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(10)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(3)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("short_values", SortOrder.ASC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(-4)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(1)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(7)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("short_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(20)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(10)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(3)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("byte_values", SortOrder.ASC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(-4)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(1)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(7)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("byte_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).intValue(), equalTo(20)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).intValue(), equalTo(10)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).intValue(), equalTo(3)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("float_values", SortOrder.ASC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).floatValue(), equalTo(-4f)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).floatValue(), equalTo(1f)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).floatValue(), equalTo(7f)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("float_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).floatValue(), equalTo(20f)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).floatValue(), equalTo(10f)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).floatValue(), equalTo(3f)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("double_values", SortOrder.ASC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), equalTo(-4d)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), equalTo(1d)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), equalTo(7d)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("double_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), equalTo(20d)); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), equalTo(10d)); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); - assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), equalTo(3d)); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("string_values", SortOrder.ASC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3))); - assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("!4")); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("01")); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2))); - assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("07")); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(10) - .addSort("string_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L)); - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20")); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10")); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); - assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03")); - } - - public void testSortOnRareField() throws IOException { - assertAcked(prepareCreate("test") - .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties") - .startObject("string_values").field("type", "string").field("index", "not_analyzed").startObject("fielddata").field("format", random().nextBoolean() ? "doc_values" : null).endObject().endObject() - .endObject().endObject().endObject())); - ensureGreen(); - client().prepareIndex("test", "type1", Integer.toString(1)).setSource(jsonBuilder().startObject() - .array("string_values", "01", "05", "10", "08") - .endObject()).execute().actionGet(); - - - refresh(); - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(3) - .addSort("string_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().hits().length, equalTo(1)); - - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(1))); - assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("10")); - - client().prepareIndex("test", "type1", Integer.toString(2)).setSource(jsonBuilder().startObject() - .array("string_values", "11", "15", "20", "07") - .endObject()).execute().actionGet(); - for (int i = 0; i < 15; i++) { - client().prepareIndex("test", "type1", Integer.toString(300 + i)).setSource(jsonBuilder().startObject() - .array("some_other_field", "foobar") - .endObject()).execute().actionGet(); + for (int i = 0; i < 10; i++) { + client().prepareIndex("test", "test", Integer.toString(i)) + .setSource(jsonBuilder().startObject().field("value", "" + i).endObject()).execute().actionGet(); } refresh(); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(2) - .addSort("string_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().hits().length, equalTo(2)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20")); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10")); - - - client().prepareIndex("test", "type1", Integer.toString(3)).setSource(jsonBuilder().startObject() - .array("string_values", "02", "01", "03", "!4") - .endObject()).execute().actionGet(); - for (int i = 0; i < 15; i++) { - client().prepareIndex("test", "type1", Integer.toString(300 + i)).setSource(jsonBuilder().startObject() - .array("some_other_field", "foobar") - .endObject()).execute().actionGet(); - } - refresh(); - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(3) - .addSort("string_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20")); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10")); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); - assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03")); - - for (int i = 0; i < 15; i++) { - client().prepareIndex("test", "type1", Integer.toString(300 + i)).setSource(jsonBuilder().startObject() - .array("some_other_field", "foobar") - .endObject()).execute().actionGet(); - refresh(); - } - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(3) - .addSort("string_values", SortOrder.DESC) - .execute().actionGet(); - - assertThat(searchResponse.getHits().hits().length, equalTo(3)); - - assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2))); - assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20")); - - assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1))); - assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10")); - - assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3))); - assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03")); - } - - public void testSortMetaField() throws Exception { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", true).endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test") - .addMapping("type", mapping)); - ensureGreen(); - final int numDocs = randomIntBetween(10, 20); - IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; - for (int i = 0; i < numDocs; ++i) { - indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000))).setSource(); - } - indexRandom(true, indexReqs); - - SortOrder order = randomFrom(SortOrder.values()); - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(randomIntBetween(1, numDocs + 5)) - .addSort("_uid", order) - .execute().actionGet(); + SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()) + .addSort(SortBuilders.scriptSort(new Script("\u0027\u0027"), "string")).setSize(10).execute().actionGet(); assertNoFailures(searchResponse); - SearchHit[] hits = searchResponse.getHits().hits(); - BytesRef previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM; - for (int i = 0; i < hits.length; ++i) { - final BytesRef uid = new BytesRef(Uid.createUid(hits[i].type(), hits[i].id())); - assertThat(previous, order == SortOrder.ASC ? lessThan(uid) : greaterThan(uid)); - previous = uid; - } - - /* - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(randomIntBetween(1, numDocs + 5)) - .addSort("_id", order) - .execute().actionGet(); - assertNoFailures(searchResponse); - hits = searchResponse.getHits().hits(); - previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM; - for (int i = 0; i < hits.length; ++i) { - final BytesRef id = new BytesRef(Uid.createUid(hits[i].type(), hits[i].id())); - assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id)); - previous = id; - }*/ - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(randomIntBetween(1, numDocs + 5)) - .addSort("_timestamp", order) - .addField("_timestamp") - .execute().actionGet(); - assertNoFailures(searchResponse); - hits = searchResponse.getHits().hits(); - Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE; - for (int i = 0; i < hits.length; ++i) { - SearchHitField timestampField = hits[i].getFields().get("_timestamp"); - Long timestamp = timestampField.getValue(); - assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp)); - previousTs = timestamp; - } - } - - /** - * Test case for issue 6150: https://github.com/elasticsearch/elasticsearch/issues/6150 - */ - public void testNestedSort() throws IOException, InterruptedException, ExecutionException { - assertAcked(prepareCreate("test") - .addMapping("type", - XContentFactory.jsonBuilder() - .startObject() - .startObject("type") - .startObject("properties") - .startObject("nested") - .field("type", "nested") - .startObject("properties") - .startObject("foo") - .field("type", "string") - .startObject("fields") - .startObject("sub") - .field("type", "string") - .field("index", "not_analyzed") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .endObject())); - ensureGreen(); - - client().prepareIndex("test", "type", "1").setSource(jsonBuilder().startObject() - .startObject("nested") - .field("foo", "bar bar") - .endObject() - .endObject()).execute().actionGet(); - refresh(); - - // We sort on nested field - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("nested.foo").setNestedPath("nested").order(SortOrder.DESC)) - .execute().actionGet(); - assertNoFailures(searchResponse); - SearchHit[] hits = searchResponse.getHits().hits(); - for (int i = 0; i < hits.length; ++i) { - assertThat(hits[i].getSortValues().length, is(1)); - Object o = hits[i].getSortValues()[0]; - assertThat(o, notNullValue()); - Text text = (Text) o; - assertThat(text.string(), is("bar")); - } - - - // We sort on nested sub field - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .addSort(SortBuilders.fieldSort("nested.foo.sub").setNestedPath("nested").order(SortOrder.DESC)) - .execute().actionGet(); - assertNoFailures(searchResponse); - hits = searchResponse.getHits().hits(); - for (int i = 0; i < hits.length; ++i) { - assertThat(hits[i].getSortValues().length, is(1)); - Object o = hits[i].getSortValues()[0]; - assertThat(o, notNullValue()); - Text text = (Text) o; - assertThat(text.string(), is("bar bar")); - } - } - - public void testSortDuelBetweenSingleShardAndMultiShardIndex() throws Exception { - String sortField = "sortField"; - assertAcked(prepareCreate("test1") - .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(2, maximumNumberOfShards())) - .addMapping("type", sortField, "type=long").get()); - assertAcked(prepareCreate("test2") - .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .addMapping("type", sortField, "type=long").get()); - - for (String index : new String[]{"test1", "test2"}) { - List docs = new ArrayList<>(); - for (int i = 0; i < 256; i++) { - docs.add(client().prepareIndex(index, "type", Integer.toString(i)).setSource(sortField, i)); - } - indexRandom(true, docs); - } - - ensureSearchable("test1", "test2"); - SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC; - int from = between(0, 256); - int size = between(0, 256); - SearchResponse multiShardResponse = client().prepareSearch("test1").setFrom(from).setSize(size).addSort(sortField, order).get(); - assertNoFailures(multiShardResponse); - SearchResponse singleShardResponse = client().prepareSearch("test2").setFrom(from).setSize(size).addSort(sortField, order).get(); - assertNoFailures(singleShardResponse); - - assertThat(multiShardResponse.getHits().totalHits(), equalTo(singleShardResponse.getHits().totalHits())); - assertThat(multiShardResponse.getHits().getHits().length, equalTo(singleShardResponse.getHits().getHits().length)); - for (int i = 0; i < multiShardResponse.getHits().getHits().length; i++) { - assertThat(multiShardResponse.getHits().getAt(i).sortValues()[0], equalTo(singleShardResponse.getHits().getAt(i).sortValues()[0])); - assertThat(multiShardResponse.getHits().getAt(i).id(), equalTo(singleShardResponse.getHits().getAt(i).id())); - } } } From 96bcb47fc9260d46bed5aeeecbdd7e71cf0b5c6b Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 8 Feb 2016 12:09:15 +0100 Subject: [PATCH 03/14] Detach QueryShardContext from IndexShard and remove obsolete threadlocals IndexShard currently holds an arbitraritly used `getQueryShardContext` that comes out of a ThreadLocal. It's usage is undefined and arbitraty since there is also such a method with different semantics on `IndexService` This commit removes the threadLocal on IndexShard as well as on the context itself. It's types are now a member and the QueryShardContext lifecycle is managed byt SearchContext which passes the types on from the SearchRequest. --- .../query/TransportValidateQueryAction.java | 3 +- .../explain/TransportExplainAction.java | 2 +- .../metadata/MetaDataCreateIndexService.java | 3 +- .../metadata/MetaDataIndexAliasesService.java | 2 +- .../org/elasticsearch/index/IndexService.java | 7 +- .../elasticsearch/index/SearchSlowLog.java | 4 +- .../index/mapper/DocumentMapperParser.java | 1 - .../index/query/HasChildQueryBuilder.java | 7 +- .../index/query/HasParentQueryBuilder.java | 6 +- .../index/query/QueryShardContext.java | 22 ++--- .../support/NestedInnerQueryParseSupport.java | 2 +- .../elasticsearch/index/shard/IndexShard.java | 27 +----- .../percolator/PercolateContext.java | 46 ++++------- .../percolator/PercolateDocumentParser.java | 9 +- .../percolator/PercolatorService.java | 9 +- .../elasticsearch/search/SearchService.java | 7 +- .../search/aggregations/AggregationPhase.java | 2 +- .../bucket/filter/FilterParser.java | 2 +- .../bucket/filters/FiltersParser.java | 4 +- .../SignificantTermsParametersParser.java | 2 +- .../innerhits/InnerHitsParseElement.java | 2 +- .../highlight/HighlighterParseElement.java | 2 +- .../search/internal/DefaultSearchContext.java | 25 +++--- .../internal/FilteredSearchContext.java | 15 ++-- .../search/internal/SearchContext.java | 8 +- .../search/lookup/DocLookup.java | 4 + .../search/query/PostFilterParseElement.java | 4 +- .../search/query/QueryParseElement.java | 4 +- .../search/rescore/RescoreParseElement.java | 4 +- .../suggest/phrase/PhraseSuggester.java | 12 ++- .../phrase/PhraseSuggestionContext.java | 1 - .../org/elasticsearch/ESExceptionTests.java | 2 +- .../ExceptionSerializationTests.java | 2 +- .../index/IndexServiceTests.java | 25 +++--- .../mapper/date/SimpleDateMappingTests.java | 4 +- .../SimpleExternalMappingTests.java | 6 +- .../internal/FieldNamesFieldMapperTests.java | 4 +- .../index/query/AbstractQueryTestCase.java | 5 +- .../query/HasChildQueryBuilderTests.java | 13 +-- .../query/HasParentQueryBuilderTests.java | 11 +-- .../index/query/NestedQueryBuilderTests.java | 82 +++++++++---------- .../query/ParentIdQueryBuilderTests.java | 4 +- .../query/plugin/CustomQueryParserIT.java | 2 +- .../index/search/MultiMatchQueryTests.java | 2 +- .../percolator/PercolatorIT.java | 7 +- .../geogrid/GeoHashGridParserTests.java | 12 +-- .../SignificanceHeuristicTests.java | 5 ++ .../search/fetch/FieldDataFieldsTests.java | 6 +- .../search/query/QueryPhaseTests.java | 6 +- .../search/sort/SortParserTests.java | 2 +- .../murmur3/Murmur3FieldMapperTests.java | 6 +- .../index/mapper/size/SizeMappingTests.java | 4 +- .../elasticsearch/test/TestSearchContext.java | 31 +++---- 53 files changed, 215 insertions(+), 274 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 72cbe37c919..00f34f486e8 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -164,7 +164,8 @@ public class TransportValidateQueryAction extends TransportBroadcastAction queryShardContextSupplier; diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java index 3589215d5f8..c84883fe737 100644 --- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.join.JoinUtil; import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.search.Queries; @@ -205,13 +206,15 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder typesContext = new ThreadLocal<>(); private final MapperService mapperService; private final ScriptService scriptService; private final SimilarityService similarityService; @@ -82,23 +82,14 @@ public class QueryShardContext { private final IndexFieldDataService indexFieldDataService; private final IndexSettings indexSettings; private final Client client; + private String[] types = Strings.EMPTY_ARRAY; - public static void setTypes(String[] types) { - typesContext.set(types); + public void setTypes(String... types) { + this.types = types; } - public static String[] getTypes() { - return typesContext.get(); - } - - public static String[] setTypesWithPrevious(String... types) { - String[] old = typesContext.get(); - setTypes(types); - return old; - } - - public static void removeTypes() { - typesContext.remove(); + public String[] getTypes() { + return types; } private final Map namedQueries = new HashMap<>(); @@ -126,6 +117,7 @@ public class QueryShardContext { public QueryShardContext(QueryShardContext source) { this(source.indexSettings, source.client, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.indicesQueriesRegistry); + this.types = source.getTypes(); } diff --git a/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java b/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java index 8504ff5ccec..890961dd2a2 100644 --- a/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java +++ b/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java @@ -61,7 +61,7 @@ public class NestedInnerQueryParseSupport { private ObjectMapper parentObjectMapper; public NestedInnerQueryParseSupport(XContentParser parser, SearchContext searchContext) { - shardContext = searchContext.indexShard().getQueryShardContext(); + shardContext = searchContext.getQueryShardContext(); parseContext = shardContext.parseContext(); shardContext.reset(parser); diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 705d4d5aa5a..0ca8ea6926f 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -27,7 +27,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.UsageTrackingQueryCachingPolicy; import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.util.CloseableThreadLocal; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.ThreadInterruptedException; import org.elasticsearch.ElasticsearchException; @@ -158,7 +157,6 @@ public class IndexShard extends AbstractIndexShardComponent { private final IndicesQueryCache indicesQueryCache; private final IndexEventListener indexEventListener; private final IndexSettings idxSettings; - private final NodeServicesProvider provider; /** How many bytes we are currently moving to disk, via either IndexWriter.flush or refresh. IndexingMemoryController polls this * across all shards to decide if throttling is necessary because moving bytes to disk is falling behind vs incoming documents @@ -253,9 +251,9 @@ public class IndexShard extends AbstractIndexShardComponent { this.engineConfig = newEngineConfig(translogConfig, cachingPolicy); this.suspendableRefContainer = new SuspendableRefContainer(); - this.provider = provider; this.searcherWrapper = indexSearcherWrapper; - this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, newQueryShardContext()); + QueryShardContext queryShardContext = new QueryShardContext(idxSettings, provider.getClient(), indexCache.bitsetFilterCache(), indexFieldDataService, mapperService, similarityService, provider.getScriptService(), provider.getIndicesQueriesRegistry()); + this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, queryShardContext); } public Store store() { @@ -814,7 +812,7 @@ public class IndexShard extends AbstractIndexShardComponent { engine.flushAndClose(); } } finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times - IOUtils.close(engine, percolatorQueriesRegistry, queryShardContextCache); + IOUtils.close(engine, percolatorQueriesRegistry); } } } @@ -1499,25 +1497,6 @@ public class IndexShard extends AbstractIndexShardComponent { } } - private CloseableThreadLocal queryShardContextCache = new CloseableThreadLocal() { - // TODO We should get rid of this threadlocal but I think it should be a sep change - @Override - protected QueryShardContext initialValue() { - return newQueryShardContext(); - } - }; - - private QueryShardContext newQueryShardContext() { - return new QueryShardContext(idxSettings, provider.getClient(), indexCache.bitsetFilterCache(), indexFieldDataService, mapperService, similarityService, provider.getScriptService(), provider.getIndicesQueriesRegistry()); - } - - /** - * Returns a threadlocal QueryShardContext for this shard. - */ - public QueryShardContext getQueryShardContext() { - return queryShardContextCache.get(); - } - EngineFactory getEngineFactory() { return engineFactory; } diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java b/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java index 2ce3236a527..4c7f22e3922 100644 --- a/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java +++ b/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.percolator; -import com.carrotsearch.hppc.ObjectObjectAssociativeContainer; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; @@ -32,7 +31,6 @@ import org.elasticsearch.action.percolate.PercolateShardRequest; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cache.recycler.PageCacheRecycler; import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.util.BigArrays; @@ -46,6 +44,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.script.ScriptService; @@ -73,11 +72,11 @@ import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rescore.RescoreSearchContext; import org.elasticsearch.search.suggest.SuggestionSearchContext; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; /** */ @@ -99,14 +98,11 @@ public class PercolateContext extends SearchContext { private final long originNanoTime = System.nanoTime(); private final long startTime; private final boolean onlyCount; - private String[] types; - private Engine.Searcher docSearcher; private Engine.Searcher engineSearcher; private ContextIndexSearcher searcher; private SearchContextHighlight highlight; - private SearchLookup searchLookup; private ParsedQuery parsedQuery; private Query query; private Query percolateQuery; @@ -115,7 +111,9 @@ public class PercolateContext extends SearchContext { private QuerySearchResult querySearchResult; private Sort sort; private final Map subPhaseContexts = new HashMap<>(); + private final QueryShardContext queryShardContext; private final Map, Collector> queryCollectors = new HashMap<>(); + private SearchLookup searchLookup; public PercolateContext(PercolateShardRequest request, SearchShardTarget searchShardTarget, IndexShard indexShard, IndexService indexService, PageCacheRecycler pageCacheRecycler, @@ -126,7 +124,6 @@ public class PercolateContext extends SearchContext { this.fieldDataService = indexService.fieldData(); this.mapperService = indexService.mapperService(); this.searchShardTarget = searchShardTarget; - this.types = new String[]{request.documentType()}; this.pageCacheRecycler = pageCacheRecycler; this.bigArrays = bigArrays.withCircuitBreaking(); this.querySearchResult = new QuerySearchResult(0, searchShardTarget); @@ -137,6 +134,8 @@ public class PercolateContext extends SearchContext { this.aliasFilter = aliasFilter; this.startTime = request.getStartTime(); this.onlyCount = request.onlyCount(); + queryShardContext = indexService.newQueryShardContext(); + queryShardContext.setTypes(request.documentType()); } // for testing: @@ -154,6 +153,7 @@ public class PercolateContext extends SearchContext { this.startTime = 0; this.numberOfShards = 0; this.onlyCount = true; + queryShardContext = new QueryShardContext(mapperService.getIndexSettings(), null, null, null, mapperService, null, null, null); } public IndexSearcher docSearcher() { @@ -162,10 +162,10 @@ public class PercolateContext extends SearchContext { public void initialize(Engine.Searcher docSearcher, ParsedDocument parsedDocument) { this.docSearcher = docSearcher; - IndexReader indexReader = docSearcher.reader(); LeafReaderContext atomicReaderContext = indexReader.leaves().get(0); - LeafSearchLookup leafLookup = lookup().getLeafSearchLookup(atomicReaderContext); + this.searchLookup = new SearchLookup(mapperService(), fieldData(), queryShardContext.getTypes()); + LeafSearchLookup leafLookup = searchLookup.getLeafSearchLookup(atomicReaderContext); leafLookup.setDocument(0); leafLookup.source().setSource(parsedDocument.source()); @@ -232,10 +232,10 @@ public class PercolateContext extends SearchContext { @Override public SearchLookup lookup() { - if (searchLookup == null) { - searchLookup = new SearchLookup(mapperService(), fieldData(), types); - } - return searchLookup; + // we cache this since it's really just a single document lookup - check the init method for details + assert searchLookup != null : "context is not initialized"; + assert Arrays.equals(searchLookup.doc().getTypes(), getQueryShardContext().getTypes()) : "types mismatch - can't return lookup"; + return this.searchLookup; } @Override @@ -265,16 +265,6 @@ public class PercolateContext extends SearchContext { return query; } - @Override - public String[] types() { - return types; - } - - public void types(String[] types) { - this.types = types; - searchLookup = new SearchLookup(mapperService(), fieldData(), types); - } - @Override public IndexFieldDataService fieldData() { return fieldDataService; @@ -341,11 +331,6 @@ public class PercolateContext extends SearchContext { return numberOfShards; } - @Override - public boolean hasTypes() { - throw new UnsupportedOperationException(); - } - @Override public float queryBoost() { throw new UnsupportedOperationException(); @@ -684,6 +669,11 @@ public class PercolateContext extends SearchContext { return queryCollectors; } + @Override + public QueryShardContext getQueryShardContext() { + return queryShardContext; + } + @Override public Profilers getProfilers() { throw new UnsupportedOperationException(); diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java b/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java index 946d30edcc4..2192876266a 100644 --- a/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java +++ b/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java @@ -23,8 +23,6 @@ import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.percolate.PercolateShardRequest; -import org.elasticsearch.cluster.action.index.MappingUpdatedAction; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -35,7 +33,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapperForType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.aggregations.AggregationPhase; @@ -78,8 +75,8 @@ public class PercolateDocumentParser { // Some queries (function_score query when for decay functions) rely on a SearchContext being set: // We switch types because this context needs to be in the context of the percolate queries in the shard and // not the in memory percolate doc - String[] previousTypes = context.types(); - context.types(new String[]{PercolatorService.TYPE_NAME}); + String[] previousTypes = context.getQueryShardContext().getTypes(); + context.getQueryShardContext().setTypes(PercolatorService.TYPE_NAME); try (XContentParser parser = XContentFactory.xContent(source).createParser(source);) { String currentFieldName = null; XContentParser.Token token; @@ -176,7 +173,7 @@ public class PercolateDocumentParser { } catch (Throwable e) { throw new ElasticsearchParseException("failed to parse request", e); } finally { - context.types(previousTypes); + context.getQueryShardContext().setTypes(previousTypes); } if (request.docSource() != null && request.docSource().length() != 0) { diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java index 95cdff02ad9..c36de9c6e4c 100644 --- a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java +++ b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java @@ -169,8 +169,8 @@ public class PercolatorService extends AbstractComponent implements Releasable { } public PercolateShardResponse percolate(PercolateShardRequest request) throws IOException { - IndexService percolateIndexService = indicesService.indexServiceSafe(request.shardId().getIndex()); - IndexShard indexShard = percolateIndexService.getShard(request.shardId().id()); + final IndexService percolateIndexService = indicesService.indexServiceSafe(request.shardId().getIndex()); + final IndexShard indexShard = percolateIndexService.getShard(request.shardId().id()); indexShard.readAllowed(); // check if we can read the shard... PercolatorQueriesRegistry percolateQueryRegistry = indexShard.percolateRegistry(); percolateQueryRegistry.prePercolate(); @@ -183,7 +183,7 @@ public class PercolatorService extends AbstractComponent implements Releasable { indexShard.shardId().getIndex().getName(), request.indices() ); - Query aliasFilter = percolateIndexService.aliasFilter(indexShard.getQueryShardContext(), filteringAliases); + Query aliasFilter = percolateIndexService.aliasFilter(percolateIndexService.newQueryShardContext(), filteringAliases); SearchShardTarget searchShardTarget = new SearchShardTarget(clusterService.localNode().id(), request.shardId().getIndex(), request.shardId().id()); final PercolateContext context = new PercolateContext( @@ -191,8 +191,7 @@ public class PercolatorService extends AbstractComponent implements Releasable { ); SearchContext.setCurrent(context); try { - ParsedDocument parsedDocument = percolateDocumentParser.parse(request, context, percolateIndexService.mapperService(), percolateIndexService.getQueryShardContext()); - + ParsedDocument parsedDocument = percolateDocumentParser.parse(request, context, percolateIndexService.mapperService(), context.getQueryShardContext()); if (context.searcher().getIndexReader().maxDoc() == 0) { return new PercolateShardResponse(Lucene.EMPTY_TOP_DOCS, Collections.emptyMap(), Collections.emptyMap(), context); } diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java index 0da838a799b..e0b30a2e346 100644 --- a/core/src/main/java/org/elasticsearch/search/SearchService.java +++ b/core/src/main/java/org/elasticsearch/search/SearchService.java @@ -646,8 +646,7 @@ public class SearchService extends AbstractLifecycleComponent imp if (source == null) { return; } - final IndexShard indexShard = context.indexShard(); - QueryShardContext queryShardContext = indexShard.getQueryShardContext(); + QueryShardContext queryShardContext = context.getQueryShardContext(); context.from(source.from()); context.size(source.size()); ObjectFloatHashMap indexBoostMap = source.indexBoost(); @@ -751,7 +750,7 @@ public class SearchService extends AbstractLifecycleComponent imp if (source.rescores() != null) { try { for (RescoreBuilder rescore : source.rescores()) { - context.addRescore(rescore.build(context.indexShard().getQueryShardContext())); + context.addRescore(rescore.build(context.getQueryShardContext())); } } catch (IOException e) { throw new SearchContextException(context, "failed to create RescoreSearchContext", e); @@ -776,7 +775,7 @@ public class SearchService extends AbstractLifecycleComponent imp if (source.highlighter() != null) { HighlightBuilder highlightBuilder = source.highlighter(); try { - context.highlight(highlightBuilder.build(context.indexShard().getQueryShardContext())); + context.highlight(highlightBuilder.build(context.getQueryShardContext())); } catch (IOException e) { throw new SearchContextException(context, "failed to create SearchContextHighlighter", e); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java index 0681996e3ec..050ffa61ebe 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java @@ -124,7 +124,7 @@ public class AggregationPhase implements SearchPhase { if (!globals.isEmpty()) { BucketCollector globalsCollector = BucketCollector.wrap(globals); Query query = Queries.newMatchAllQuery(); - Query searchFilter = context.searchFilter(context.types()); + Query searchFilter = context.searchFilter(context.getQueryShardContext().getTypes()); if (searchFilter != null) { BooleanQuery filtered = new BooleanQuery.Builder() diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterParser.java index 48702dab230..d532dc4c922 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterParser.java @@ -39,7 +39,7 @@ public class FilterParser implements Aggregator.Parser { @Override public AggregatorFactory parse(String aggregationName, XContentParser parser, SearchContext context) throws IOException { - ParsedQuery filter = context.indexShard().getQueryShardContext().parseInnerFilter(parser); + ParsedQuery filter = context.getQueryShardContext().parseInnerFilter(parser); return new FilterAggregator.Factory(aggregationName, filter == null ? new MatchAllDocsQuery() : filter.query()); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersParser.java index 8ed37078012..b7e5e6ff35c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersParser.java @@ -82,7 +82,7 @@ public class FiltersParser implements Aggregator.Parser { if (token == XContentParser.Token.FIELD_NAME) { key = parser.currentName(); } else { - ParsedQuery filter = context.indexShard().getQueryShardContext().parseInnerFilter(parser); + ParsedQuery filter = context.getQueryShardContext().parseInnerFilter(parser); filters.add(new FiltersAggregator.KeyedFilter(key, filter == null ? Queries.newMatchAllQuery() : filter.query())); } } @@ -95,7 +95,7 @@ public class FiltersParser implements Aggregator.Parser { keyed = false; int idx = 0; while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - ParsedQuery filter = context.indexShard().getQueryShardContext().parseInnerFilter(parser); + ParsedQuery filter = context.getQueryShardContext().parseInnerFilter(parser); filters.add(new FiltersAggregator.KeyedFilter(String.valueOf(idx), filter == null ? Queries.newMatchAllQuery() : filter.query())); idx++; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParametersParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParametersParser.java index 020229867f6..3019765e385 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParametersParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParametersParser.java @@ -66,7 +66,7 @@ public class SignificantTermsParametersParser extends AbstractTermsParametersPar if (significanceHeuristicParser != null) { significanceHeuristic = significanceHeuristicParser.parse(parser, context.parseFieldMatcher(), context); } else if (context.parseFieldMatcher().match(currentFieldName, BACKGROUND_FILTER)) { - filter = context.indexShard().getQueryShardContext().parseInnerFilter(parser).query(); + filter = context.getQueryShardContext().parseInnerFilter(parser).query(); } else { throw new SearchParseException(context, "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "].", parser.getTokenLocation()); diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java index d813dea1deb..077268ac52f 100644 --- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsParseElement.java @@ -59,7 +59,7 @@ public class InnerHitsParseElement implements SearchParseElement { @Override public void parse(XContentParser parser, SearchContext searchContext) throws Exception { - QueryShardContext context = searchContext.indexShard().getQueryShardContext(); + QueryShardContext context = searchContext.getQueryShardContext(); context.reset(parser); Map topLevelInnerHits = parseInnerHits(parser, context, searchContext); if (topLevelInnerHits != null) { diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java index 38534ba0ff3..b774acad2ff 100644 --- a/core/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlighterParseElement.java @@ -54,7 +54,7 @@ public class HighlighterParseElement implements SearchParseElement { @Override public void parse(XContentParser parser, SearchContext context) throws Exception { try { - context.highlight(parse(parser, context.indexShard().getQueryShardContext())); + context.highlight(parse(parser, context.getQueryShardContext())); } catch (IllegalArgumentException ex) { throw new SearchParseException(context, "Error while trying to parse Highlighter element in request", parser.getTokenLocation()); } diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java index 51e9ea4f5d1..32aa45f00f3 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java @@ -29,7 +29,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.util.Counter; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cache.recycler.PageCacheRecycler; @@ -53,6 +52,7 @@ import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.script.ScriptService; @@ -62,7 +62,6 @@ import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseContext; -import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; import org.elasticsearch.search.fetch.script.ScriptFieldsContext; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.SearchContextHighlight; @@ -150,6 +149,7 @@ public class DefaultSearchContext extends SearchContext { private final Map subPhaseContexts = new HashMap<>(); private final Map, Collector> queryCollectors = new HashMap<>(); + private final QueryShardContext queryShardContext; public DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget, Engine.Searcher engineSearcher, IndexService indexService, IndexShard indexShard, @@ -175,6 +175,8 @@ public class DefaultSearchContext extends SearchContext { this.searcher = new ContextIndexSearcher(engineSearcher, indexService.cache().query(), indexShard.getQueryCachingPolicy()); this.timeEstimateCounter = timeEstimateCounter; this.timeoutInMillis = timeout.millis(); + queryShardContext = indexService.newQueryShardContext(); + queryShardContext.setTypes(request.types()); } @Override @@ -206,7 +208,7 @@ public class DefaultSearchContext extends SearchContext { } // initialize the filtering alias based on the provided filters - aliasFilter = indexService.aliasFilter(indexShard.getQueryShardContext(), request.filteringAliases()); + aliasFilter = indexService.aliasFilter(queryShardContext, request.filteringAliases()); if (query() == null) { parsedQuery(ParsedQuery.parsedMatchAllQuery()); @@ -223,7 +225,7 @@ public class DefaultSearchContext extends SearchContext { } private ParsedQuery buildFilteredQuery() { - Query searchFilter = searchFilter(types()); + Query searchFilter = searchFilter(queryShardContext.getTypes()); if (searchFilter == null) { return originalQuery; } @@ -312,16 +314,6 @@ public class DefaultSearchContext extends SearchContext { return request.numberOfShards(); } - @Override - public boolean hasTypes() { - return request.types() != null && request.types().length > 0; - } - - @Override - public String[] types() { - return request.types(); - } - @Override public float queryBoost() { return queryBoost; @@ -765,6 +757,11 @@ public class DefaultSearchContext extends SearchContext { return queryCollectors; } + @Override + public QueryShardContext getQueryShardContext() { + return queryShardContext; + } + @Override public Profilers getProfilers() { return profilers; diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 449e3207ad5..c47d32b9388 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.script.ScriptService; @@ -117,16 +118,6 @@ public abstract class FilteredSearchContext extends SearchContext { return in.numberOfShards(); } - @Override - public boolean hasTypes() { - return in.hasTypes(); - } - - @Override - public String[] types() { - return in.types(); - } - @Override public float queryBoost() { return in.queryBoost(); @@ -525,4 +516,8 @@ public abstract class FilteredSearchContext extends SearchContext { @Override public Map, Collector> queryCollectors() { return in.queryCollectors();} + @Override + public QueryShardContext getQueryShardContext() { + return in.getQueryShardContext(); + } } diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java index a93cde1a8a2..7bef5679511 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -72,12 +72,10 @@ public abstract class SearchContext implements Releasable { public static void setCurrent(SearchContext value) { current.set(value); - QueryShardContext.setTypes(value.types()); } public static void removeCurrent() { current.remove(); - QueryShardContext.removeTypes(); } public static SearchContext current() { @@ -134,10 +132,6 @@ public abstract class SearchContext implements Releasable { public abstract int numberOfShards(); - public abstract boolean hasTypes(); - - public abstract String[] types(); - public abstract float queryBoost(); public abstract SearchContext queryBoost(float queryBoost); @@ -379,4 +373,6 @@ public abstract class SearchContext implements Releasable { CONTEXT } + public abstract QueryShardContext getQueryShardContext(); + } diff --git a/core/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/core/src/main/java/org/elasticsearch/search/lookup/DocLookup.java index b32a6aea203..3eea11a9f5c 100644 --- a/core/src/main/java/org/elasticsearch/search/lookup/DocLookup.java +++ b/core/src/main/java/org/elasticsearch/search/lookup/DocLookup.java @@ -51,4 +51,8 @@ public class DocLookup { public LeafDocLookup getLeafDocLookup(LeafReaderContext context) { return new LeafDocLookup(mapperService, fieldDataService, types, context); } + + public String[] getTypes() { + return types; + } } diff --git a/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java b/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java index 3f3093b6996..6995b6ff8a7 100644 --- a/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java @@ -30,9 +30,9 @@ public class PostFilterParseElement implements SearchParseElement { @Override public void parse(XContentParser parser, SearchContext context) throws Exception { - ParsedQuery postFilter = context.indexShard().getQueryShardContext().parseInnerFilter(parser); + ParsedQuery postFilter = context.getQueryShardContext().parseInnerFilter(parser); if (postFilter != null) { context.parsedPostFilter(postFilter); } } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java b/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java index 12a080acdea..094a29cd6b1 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java @@ -30,6 +30,6 @@ public class QueryParseElement implements SearchParseElement { @Override public void parse(XContentParser parser, SearchContext context) throws Exception { - context.parsedQuery(context.indexShard().getQueryShardContext().parse(parser)); + context.parsedQuery(context.getQueryShardContext().parse(parser)); } -} \ No newline at end of file +} diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescoreParseElement.java b/core/src/main/java/org/elasticsearch/search/rescore/RescoreParseElement.java index 149db6cec2c..702d3666692 100644 --- a/core/src/main/java/org/elasticsearch/search/rescore/RescoreParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/rescore/RescoreParseElement.java @@ -36,10 +36,10 @@ public class RescoreParseElement implements SearchParseElement { public void parse(XContentParser parser, SearchContext context) throws Exception { if (parser.currentToken() == XContentParser.Token.START_ARRAY) { while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - context.addRescore(parseSingleRescoreContext(parser, context.indexShard().getQueryShardContext())); + context.addRescore(parseSingleRescoreContext(parser, context.getQueryShardContext())); } } else { - context.addRescore(parseSingleRescoreContext(parser, context.indexShard().getQueryShardContext())); + context.addRescore(parseSingleRescoreContext(parser, context.getQueryShardContext())); } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index c7fa6fae302..7838eacd960 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.query.ParsedQuery; -import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.script.CompiledScript; import org.elasticsearch.script.ExecutableScript; @@ -65,7 +64,7 @@ public final class PhraseSuggester extends Suggester { /* * More Ideas: * - add ability to find whitespace problems -> we can build a poor mans decompounder with our index based on a automaton? - * - add ability to build different error models maybe based on a confusion matrix? + * - add ability to build different error models maybe based on a confusion matrix? * - try to combine a token with its subsequent token to find / detect word splits (optional) * - for this to work we need some way to defined the position length of a candidate * - phonetic filters could be interesting here too for candidate selection @@ -84,8 +83,8 @@ public final class PhraseSuggester extends Suggester { DirectSpellChecker directSpellChecker = SuggestUtils.getDirectSpellChecker(generator); Terms terms = MultiFields.getTerms(indexReader, generator.field()); if (terms != null) { - gens.add(new DirectCandidateGenerator(directSpellChecker, generator.field(), generator.suggestMode(), - indexReader, realWordErrorLikelihood, generator.size(), generator.preFilter(), generator.postFilter(), terms)); + gens.add(new DirectCandidateGenerator(directSpellChecker, generator.field(), generator.suggestMode(), + indexReader, realWordErrorLikelihood, generator.size(), generator.preFilter(), generator.postFilter(), terms)); } } final String suggestField = suggestion.getField(); @@ -119,8 +118,7 @@ public final class PhraseSuggester extends Suggester { final ExecutableScript executable = scriptService.executable(collateScript, vars); final BytesReference querySource = (BytesReference) executable.run(); IndexService indexService = indicesService.indexService(suggestion.getIndex()); - IndexShard shard = indexService.getShard(suggestion.getShard()); - final ParsedQuery parsedQuery = shard.getQueryShardContext().parse(querySource); + final ParsedQuery parsedQuery = indexService.newQueryShardContext().parse(querySource); collateMatch = Lucene.exists(searcher, parsedQuery.query()); } if (!collateMatch && !collatePrune) { @@ -152,7 +150,7 @@ public final class PhraseSuggester extends Suggester { ScriptService scriptService() { return scriptService; } - + @Override public SuggestContextParser getContextParser() { return new PhraseSuggestParser(this); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java index 04d06abbfd0..736b297fab7 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java @@ -42,7 +42,6 @@ class PhraseSuggestionContext extends SuggestionContext { private BytesRef preTag; private BytesRef postTag; private CompiledScript collateQueryScript; - private CompiledScript collateFilterScript; private Map collateScriptParams = new HashMap<>(1); private WordScorer.WordScorerFactory scorer; diff --git a/core/src/test/java/org/elasticsearch/ESExceptionTests.java b/core/src/test/java/org/elasticsearch/ESExceptionTests.java index 75a69cd3e55..ad3f0632019 100644 --- a/core/src/test/java/org/elasticsearch/ESExceptionTests.java +++ b/core/src/test/java/org/elasticsearch/ESExceptionTests.java @@ -194,7 +194,7 @@ public class ESExceptionTests extends ESTestCase { public void testToXContent() throws IOException { { - ElasticsearchException ex = new SearchParseException(new TestSearchContext(), "foo", new XContentLocation(1,0)); + ElasticsearchException ex = new SearchParseException(new TestSearchContext(null), "foo", new XContentLocation(1,0)); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); ex.toXContent(builder, PARAMS); diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 57b1fdac645..606ff09a60f 100644 --- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -439,7 +439,7 @@ public class ExceptionSerializationTests extends ESTestCase { } public void testSearchParseException() throws IOException { - SearchContext ctx = new TestSearchContext(); + SearchContext ctx = new TestSearchContext(null); SearchParseException ex = serialize(new SearchParseException(ctx, "foo", new XContentLocation(66, 666))); assertEquals("foo", ex.getMessage()); assertEquals(66, ex.getLineNumber()); diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java index 252fa9d3f4e..3a4020e4103 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -39,7 +39,6 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; -import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -87,17 +86,17 @@ public class IndexServiceTests extends ESSingleNodeTestCase { assertThat(indexService.getMetaData().getAliases().containsKey("dogs"), equalTo(true)); assertThat(indexService.getMetaData().getAliases().containsKey("turtles"), equalTo(false)); - assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "cats").toString(), equalTo("animal:cat")); - assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "cats", "dogs").toString(), equalTo("animal:cat animal:dog")); + assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "cats").toString(), equalTo("animal:cat")); + assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "cats", "dogs").toString(), equalTo("animal:cat animal:dog")); // Non-filtering alias should turn off all filters because filters are ORed - assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "all"), nullValue()); - assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "cats", "all"), nullValue()); - assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "all", "cats"), nullValue()); + assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "all"), nullValue()); + assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "cats", "all"), nullValue()); + assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "all", "cats"), nullValue()); add(indexService, "cats", filter(termQuery("animal", "feline"))); add(indexService, "dogs", filter(termQuery("animal", "canine"))); - assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:canine animal:feline")); + assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:canine animal:feline")); } public void testAliasFilters() throws Exception { @@ -107,14 +106,14 @@ public class IndexServiceTests extends ESSingleNodeTestCase { add(indexService, "cats", filter(termQuery("animal", "cat"))); add(indexService, "dogs", filter(termQuery("animal", "dog"))); - assertThat(indexService.aliasFilter(shard.getQueryShardContext()), nullValue()); - assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "dogs").toString(), equalTo("animal:dog")); - assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:dog animal:cat")); + assertThat(indexService.aliasFilter(indexService.newQueryShardContext()), nullValue()); + assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "dogs").toString(), equalTo("animal:dog")); + assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:dog animal:cat")); add(indexService, "cats", filter(termQuery("animal", "feline"))); add(indexService, "dogs", filter(termQuery("animal", "canine"))); - assertThat(indexService.aliasFilter(shard.getQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:canine animal:feline")); + assertThat(indexService.aliasFilter(indexService.newQueryShardContext(), "dogs", "cats").toString(), equalTo("animal:canine animal:feline")); } public void testRemovedAliasFilter() throws Exception { @@ -124,7 +123,7 @@ public class IndexServiceTests extends ESSingleNodeTestCase { add(indexService, "cats", filter(termQuery("animal", "cat"))); remove(indexService, "cats"); try { - indexService.aliasFilter(shard.getQueryShardContext(), "cats"); + indexService.aliasFilter(indexService.newQueryShardContext(), "cats"); fail("Expected InvalidAliasNameException"); } catch (InvalidAliasNameException e) { assertThat(e.getMessage(), containsString("Invalid alias name [cats]")); @@ -139,7 +138,7 @@ public class IndexServiceTests extends ESSingleNodeTestCase { add(indexService, "dogs", filter(termQuery("animal", "dog"))); try { - indexService.aliasFilter(shard.getQueryShardContext(), "unknown"); + indexService.aliasFilter(indexService.newQueryShardContext(), "unknown"); fail(); } catch (InvalidAliasNameException e) { // all is well diff --git a/core/src/test/java/org/elasticsearch/index/mapper/date/SimpleDateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/date/SimpleDateMappingTests.java index f9531c3aff3..b6a7dee23ba 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/date/SimpleDateMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/date/SimpleDateMappingTests.java @@ -260,7 +260,7 @@ public class SimpleDateMappingTests extends ESSingleNodeTestCase { NumericRangeQuery rangeQuery; try { - SearchContext.setCurrent(new TestSearchContext()); + SearchContext.setCurrent(new TestSearchContext(null)); rangeQuery = (NumericRangeQuery) defaultMapper.mappers().smartNameFieldMapper("date_field").fieldType().rangeQuery("10:00:00", "11:00:00", true, true).rewrite(null); } finally { SearchContext.removeCurrent(); @@ -286,7 +286,7 @@ public class SimpleDateMappingTests extends ESSingleNodeTestCase { NumericRangeQuery rangeQuery; try { - SearchContext.setCurrent(new TestSearchContext()); + SearchContext.setCurrent(new TestSearchContext(null)); rangeQuery = (NumericRangeQuery) defaultMapper.mappers().smartNameFieldMapper("date_field").fieldType().rangeQuery("Jan 02 10:00:00", "Jan 02 11:00:00", true, true).rewrite(null); } finally { SearchContext.removeCurrent(); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java index 96c099df6d3..bf92991e039 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java @@ -63,7 +63,7 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { Collections.singletonMap(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser())); DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::getQueryShardContext); + indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( XContentFactory.jsonBuilder().startObject().startObject("type") .startObject(ExternalMetadataMapper.CONTENT_TYPE) @@ -109,7 +109,7 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap()); DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::getQueryShardContext); + indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") @@ -168,7 +168,7 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap()); DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::getQueryShardContext); + indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java index e23ea15b64f..2646d944711 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java @@ -236,9 +236,9 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { IndicesModule indicesModule = new IndicesModule(); indicesModule.registerMetadataMapper("_dummy", new DummyMetadataFieldMapper.TypeParser()); final MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); - MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::getQueryShardContext); + MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), mapperService, - indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::getQueryShardContext); + indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); ParsedDocument parsedDocument = mapper.parse("index", "type", "id", new BytesArray("{}")); diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java index 2435a34080d..e8f20cb855c 100644 --- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java @@ -332,15 +332,14 @@ public abstract class AbstractQueryTestCase> } protected void setSearchContext(String[] types) { - TestSearchContext testSearchContext = new TestSearchContext(); - testSearchContext.setTypes(types); + TestSearchContext testSearchContext = new TestSearchContext(queryShardContext); + testSearchContext.getQueryShardContext().setTypes(types); SearchContext.setCurrent(testSearchContext); } @After public void afterTest() { clientInvocationHandler.delegate = null; - QueryShardContext.removeTypes(); SearchContext.removeCurrent(); } diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java index 8a2792eaf44..fb618868417 100644 --- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java @@ -84,7 +84,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase() { - @Override - public int compare(PercolateResponse.Match a, PercolateResponse.Match b) { - return a.getId().compareTo(b.getId()); - } - }); + Arrays.sort(matches, (a, b) -> a.getId().compareTo(b.getId())); assertThat(matches[0].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the lazy dog")); assertThat(matches[1].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the lazy dog")); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java index cd7dadd7eeb..ce26728b00b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java @@ -27,7 +27,7 @@ import org.elasticsearch.test.TestSearchContext; public class GeoHashGridParserTests extends ESTestCase { public void testParseValidFromInts() throws Exception { - SearchContext searchContext = new TestSearchContext(); + SearchContext searchContext = new TestSearchContext(null); int precision = randomIntBetween(1, 12); XContentParser stParser = JsonXContent.jsonXContent.createParser( "{\"field\":\"my_loc\", \"precision\":" + precision + ", \"size\": 500, \"shard_size\": 550}"); @@ -37,7 +37,7 @@ public class GeoHashGridParserTests extends ESTestCase { } public void testParseValidFromStrings() throws Exception { - SearchContext searchContext = new TestSearchContext(); + SearchContext searchContext = new TestSearchContext(null); int precision = randomIntBetween(1, 12); XContentParser stParser = JsonXContent.jsonXContent.createParser( "{\"field\":\"my_loc\", \"precision\":\"" + precision + "\", \"size\": \"500\", \"shard_size\": \"550\"}"); @@ -47,7 +47,7 @@ public class GeoHashGridParserTests extends ESTestCase { } public void testParseErrorOnNonIntPrecision() throws Exception { - SearchContext searchContext = new TestSearchContext(); + SearchContext searchContext = new TestSearchContext(null); XContentParser stParser = JsonXContent.jsonXContent.createParser("{\"field\":\"my_loc\", \"precision\":\"2.0\"}"); GeoHashGridParser parser = new GeoHashGridParser(); try { @@ -59,7 +59,7 @@ public class GeoHashGridParserTests extends ESTestCase { } public void testParseErrorOnBooleanPrecision() throws Exception { - SearchContext searchContext = new TestSearchContext(); + SearchContext searchContext = new TestSearchContext(null); XContentParser stParser = JsonXContent.jsonXContent.createParser("{\"field\":\"my_loc\", \"precision\":false}"); GeoHashGridParser parser = new GeoHashGridParser(); try { @@ -71,7 +71,7 @@ public class GeoHashGridParserTests extends ESTestCase { } public void testParseErrorOnPrecisionOutOfRange() throws Exception { - SearchContext searchContext = new TestSearchContext(); + SearchContext searchContext = new TestSearchContext(null); XContentParser stParser = JsonXContent.jsonXContent.createParser("{\"field\":\"my_loc\", \"precision\":\"13\"}"); GeoHashGridParser parser = new GeoHashGridParser(); try { @@ -81,4 +81,4 @@ public class GeoHashGridParserTests extends ESTestCase { assertEquals("Invalid geohash aggregation precision of 13. Must be between 1 and 12.", ex.getMessage()); } } -} \ No newline at end of file +} diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java index 59c110e7744..0a094c3b2d8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java @@ -69,6 +69,11 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; */ public class SignificanceHeuristicTests extends ESTestCase { static class SignificantTermsTestSearchContext extends TestSearchContext { + + public SignificantTermsTestSearchContext() { + super(null); + } + @Override public int numberOfShards() { return 1; diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java b/core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java index 7405ceef5a5..ba7f70c63a7 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java @@ -40,7 +40,7 @@ public class FieldDataFieldsTests extends ESTestCase { parser.nextToken(); parser.nextToken(); parser.nextToken(); - SearchContext context = new TestSearchContext(); + SearchContext context = new TestSearchContext(null); parseElement.parse(parser, context); } @@ -52,7 +52,7 @@ public class FieldDataFieldsTests extends ESTestCase { parser.nextToken(); parser.nextToken(); parser.nextToken(); - SearchContext context = new TestSearchContext(); + SearchContext context = new TestSearchContext(null); parseElement.parse(parser, context); } @@ -69,7 +69,7 @@ public class FieldDataFieldsTests extends ESTestCase { parser.nextToken(); parser.nextToken(); parser.nextToken(); - SearchContext context = new TestSearchContext(); + SearchContext context = new TestSearchContext(null); try { parseElement.parse(parser, context); fail("Expected IllegalStateException"); diff --git a/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 098825a90b1..9643f248d3e 100644 --- a/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -51,7 +51,7 @@ import java.util.concurrent.atomic.AtomicBoolean; public class QueryPhaseTests extends ESTestCase { private void countTestCase(Query query, IndexReader reader, boolean shouldCollect) throws Exception { - TestSearchContext context = new TestSearchContext(); + TestSearchContext context = new TestSearchContext(null); context.parsedQuery(new ParsedQuery(query)); context.setSize(0); @@ -120,7 +120,7 @@ public class QueryPhaseTests extends ESTestCase { } public void testPostFilterDisablesCountOptimization() throws Exception { - TestSearchContext context = new TestSearchContext(); + TestSearchContext context = new TestSearchContext(null); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); context.setSize(0); @@ -143,7 +143,7 @@ public class QueryPhaseTests extends ESTestCase { } public void testMinScoreDisablesCountOptimization() throws Exception { - TestSearchContext context = new TestSearchContext(); + TestSearchContext context = new TestSearchContext(null); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); context.setSize(0); diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java index e39fd81b28d..cbd7b5468b2 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SortParserTests.java @@ -36,7 +36,7 @@ public class SortParserTests extends ESSingleNodeTestCase { mapping.startObject().startObject("type").startObject("properties").startObject("location").field("type", "geo_point").endObject().endObject().endObject().endObject(); IndexService indexService = createIndex("testidx", Settings.settingsBuilder().build(), "type", mapping); TestSearchContext context = (TestSearchContext) createSearchContext(indexService); - context.setTypes("type"); + context.getQueryShardContext().setTypes("type"); XContentBuilder sortBuilder = jsonBuilder(); sortBuilder.startObject(); diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 1da604c4184..072c0db3e59 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -60,7 +60,7 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { Collections.singletonMap(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()), Collections.emptyMap()); parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::getQueryShardContext); + indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); } public void testDefaults() throws Exception { @@ -136,7 +136,7 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); indexService = createIndex("test_bwc", settings); parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::getQueryShardContext); + indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field") .field("type", "murmur3") @@ -152,7 +152,7 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); indexService = createIndex("test_bwc", settings); parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::getQueryShardContext); + indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field") .field("type", "murmur3") diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java index a44dddda3ed..d6b64df9e5d 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java @@ -66,7 +66,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase { Map metadataMappers = new HashMap<>(); IndicesModule indices = new IndicesModule(); indices.registerMetadataMapper(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); - mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), indices.getMapperRegistry(), indexService::getQueryShardContext); + mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), indices.getMapperRegistry(), indexService::newQueryShardContext); parser = mapperService.documentMapperParser(); } @@ -98,7 +98,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase { Collections.emptyMap(), Collections.singletonMap(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser())); parser = new DocumentMapperParser(indexService.getIndexSettings(), mapperService, - indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::getQueryShardContext); + indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping)); BytesReference source = XContentFactory.jsonBuilder() diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 814e0c0184f..303203c6c8a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.test; -import com.carrotsearch.hppc.ObjectObjectAssociativeContainer; import org.apache.lucene.search.Collector; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.Query; @@ -27,7 +26,6 @@ import org.apache.lucene.util.Counter; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cache.recycler.PageCacheRecycler; import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.analysis.AnalysisService; @@ -38,6 +36,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.script.ScriptService; @@ -47,7 +46,6 @@ import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseContext; -import org.elasticsearch.search.fetch.innerhits.InnerHitsContext; import org.elasticsearch.search.fetch.script.ScriptFieldsContext; import org.elasticsearch.search.fetch.source.FetchSourceContext; import org.elasticsearch.search.highlight.SearchContextHighlight; @@ -62,11 +60,9 @@ import org.elasticsearch.search.rescore.RescoreSearchContext; import org.elasticsearch.search.suggest.SuggestionSearchContext; import org.elasticsearch.threadpool.ThreadPool; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; public class TestSearchContext extends SearchContext { @@ -80,6 +76,7 @@ public class TestSearchContext extends SearchContext { final IndexShard indexShard; final Counter timeEstimateCounter = Counter.newCounter(); final QuerySearchResult queryResult = new QuerySearchResult(); + final QueryShardContext queryShardContext; ScriptService scriptService; ParsedQuery originalQuery; ParsedQuery postFilter; @@ -89,7 +86,6 @@ public class TestSearchContext extends SearchContext { ContextIndexSearcher searcher; int size; private int terminateAfter = DEFAULT_TERMINATE_AFTER; - private String[] types; private SearchContextAggregations aggregations; private final long originNanoTime = System.nanoTime(); @@ -105,9 +101,10 @@ public class TestSearchContext extends SearchContext { this.threadPool = threadPool; this.indexShard = indexService.getShardOrNull(0); this.scriptService = scriptService; + queryShardContext = indexService.newQueryShardContext(); } - public TestSearchContext() { + public TestSearchContext(QueryShardContext queryShardContext) { super(ParseFieldMatcher.STRICT); this.pageCacheRecycler = null; this.bigArrays = null; @@ -117,10 +114,7 @@ public class TestSearchContext extends SearchContext { this.fixedBitSetFilterCache = null; this.indexShard = null; scriptService = null; - } - - public void setTypes(String... types) { - this.types = types; + this.queryShardContext = queryShardContext; } @Override @@ -167,16 +161,6 @@ public class TestSearchContext extends SearchContext { return 1; } - @Override - public boolean hasTypes() { - return false; - } - - @Override - public String[] types() { - return new String[0]; - } - @Override public float queryBoost() { return 0; @@ -590,4 +574,9 @@ public class TestSearchContext extends SearchContext { @Override public Map, Collector> queryCollectors() {return queryCollectors;} + @Override + public QueryShardContext getQueryShardContext() { + return queryShardContext; + } + } From ef88943e6ecc9ff6e9351b3f46ee722d048645cc Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 8 Feb 2016 09:51:37 -0500 Subject: [PATCH 04/14] Cleanup JavaVersion This commit includes a few minor cleanups to o/e/b/JavaVersion.java: - Stronger argument checking in JavaVersion#parse - Use JDK 8 string joiner - Keep an immutable copy of the version sequence --- .../org/elasticsearch/bootstrap/JavaVersion.java | 16 +++++++--------- .../bootstrap/JavaVersionTests.java | 6 ++++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java b/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java index bf8e9961113..c37f37318fa 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java @@ -19,28 +19,26 @@ package org.elasticsearch.bootstrap; -import org.elasticsearch.common.Strings; - import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; class JavaVersion implements Comparable { private final List version; public List getVersion() { - return Collections.unmodifiableList(version); + return version; } private JavaVersion(List version) { - this.version = version; + this.version = Collections.unmodifiableList(version); } public static JavaVersion parse(String value) { - if (value == null) { - throw new NullPointerException("value"); - } - if ("".equals(value)) { + Objects.requireNonNull(value); + if (!isValid(value)) { throw new IllegalArgumentException("value"); } @@ -79,6 +77,6 @@ class JavaVersion implements Comparable { @Override public String toString() { - return Strings.collectionToDelimitedString(version, "."); + return version.stream().map(v -> Integer.toString(v)).collect(Collectors.joining(".")); } } diff --git a/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java b/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java index 21bfa05c1d4..d2ef349625e 100644 --- a/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java +++ b/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java @@ -36,8 +36,10 @@ public class JavaVersionTests extends ESTestCase { } public void testToString() { - JavaVersion javaVersion = JavaVersion.parse("1.7.0"); - assertThat("1.7.0", is(javaVersion.toString())); + JavaVersion javaVersion170 = JavaVersion.parse("1.7.0"); + assertThat(javaVersion170.toString(), is("1.7.0")); + JavaVersion javaVersion9 = JavaVersion.parse("9"); + assertThat(javaVersion9.toString(), is("9")); } public void testCompare() { From 12477f38b4890d86cb98bea30f70c0dc17d18393 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 27 Jan 2016 11:00:19 +0100 Subject: [PATCH 05/14] Fix serialization of `search_analyzer`. We currently have a bug that it will be omitted if the index analyzer is the default analyzer. --- .../index/mapper/FieldMapper.java | 23 ++++++++++----- .../index/mapper/core/StringFieldMapper.java | 12 ++------ .../string/SimpleStringMappingTests.java | 28 +++++++++++++++++++ 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index a9838503566..fd35398a9dc 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -421,8 +421,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable { builder.field("index_options", indexOptionToString(fieldType().indexOptions())); } - doXContentAnalyzers(builder, includeDefaults); - if (fieldType().similarity() != null) { builder.field("similarity", fieldType().similarity().name()); } else if (includeDefaults) { @@ -439,15 +437,26 @@ public abstract class FieldMapper extends Mapper implements Cloneable { } } - protected void doXContentAnalyzers(XContentBuilder builder, boolean includeDefaults) throws IOException { + protected final void doXContentAnalyzers(XContentBuilder builder, boolean includeDefaults) throws IOException { + if (fieldType.tokenized() == false) { + return; + } if (fieldType().indexAnalyzer() == null) { if (includeDefaults) { builder.field("analyzer", "default"); } - } else if (includeDefaults || fieldType().indexAnalyzer().name().startsWith("_") == false && fieldType().indexAnalyzer().name().equals("default") == false) { - builder.field("analyzer", fieldType().indexAnalyzer().name()); - if (fieldType().searchAnalyzer().name().equals(fieldType().indexAnalyzer().name()) == false) { - builder.field("search_analyzer", fieldType().searchAnalyzer().name()); + } else { + boolean hasDefaultIndexAnalyzer = fieldType().indexAnalyzer().name().equals("default"); + boolean hasDifferentSearchAnalyzer = fieldType().searchAnalyzer().name().equals(fieldType().indexAnalyzer().name()) == false; + boolean hasDifferentSearchQuoteAnalyzer = fieldType().searchAnalyzer().name().equals(fieldType().searchQuoteAnalyzer().name()) == false; + if (includeDefaults || hasDefaultIndexAnalyzer == false || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) { + builder.field("analyzer", fieldType().indexAnalyzer().name()); + if (hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) { + builder.field("search_analyzer", fieldType().searchAnalyzer().name()); + if (hasDifferentSearchQuoteAnalyzer) { + builder.field("search_quote_analyzer", fieldType().searchQuoteAnalyzer().name()); + } + } } } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java index 918731d0244..05ef8c88e0c 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java @@ -404,6 +404,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); + doXContentAnalyzers(builder, includeDefaults); if (includeDefaults || fieldType().nullValue() != null) { builder.field("null_value", fieldType().nullValue()); @@ -417,16 +418,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc if (includeDefaults || positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) { builder.field("position_increment_gap", positionIncrementGap); } - NamedAnalyzer searchQuoteAnalyzer = fieldType().searchQuoteAnalyzer(); - if (searchQuoteAnalyzer != null && !searchQuoteAnalyzer.name().equals(fieldType().searchAnalyzer().name())) { - builder.field("search_quote_analyzer", searchQuoteAnalyzer.name()); - } else if (includeDefaults) { - if (searchQuoteAnalyzer == null) { - builder.field("search_quote_analyzer", "default"); - } else { - builder.field("search_quote_analyzer", searchQuoteAnalyzer.name()); - } - } + if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) { builder.field("ignore_above", ignoreAbove); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java index 6114185ccf6..f100182ebf5 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/string/SimpleStringMappingTests.java @@ -50,6 +50,7 @@ import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import org.junit.Before; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Map; @@ -280,6 +281,33 @@ public class SimpleStringMappingTests extends ESSingleNodeTestCase { } } + public void testSearchAnalyzerSerialization() throws IOException { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "string") + .field("analyzer", "standard") + .field("search_analyzer", "keyword") + .endObject() + .endObject().endObject().endObject().string(); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + // special case: default index analyzer + mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "string") + .field("analyzer", "default") + .field("search_analyzer", "keyword") + .endObject() + .endObject().endObject().endObject().string(); + + mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + } + private Map getSerializedMap(String fieldName, DocumentMapper mapper) throws Exception { FieldMapper fieldMapper = mapper.mappers().smartNameFieldMapper(fieldName); XContentBuilder builder = JsonXContent.contentBuilder().startObject(); From cbeae225b949f38b13523a30fc00a42962f3168d Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 8 Feb 2016 10:54:50 -0500 Subject: [PATCH 06/14] Speedup MessageDigestTests#testToHexString The purpose of this commit is to speed up the runtime of MessageDigestTests#testToHexString. As written, the test contains a loop that creates 1024 test cases leading to a test runtime on the order of a few seconds. Given build infrastructure, a single test case should suffice. Therefore, this commit removes this loop so that the test can execute on the order of a couple hundred milliseconds. --- .../common/hash/MessageDigestsTests.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java b/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java index 4063c1b7be4..e3c085f0328 100644 --- a/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java +++ b/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java @@ -59,16 +59,14 @@ public class MessageDigestsTests extends ESTestCase { } public void testToHexString() throws Exception { - for (int i = 0; i < 1024; i++) { - BigInteger expected = BigInteger.probablePrime(256, random()); - byte[] bytes = expected.toByteArray(); - String hex = MessageDigests.toHexString(bytes); - String zeros = new String(new char[2 * bytes.length]).replace("\0", "0"); - String expectedAsString = expected.toString(16); - String expectedHex = zeros.substring(expectedAsString.length()) + expectedAsString; - assertEquals(expectedHex, hex); - BigInteger actual = new BigInteger(hex, 16); - assertEquals(expected, actual); - } + BigInteger expected = BigInteger.probablePrime(256, random()); + byte[] bytes = expected.toByteArray(); + String hex = MessageDigests.toHexString(bytes); + String zeros = new String(new char[2 * bytes.length]).replace("\0", "0"); + String expectedAsString = expected.toString(16); + String expectedHex = zeros.substring(expectedAsString.length()) + expectedAsString; + assertEquals(expectedHex, hex); + BigInteger actual = new BigInteger(hex, 16); + assertEquals(expected, actual); } } From 2412dbe599af6ce095527334eea1690b1d2be6a7 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 8 Feb 2016 17:16:12 +0100 Subject: [PATCH 07/14] apply feedback from @colings86 --- .../org/elasticsearch/percolator/PercolateContext.java | 4 ++-- .../percolator/PercolateDocumentParserTests.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java b/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java index 4c7f22e3922..7572ebcba87 100644 --- a/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java +++ b/core/src/main/java/org/elasticsearch/percolator/PercolateContext.java @@ -139,7 +139,7 @@ public class PercolateContext extends SearchContext { } // for testing: - PercolateContext(PercolateShardRequest request, SearchShardTarget searchShardTarget, MapperService mapperService) { + PercolateContext(PercolateShardRequest request, SearchShardTarget searchShardTarget, MapperService mapperService, QueryShardContext queryShardContext) { super(null); this.searchShardTarget = searchShardTarget; this.mapperService = mapperService; @@ -153,7 +153,7 @@ public class PercolateContext extends SearchContext { this.startTime = 0; this.numberOfShards = 0; this.onlyCount = true; - queryShardContext = new QueryShardContext(mapperService.getIndexSettings(), null, null, null, mapperService, null, null, null); + this.queryShardContext = queryShardContext; } public IndexSearcher docSearcher() { diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java b/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java index ab0e86fd2af..17426f111b3 100644 --- a/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java +++ b/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java @@ -104,7 +104,7 @@ public class PercolateDocumentParserTests extends ESTestCase { .endObject(); Mockito.when(request.source()).thenReturn(source.bytes()); - PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService); + PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext); ParsedDocument parsedDocument = parser.parse(request, context, mapperService, queryShardContext); assertThat(parsedDocument.rootDoc().get("field1"), equalTo("value1")); } @@ -123,7 +123,7 @@ public class PercolateDocumentParserTests extends ESTestCase { .endObject(); Mockito.when(request.source()).thenReturn(source.bytes()); - PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService); + PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext); ParsedDocument parsedDocument = parser.parse(request, context, mapperService, queryShardContext); assertThat(parsedDocument.rootDoc().get("field1"), equalTo("value1")); assertThat(context.percolateQuery(), equalTo(new TermQuery(new Term("field1", "value1")))); @@ -147,7 +147,7 @@ public class PercolateDocumentParserTests extends ESTestCase { Mockito.when(request.source()).thenReturn(source.bytes()); Mockito.when(request.docSource()).thenReturn(docSource.bytes()); - PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService); + PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext); ParsedDocument parsedDocument = parser.parse(request, context, mapperService, queryShardContext); assertThat(parsedDocument.rootDoc().get("field1"), equalTo("value1")); assertThat(context.percolateQuery(), equalTo(new TermQuery(new Term("field1", "value1")))); @@ -174,7 +174,7 @@ public class PercolateDocumentParserTests extends ESTestCase { Mockito.when(request.source()).thenReturn(source.bytes()); Mockito.when(request.docSource()).thenReturn(docSource.bytes()); - PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService); + PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext); try { parser.parse(request, context, mapperService, queryShardContext); } catch (IllegalArgumentException e) { From 7ea4f7400baaa638a78cf24baed09d4283f1e4ac Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 8 Feb 2016 13:14:14 +0100 Subject: [PATCH 08/14] Require that relocation source is marked as relocating before starting recovery to relocation target --- .../elasticsearch/indices/recovery/RecoverySource.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java index f24452bf007..105dd3c445f 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java @@ -93,6 +93,14 @@ public class RecoverySource extends AbstractComponent implements IndexEventListe logger.debug("delaying recovery of {} as source node {} is unknown", request.shardId(), request.targetNode()); throw new DelayRecoveryException("source node does not have the node [" + request.targetNode() + "] in its state yet.."); } + + ShardRouting routingEntry = shard.routingEntry(); + if (request.recoveryType() == RecoveryState.Type.PRIMARY_RELOCATION && + (routingEntry.relocating() == false || routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) { + logger.debug("delaying recovery of {} as source shard is not marked yet as relocating to {}", request.shardId(), request.targetNode()); + throw new DelayRecoveryException("source shard is not marked yet as relocating to [" + request.targetNode() + "]"); + } + ShardRouting targetShardRouting = null; for (ShardRouting shardRouting : node) { if (shardRouting.shardId().equals(request.shardId())) { From 46ee56869535c38286d334f728bc0a189dd96b64 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 8 Feb 2016 11:21:15 -0500 Subject: [PATCH 09/14] Log warning if max file descriptors too low This commit adds logging for a warning message if the max file descriptor count is too low. Closes #16506 --- .../org/elasticsearch/env/NodeEnvironment.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 80c427c9cdc..5eecafa252f 100644 --- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.store.FsDirectoryService; import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.monitor.fs.FsProbe; import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.monitor.process.ProcessProbe; import java.io.Closeable; import java.io.IOException; @@ -221,6 +222,7 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { maybeLogPathDetails(); maybeLogHeapDetails(); + maybeWarnFileDescriptors(); applySegmentInfosTrace(settings); } @@ -313,6 +315,20 @@ public class NodeEnvironment extends AbstractComponent implements Closeable { logger.info("heap size [{}], compressed ordinary object pointers [{}]", maxHeapSize, useCompressedOops); } + private void maybeWarnFileDescriptors() { + long maxFileDescriptorCount = ProcessProbe.getInstance().getMaxFileDescriptorCount(); + if (maxFileDescriptorCount == -1) { + return; + } + int fileDescriptorCountThreshold = (1 << 16); + if (maxFileDescriptorCount < fileDescriptorCountThreshold) { + logger.warn( + "max file descriptors [{}] for elasticsearch process likely too low, consider increasing to at least [{}]", + maxFileDescriptorCount, + fileDescriptorCountThreshold); + } + } + @SuppressForbidden(reason = "System.out.*") static void applySegmentInfosTrace(Settings settings) { if (ENABLE_LUCENE_SEGMENT_INFOS_TRACE_SETTING.get(settings)) { From cc0d0525fcff875c0fcee45d435c281dee5c2510 Mon Sep 17 00:00:00 2001 From: Spencer Date: Mon, 8 Feb 2016 10:49:27 -0700 Subject: [PATCH 10/14] Fix asciidoc typo --- docs/reference/migration/migrate_3_0.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/migration/migrate_3_0.asciidoc b/docs/reference/migration/migrate_3_0.asciidoc index bae484f4a13..9e95f9dd865 100644 --- a/docs/reference/migration/migrate_3_0.asciidoc +++ b/docs/reference/migration/migrate_3_0.asciidoc @@ -112,6 +112,7 @@ The `GET` HTTP verb for `/_forcemerge` is no longer supported, please use the ==== Deprecated queries removed The following deprecated queries have been removed: + * `filtered`: use `bool` query instead, which supports `filter` clauses too * `and`: use `must` clauses in a `bool` query instead * `or`: use should clauses in a `bool` query instead From 06e526ba5805e5ea34dc8ca7db8414fda3c0b838 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 8 Feb 2016 13:13:32 -0500 Subject: [PATCH 11/14] Register bootstrap settings This commit registers bootstrap settings used on startup. Without registration, setting any of these settings causes node startup to fail. By registering these settings (rather than clearing) after use, we enable them to be visible in any APIs that show all settings. Closes #16513 --- .../elasticsearch/bootstrap/Bootstrap.java | 27 ++++-------- .../bootstrap/BootstrapSettings.java | 43 +++++++++++++++++++ .../common/settings/ClusterSettings.java | 8 +++- .../bootstrap/BootstrapSettingsTests.java | 35 +++++++++++++++ 4 files changed, 94 insertions(+), 19 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java create mode 100644 core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index ed99f9b3894..a1468f47fc8 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.logging.log4j.LogConfigurator; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.monitor.jvm.JvmInfo; @@ -148,10 +149,11 @@ final class Bootstrap { } private void setup(boolean addShutdownHook, Settings settings, Environment environment) throws Exception { - initializeNatives(environment.tmpFile(), - settings.getAsBoolean("bootstrap.mlockall", false), - settings.getAsBoolean("bootstrap.seccomp", true), - settings.getAsBoolean("bootstrap.ctrlhandler", true)); + initializeNatives( + environment.tmpFile(), + BootstrapSettings.MLOCKALL_SETTING.get(settings), + BootstrapSettings.SECCOMP_SETTING.get(settings), + BootstrapSettings.CTRLHANDLER_SETTING.get(settings)); // initialize probes before the security manager is installed initializeProbes(); @@ -186,22 +188,11 @@ final class Bootstrap { node = new Node(nodeSettings); } - /** - * option for elasticsearch.yml etc to turn off our security manager completely, - * for example if you want to have your own configuration or just disable. - */ - // TODO: remove this: http://www.openbsd.org/papers/hackfest2015-pledge/mgp00005.jpg - static final String SECURITY_SETTING = "security.manager.enabled"; - /** - * option for elasticsearch.yml to fully respect the system policy, including bad defaults - * from java. - */ - // TODO: remove this hack when insecure defaults are removed from java - static final String SECURITY_FILTER_BAD_DEFAULTS_SETTING = "security.manager.filter_bad_defaults"; + private void setupSecurity(Settings settings, Environment environment) throws Exception { - if (settings.getAsBoolean(SECURITY_SETTING, true)) { - Security.configure(environment, settings.getAsBoolean(SECURITY_FILTER_BAD_DEFAULTS_SETTING, true)); + if (BootstrapSettings.SECURITY_MANAGER_ENABLED_SETTING.get(settings)) { + Security.configure(environment, BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(settings)); } } diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java new file mode 100644 index 00000000000..adf969875ba --- /dev/null +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Scope; + +public class BootstrapSettings { + + // TODO: remove this: http://www.openbsd.org/papers/hackfest2015-pledge/mgp00005.jpg + /** + * option to turn off our security manager completely, for example + * if you want to have your own configuration or just disable + */ + public static final Setting SECURITY_MANAGER_ENABLED_SETTING = + Setting.boolSetting("security.manager.enabled", true, false, Scope.CLUSTER); + + // TODO: remove this hack when insecure defaults are removed from java + public static final Setting SECURITY_FILTER_BAD_DEFAULTS_SETTING = + Setting.boolSetting("security.manager.filter_bad_defaults", true, false, Scope.CLUSTER); + + public static final Setting MLOCKALL_SETTING = Setting.boolSetting("bootstrap.mlockall", false, false, Scope.CLUSTER); + public static final Setting SECCOMP_SETTING = Setting.boolSetting("bootstrap.seccomp", true, false, Scope.CLUSTER); + public static final Setting CTRLHANDLER_SETTING = Setting.boolSetting("bootstrap.ctrlhandler", true, false, Scope.CLUSTER); + +} diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index e4b92d82d20..033423b08ef 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -88,6 +88,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportSettings; import org.elasticsearch.transport.netty.NettyTransport; import org.elasticsearch.tribe.TribeService; +import org.elasticsearch.bootstrap.BootstrapSettings; import java.util.Arrays; import java.util.Collections; @@ -378,6 +379,11 @@ public final class ClusterSettings extends AbstractScopedSettings { PageCacheRecycler.WEIGHT_LONG_SETTING, PageCacheRecycler.WEIGHT_OBJECTS_SETTING, PageCacheRecycler.TYPE_SETTING, - PluginsService.MANDATORY_SETTING + PluginsService.MANDATORY_SETTING, + BootstrapSettings.SECURITY_MANAGER_ENABLED_SETTING, + BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING, + BootstrapSettings.MLOCKALL_SETTING, + BootstrapSettings.SECCOMP_SETTING, + BootstrapSettings.CTRLHANDLER_SETTING ))); } diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java new file mode 100644 index 00000000000..0570a69f0b9 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.bootstrap; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +public class BootstrapSettingsTests extends ESTestCase { + + public void testDefaultSettings() { + assertTrue(BootstrapSettings.SECURITY_MANAGER_ENABLED_SETTING.get(Settings.EMPTY)); + assertTrue(BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(Settings.EMPTY)); + assertFalse(BootstrapSettings.MLOCKALL_SETTING.get(Settings.EMPTY)); + assertTrue(BootstrapSettings.SECCOMP_SETTING.get(Settings.EMPTY)); + assertTrue(BootstrapSettings.CTRLHANDLER_SETTING.get(Settings.EMPTY)); + } + +} From 10f55a2df114727797ad249cb3a3abec374b8f90 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 8 Feb 2016 20:27:22 +0100 Subject: [PATCH 12/14] simplify method signature --- .../percolator/PercolateDocumentParser.java | 10 +++++----- .../elasticsearch/percolator/PercolatorService.java | 2 +- .../percolator/PercolateDocumentParserTests.java | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java b/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java index 2192876266a..50db3cecaa6 100644 --- a/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java +++ b/core/src/main/java/org/elasticsearch/percolator/PercolateDocumentParser.java @@ -57,7 +57,7 @@ public class PercolateDocumentParser { this.aggregationPhase = aggregationPhase; } - public ParsedDocument parse(PercolateShardRequest request, PercolateContext context, MapperService mapperService, QueryShardContext queryShardContext) { + public ParsedDocument parse(final PercolateShardRequest request, final PercolateContext context, final MapperService mapperService) { BytesReference source = request.source(); if (source == null || source.length() == 0) { if (request.docSource() != null && request.docSource().length() != 0) { @@ -70,13 +70,13 @@ public class PercolateDocumentParser { // TODO: combine all feature parse elements into one map Map hlElements = highlightPhase.parseElements(); Map aggregationElements = aggregationPhase.parseElements(); - + final QueryShardContext queryShardContext = context.getQueryShardContext(); ParsedDocument doc = null; // Some queries (function_score query when for decay functions) rely on a SearchContext being set: // We switch types because this context needs to be in the context of the percolate queries in the shard and // not the in memory percolate doc - String[] previousTypes = context.getQueryShardContext().getTypes(); - context.getQueryShardContext().setTypes(PercolatorService.TYPE_NAME); + final String[] previousTypes = queryShardContext.getTypes(); + queryShardContext.setTypes(PercolatorService.TYPE_NAME); try (XContentParser parser = XContentFactory.xContent(source).createParser(source);) { String currentFieldName = null; XContentParser.Token token; @@ -173,7 +173,7 @@ public class PercolateDocumentParser { } catch (Throwable e) { throw new ElasticsearchParseException("failed to parse request", e); } finally { - context.getQueryShardContext().setTypes(previousTypes); + queryShardContext.setTypes(previousTypes); } if (request.docSource() != null && request.docSource().length() != 0) { diff --git a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java index c36de9c6e4c..3fb2b348487 100644 --- a/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java +++ b/core/src/main/java/org/elasticsearch/percolator/PercolatorService.java @@ -191,7 +191,7 @@ public class PercolatorService extends AbstractComponent implements Releasable { ); SearchContext.setCurrent(context); try { - ParsedDocument parsedDocument = percolateDocumentParser.parse(request, context, percolateIndexService.mapperService(), context.getQueryShardContext()); + ParsedDocument parsedDocument = percolateDocumentParser.parse(request, context, percolateIndexService.mapperService()); if (context.searcher().getIndexReader().maxDoc() == 0) { return new PercolateShardResponse(Lucene.EMPTY_TOP_DOCS, Collections.emptyMap(), Collections.emptyMap(), context); } diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java b/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java index 17426f111b3..91fb8b5cd8f 100644 --- a/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java +++ b/core/src/test/java/org/elasticsearch/percolator/PercolateDocumentParserTests.java @@ -105,7 +105,7 @@ public class PercolateDocumentParserTests extends ESTestCase { Mockito.when(request.source()).thenReturn(source.bytes()); PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext); - ParsedDocument parsedDocument = parser.parse(request, context, mapperService, queryShardContext); + ParsedDocument parsedDocument = parser.parse(request, context, mapperService); assertThat(parsedDocument.rootDoc().get("field1"), equalTo("value1")); } @@ -124,7 +124,7 @@ public class PercolateDocumentParserTests extends ESTestCase { Mockito.when(request.source()).thenReturn(source.bytes()); PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext); - ParsedDocument parsedDocument = parser.parse(request, context, mapperService, queryShardContext); + ParsedDocument parsedDocument = parser.parse(request, context, mapperService); assertThat(parsedDocument.rootDoc().get("field1"), equalTo("value1")); assertThat(context.percolateQuery(), equalTo(new TermQuery(new Term("field1", "value1")))); assertThat(context.trackScores(), is(true)); @@ -148,7 +148,7 @@ public class PercolateDocumentParserTests extends ESTestCase { Mockito.when(request.docSource()).thenReturn(docSource.bytes()); PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext); - ParsedDocument parsedDocument = parser.parse(request, context, mapperService, queryShardContext); + ParsedDocument parsedDocument = parser.parse(request, context, mapperService); assertThat(parsedDocument.rootDoc().get("field1"), equalTo("value1")); assertThat(context.percolateQuery(), equalTo(new TermQuery(new Term("field1", "value1")))); assertThat(context.trackScores(), is(true)); @@ -176,7 +176,7 @@ public class PercolateDocumentParserTests extends ESTestCase { PercolateContext context = new PercolateContext(request, new SearchShardTarget("_node", new Index("_index", "_na_"), 0), mapperService, queryShardContext); try { - parser.parse(request, context, mapperService, queryShardContext); + parser.parse(request, context, mapperService); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), equalTo("Can't specify the document to percolate in the source of the request and as document id")); } From c121af63b45a61daee159c220929626bb5dccef8 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 8 Feb 2016 17:24:09 -0500 Subject: [PATCH 13/14] BootstrapSettings final with private constructor --- .../java/org/elasticsearch/bootstrap/BootstrapSettings.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java index adf969875ba..9122504f0e8 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java @@ -22,7 +22,10 @@ package org.elasticsearch.bootstrap; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Scope; -public class BootstrapSettings { +public final class BootstrapSettings { + + private BootstrapSettings() { + } // TODO: remove this: http://www.openbsd.org/papers/hackfest2015-pledge/mgp00005.jpg /** From 3bad16c1caef3ae79793181ebc0057a689544332 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 8 Feb 2016 17:46:41 -0800 Subject: [PATCH 14/14] Rename variables. This PR renames the following three variables to fix a typo `settting` into `setting`. * Rename a static class member: INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING -> INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING * Rename a parameter: aSettting --> aSetting * Rename a local variable: indexSetttings -> indexSettings --- .../common/settings/IndexScopedSettings.java | 2 +- .../java/org/elasticsearch/common/settings/Setting.java | 4 ++-- .../main/java/org/elasticsearch/index/IndexSettings.java | 6 +++--- .../java/org/elasticsearch/index/IndexSettingsTests.java | 8 ++++---- .../elasticsearch/index/IndexWithShadowReplicasIT.java | 2 +- .../org/elasticsearch/index/shard/IndexShardTests.java | 6 +++--- .../org/elasticsearch/index/shard/ShardPathTests.java | 8 ++++---- .../org/elasticsearch/index/store/CorruptedFileIT.java | 8 ++++---- .../elasticsearch/index/store/CorruptedTranslogIT.java | 4 ++-- .../java/org/elasticsearch/indices/flush/FlushIT.java | 2 +- .../main/java/org/elasticsearch/test/ESIntegTestCase.java | 4 ++-- 11 files changed, 27 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 4197da980bc..157bbfbd5b9 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -117,7 +117,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING, EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING, - IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING, + IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, IndexFieldDataService.INDEX_FIELDDATA_CACHE_KEY, FieldMapper.IGNORE_MALFORMED_SETTING, FieldMapper.COERCE_SETTING, diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java index 17d670b254a..22485ecc3f9 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -256,8 +256,8 @@ public class Setting extends ToXContentToBytes { * this is used for settings that depend on each other... see {@link org.elasticsearch.common.settings.AbstractScopedSettings#addSettingsUpdateConsumer(Setting, Setting, BiConsumer)} and it's * usage for details. */ - static AbstractScopedSettings.SettingUpdater> compoundUpdater(final BiConsumer consumer, final Setting aSettting, final Setting bSetting, ESLogger logger) { - final AbstractScopedSettings.SettingUpdater aSettingUpdater = aSettting.newUpdater(null, logger); + static AbstractScopedSettings.SettingUpdater> compoundUpdater(final BiConsumer consumer, final Setting aSetting, final Setting bSetting, ESLogger logger) { + final AbstractScopedSettings.SettingUpdater aSettingUpdater = aSetting.newUpdater(null, logger); final AbstractScopedSettings.SettingUpdater bSettingUpdater = bSetting.newUpdater(null, logger); return new AbstractScopedSettings.SettingUpdater>() { @Override diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java index d6a4e901e55..da15d29d191 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -82,7 +82,7 @@ public final class IndexSettings { public static final Setting MAX_RESULT_WINDOW_SETTING = Setting.intSetting("index.max_result_window", 10000, 1, true, Setting.Scope.INDEX); public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS); public static final Setting INDEX_REFRESH_INTERVAL_SETTING = Setting.timeSetting("index.refresh_interval", DEFAULT_REFRESH_INTERVAL, new TimeValue(-1, TimeUnit.MILLISECONDS), true, Setting.Scope.INDEX); - public static final Setting INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING = Setting.byteSizeSetting("index.translog.flush_threshold_size", new ByteSizeValue(512, ByteSizeUnit.MB), true, Setting.Scope.INDEX); + public static final Setting INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING = Setting.byteSizeSetting("index.translog.flush_threshold_size", new ByteSizeValue(512, ByteSizeUnit.MB), true, Setting.Scope.INDEX); /** @@ -197,7 +197,7 @@ public final class IndexSettings { this.durability = scopedSettings.get(INDEX_TRANSLOG_DURABILITY_SETTING); syncInterval = INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.get(settings); refreshInterval = scopedSettings.get(INDEX_REFRESH_INTERVAL_SETTING); - flushThresholdSize = scopedSettings.get(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING); + flushThresholdSize = scopedSettings.get(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING); mergeSchedulerConfig = new MergeSchedulerConfig(this); gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis(); warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING); @@ -222,7 +222,7 @@ public final class IndexSettings { scopedSettings.addSettingsUpdateConsumer(MAX_RESULT_WINDOW_SETTING, this::setMaxResultWindow); scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer); scopedSettings.addSettingsUpdateConsumer(INDEX_GC_DELETES_SETTING, this::setGCDeletes); - scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING, this::setTranslogFlushThresholdSize); + scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, this::setTranslogFlushThresholdSize); scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval); } diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 09fec10a621..677c8358fb0 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -272,16 +272,16 @@ public class IndexSettingsTests extends ESTestCase { public void testTranslogFlushSizeThreshold() { ByteSizeValue translogFlushThresholdSize = new ByteSizeValue(Math.abs(randomInt())); - ByteSizeValue actualValue = ByteSizeValue.parseBytesSizeValue(translogFlushThresholdSize.toString(), IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey()); + ByteSizeValue actualValue = ByteSizeValue.parseBytesSizeValue(translogFlushThresholdSize.toString(), IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey()); IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), translogFlushThresholdSize.toString()) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), translogFlushThresholdSize.toString()) .build()); IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY); assertEquals(actualValue, settings.getFlushThresholdSize()); ByteSizeValue newTranslogFlushThresholdSize = new ByteSizeValue(Math.abs(randomInt())); - ByteSizeValue actualNewTranslogFlushThresholdSize = ByteSizeValue.parseBytesSizeValue(newTranslogFlushThresholdSize.toString(), IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey()); - settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), newTranslogFlushThresholdSize.toString()).build())); + ByteSizeValue actualNewTranslogFlushThresholdSize = ByteSizeValue.parseBytesSizeValue(newTranslogFlushThresholdSize.toString(), IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey()); + settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), newTranslogFlushThresholdSize.toString()).build())); assertEquals(actualNewTranslogFlushThresholdSize, settings.getFlushThresholdSize()); } } diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java index 5d54f7731c2..4ef513a9be3 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java @@ -182,7 +182,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase { Settings idxSettings = Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2) - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) .put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString()) .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true) .put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true) diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 778831245d1..e5f0ade2d50 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -704,7 +704,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndexService test = indicesService.indexService("test"); IndexShard shard = test.getShardOrNull(0); assertFalse(shard.shouldFlush()); - client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(133 /* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get(); + client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(133 /* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get(); client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get(); assertFalse(shard.shouldFlush()); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, new ParseContext.Document(), new BytesArray(new byte[]{1}), null); @@ -720,7 +720,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { shard.getEngine().getTranslog().sync(); long size = shard.getEngine().getTranslog().sizeInBytes(); logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration()); - client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(size, ByteSizeUnit.BYTES)) + client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(size, ByteSizeUnit.BYTES)) .build()).get(); client().prepareDelete("test", "test", "2").get(); logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", shard.getEngine().getTranslog().sizeInBytes(), shard.getEngine().getTranslog().totalOperations(), shard.getEngine().getTranslog().getGeneration()); @@ -738,7 +738,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { IndexService test = indicesService.indexService("test"); final IndexShard shard = test.getShardOrNull(0); assertFalse(shard.shouldFlush()); - client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(133/* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get(); + client().admin().indices().prepareUpdateSettings("test").setSettings(settingsBuilder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(133/* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get(); client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get(); assertFalse(shard.shouldFlush()); final AtomicBoolean running = new AtomicBoolean(true); diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java index 640ae9e606c..2a52e8c557c 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java @@ -109,7 +109,7 @@ public class ShardPathTests extends ESTestCase { public void testGetRootPaths() throws IOException { boolean useCustomDataPath = randomBoolean(); - final Settings indexSetttings; + final Settings indexSettings; final Settings nodeSettings; Settings.Builder indexSettingsBuilder = settingsBuilder() .put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF") @@ -118,7 +118,7 @@ public class ShardPathTests extends ESTestCase { if (useCustomDataPath) { final Path path = createTempDir(); final boolean includeNodeId = randomBoolean(); - indexSetttings = indexSettingsBuilder.put(IndexMetaData.SETTING_DATA_PATH, "custom").build(); + indexSettings = indexSettingsBuilder.put(IndexMetaData.SETTING_DATA_PATH, "custom").build(); nodeSettings = settingsBuilder().put(Environment.PATH_SHARED_DATA_SETTING.getKey(), path.toAbsolutePath().toAbsolutePath()) .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), includeNodeId).build(); if (includeNodeId) { @@ -128,7 +128,7 @@ public class ShardPathTests extends ESTestCase { } } else { customPath = null; - indexSetttings = indexSettingsBuilder.build(); + indexSettings = indexSettingsBuilder.build(); nodeSettings = Settings.EMPTY; } try (final NodeEnvironment env = newNodeEnvironment(nodeSettings)) { @@ -136,7 +136,7 @@ public class ShardPathTests extends ESTestCase { Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path); - ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), indexSetttings)); + ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), indexSettings)); boolean found = false; for (Path p : env.nodeDataPaths()) { if (p.equals(shardPath.getRootStatePath())) { diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java index 0f680307455..79f8a0cc666 100644 --- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -146,7 +146,7 @@ public class CorruptedFileIT extends ESIntegTestCase { .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1") .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files )); ensureGreen(); disableAllocation("test"); @@ -250,7 +250,7 @@ public class CorruptedFileIT extends ESIntegTestCase { .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files )); ensureGreen(); IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; @@ -475,7 +475,7 @@ public class CorruptedFileIT extends ESIntegTestCase { .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") // no replicas for this test .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files )); ensureGreen(); IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; @@ -529,7 +529,7 @@ public class CorruptedFileIT extends ESIntegTestCase { .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, cluster().numDataNodes() - 1) .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose - .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files + .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files )); ensureGreen(); IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java index 500cf07692e..ae158b87c59 100644 --- a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java +++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java @@ -167,13 +167,13 @@ public class CorruptedTranslogIT extends ESIntegTestCase { /** Disables translog flushing for the specified index */ private static void disableTranslogFlush(String index) { - Settings settings = Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)).build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)).build(); client().admin().indices().prepareUpdateSettings(index).setSettings(settings).get(); } /** Enables translog flushing for the specified index */ private static void enableTranslogFlush(String index) { - Settings settings = Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)).build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB)).build(); client().admin().indices().prepareUpdateSettings(index).setSettings(settings).get(); } } diff --git a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java index d74e490fdbc..60a56c11251 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java @@ -154,7 +154,7 @@ public class FlushIT extends ESIntegTestCase { createIndex("test"); client().admin().indices().prepareUpdateSettings("test").setSettings( - Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)).put("index.refresh_interval", -1).put("index.number_of_replicas", internalCluster().numDataNodes() - 1)) + Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)).put("index.refresh_interval", -1).put("index.number_of_replicas", internalCluster().numDataNodes() - 1)) .get(); ensureGreen(); final AtomicBoolean stop = new AtomicBoolean(false); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index d24baeb6c10..bb03cf8ba47 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -515,10 +515,10 @@ public abstract class ESIntegTestCase extends ESTestCase { private static Settings.Builder setRandomIndexTranslogSettings(Random random, Settings.Builder builder) { if (random.nextBoolean()) { - builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 300), ByteSizeUnit.MB)); + builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(RandomInts.randomIntBetween(random, 1, 300), ByteSizeUnit.MB)); } if (random.nextBoolean()) { - builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)); // just don't flush + builder.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)); // just don't flush } if (random.nextBoolean()) { builder.put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), RandomPicks.randomFrom(random, Translog.Durability.values()));