diff --git a/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java b/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java index c8a75f447b7..665d17a2f86 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java +++ b/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrapper.java @@ -36,10 +36,12 @@ public interface IndexSearcherWrapper { DirectoryReader wrap(DirectoryReader reader); /** - * @param searcher The provided index searcher to be wrapped to add custom functionality + * @param engineConfig The engine config which can be used to get the query cache and query cache policy from + * when creating a new index searcher + * @param searcher The provided index searcher to be wrapped to add custom functionality * @return a new index searcher wrapping the provided index searcher or if no wrapping was performed * the provided index searcher */ - IndexSearcher wrap(IndexSearcher searcher) throws EngineException; + IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException; } diff --git a/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrappingService.java b/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrappingService.java index a0ea90e024e..23d05f01dc7 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrappingService.java +++ b/core/src/main/java/org/elasticsearch/index/engine/IndexSearcherWrappingService.java @@ -77,7 +77,7 @@ public final class IndexSearcherWrappingService { // TODO: Right now IndexSearcher isn't wrapper friendly, when it becomes wrapper friendly we should revise this extension point // For example if IndexSearcher#rewrite() is overwritten than also IndexSearcher#createNormalizedWeight needs to be overwritten // This needs to be fixed before we can allow the IndexSearcher from Engine to be wrapped multiple times - IndexSearcher indexSearcher = wrapper.wrap(innerIndexSearcher); + IndexSearcher indexSearcher = wrapper.wrap(engineConfig, innerIndexSearcher); if (reader == engineSearcher.reader() && indexSearcher == innerIndexSearcher) { return engineSearcher; } else { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 5c6a10635ab..33281dc86f4 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -101,8 +101,7 @@ public class DocumentMapperParser { .put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser()) .put(TypeParsers.MULTI_FIELD_CONTENT_TYPE, TypeParsers.multiFieldConverterTypeParser) .put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()) - .put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()) - .put(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + .put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); if (ShapesAvailability.JTS_AVAILABLE) { typeParsersBuilder.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java index 73c92ded424..41b657a73ca 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java @@ -84,10 +84,6 @@ public final class MapperBuilders { return new LongFieldMapper.Builder(name); } - public static Murmur3FieldMapper.Builder murmur3Field(String name) { - return new Murmur3FieldMapper.Builder(name); - } - public static FloatFieldMapper.Builder floatField(String name) { return new FloatFieldMapper.Builder(name); } diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java index ba2ef65309b..3ae7b89a12e 100644 --- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryParser.java @@ -255,6 +255,7 @@ public class HasChildQueryParser extends BaseQueryParserTemp { String joinField = ParentFieldMapper.joinField(parentType); IndexReader indexReader = searchContext.searcher().getIndexReader(); IndexSearcher indexSearcher = new IndexSearcher(indexReader); + indexSearcher.setQueryCache(null); IndexParentChildFieldData indexParentChildFieldData = parentChildIndexFieldData.loadGlobal(indexReader); MultiDocValues.OrdinalMap ordinalMap = ParentChildIndexFieldData.getOrdinalMap(indexParentChildFieldData, parentType); return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode, ordinalMap, minChildren, maxChildren); diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java index 132f3994d17..759c9e5e150 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -37,6 +37,7 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCacheListener; import org.elasticsearch.indices.flush.SyncedFlushService; import org.elasticsearch.indices.memory.IndexingMemoryController; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.indices.recovery.RecoverySource; import org.elasticsearch.indices.recovery.RecoveryTarget; @@ -154,6 +155,7 @@ public class IndicesModule extends AbstractModule { protected void bindQueryParsersExtension() { queryParsers.bind(binder()); + bind(IndicesQueriesRegistry.class).asEagerSingleton(); } protected void bindHunspellExtension() { diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginManager.java b/core/src/main/java/org/elasticsearch/plugins/PluginManager.java index 9f0134f305b..a692504664d 100644 --- a/core/src/main/java/org/elasticsearch/plugins/PluginManager.java +++ b/core/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -86,6 +86,7 @@ public class PluginManager { "elasticsearch-delete-by-query", "elasticsearch-lang-javascript", "elasticsearch-lang-python", + "elasticsearch-mapper-murmur3", "elasticsearch-mapper-size" ).build(); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java index 6a72718f134..0a3918de46b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java @@ -56,7 +56,6 @@ import java.util.Map; public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue { private final int precision; - private final boolean rehash; private final ValuesSource valuesSource; // Expensive to initialize, so we only initialize it when we have an actual value source @@ -66,11 +65,10 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue private Collector collector; private ValueFormatter formatter; - public CardinalityAggregator(String name, ValuesSource valuesSource, boolean rehash, int precision, ValueFormatter formatter, + public CardinalityAggregator(String name, ValuesSource valuesSource, int precision, ValueFormatter formatter, AggregationContext context, Aggregator parent, List pipelineAggregators, Map metaData) throws IOException { super(name, context, parent, pipelineAggregators, metaData); this.valuesSource = valuesSource; - this.rehash = rehash; this.precision = precision; this.counts = valuesSource == null ? null : new HyperLogLogPlusPlus(precision, context.bigArrays(), 1); this.formatter = formatter; @@ -85,13 +83,6 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue if (valuesSource == null) { return new EmptyCollector(); } - // if rehash is false then the value source is either already hashed, or the user explicitly - // requested not to hash the values (perhaps they already hashed the values themselves before indexing the doc) - // so we can just work with the original value source as is - if (!rehash) { - MurmurHash3Values hashValues = MurmurHash3Values.cast(((ValuesSource.Numeric) valuesSource).longValues(ctx)); - return new DirectCollector(counts, hashValues); - } if (valuesSource instanceof ValuesSource.Numeric) { ValuesSource.Numeric source = (ValuesSource.Numeric) valuesSource; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java index 9320020ca5b..1e660f06f45 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.metrics.cardinality; -import org.elasticsearch.search.aggregations.AggregationExecutionException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregator; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; @@ -35,12 +34,10 @@ import java.util.Map; final class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory { private final long precisionThreshold; - private final boolean rehash; - CardinalityAggregatorFactory(String name, ValuesSourceConfig config, long precisionThreshold, boolean rehash) { + CardinalityAggregatorFactory(String name, ValuesSourceConfig config, long precisionThreshold) { super(name, InternalCardinality.TYPE.name(), config); this.precisionThreshold = precisionThreshold; - this.rehash = rehash; } private int precision(Aggregator parent) { @@ -50,16 +47,13 @@ final class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory pipelineAggregators, Map metaData) throws IOException { - return new CardinalityAggregator(name, null, true, precision(parent), config.formatter(), context, parent, pipelineAggregators, metaData); + return new CardinalityAggregator(name, null, precision(parent), config.formatter(), context, parent, pipelineAggregators, metaData); } @Override protected Aggregator doCreateInternal(ValuesSource valuesSource, AggregationContext context, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { - if (!(valuesSource instanceof ValuesSource.Numeric) && !rehash) { - throw new AggregationExecutionException("Turning off rehashing for cardinality aggregation [" + name + "] on non-numeric values in not allowed"); - } - return new CardinalityAggregator(name, valuesSource, rehash, precision(parent), config.formatter(), context, parent, pipelineAggregators, + return new CardinalityAggregator(name, valuesSource, precision(parent), config.formatter(), context, parent, pipelineAggregators, metaData); } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java index afeca77d3a9..68339457fe7 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java @@ -21,11 +21,9 @@ package org.elasticsearch.search.aggregations.metrics.cardinality; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.core.Murmur3FieldMapper; import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.aggregations.support.ValuesSourceParser; import org.elasticsearch.search.internal.SearchContext; @@ -35,6 +33,7 @@ import java.io.IOException; public class CardinalityParser implements Aggregator.Parser { private static final ParseField PRECISION_THRESHOLD = new ParseField("precision_threshold"); + private static final ParseField REHASH = new ParseField("rehash").withAllDeprecated("no replacement - values will always be rehashed"); @Override public String type() { @@ -44,10 +43,9 @@ public class CardinalityParser implements Aggregator.Parser { @Override public AggregatorFactory parse(String name, XContentParser parser, SearchContext context) throws IOException { - ValuesSourceParser vsParser = ValuesSourceParser.any(name, InternalCardinality.TYPE, context).formattable(false).build(); + ValuesSourceParser vsParser = ValuesSourceParser.any(name, InternalCardinality.TYPE, context).formattable(false).build(); long precisionThreshold = -1; - Boolean rehash = null; XContentParser.Token token; String currentFieldName = null; @@ -57,8 +55,8 @@ public class CardinalityParser implements Aggregator.Parser { } else if (vsParser.token(currentFieldName, token, parser)) { continue; } else if (token.isValue()) { - if ("rehash".equals(currentFieldName)) { - rehash = parser.booleanValue(); + if (context.parseFieldMatcher().match(currentFieldName, REHASH)) { + // ignore } else if (context.parseFieldMatcher().match(currentFieldName, PRECISION_THRESHOLD)) { precisionThreshold = parser.longValue(); } else { @@ -70,15 +68,7 @@ public class CardinalityParser implements Aggregator.Parser { } } - ValuesSourceConfig config = vsParser.config(); - - if (rehash == null && config.fieldContext() != null && config.fieldContext().fieldType() instanceof Murmur3FieldMapper.Murmur3FieldType) { - rehash = false; - } else if (rehash == null) { - rehash = true; - } - - return new CardinalityAggregatorFactory(name, config, precisionThreshold, rehash); + return new CardinalityAggregatorFactory(name, vsParser.config(), precisionThreshold); } diff --git a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help index 26fbf6a2313..61d47759a7a 100644 --- a/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help +++ b/core/src/main/resources/org/elasticsearch/plugins/plugin-install.help @@ -43,6 +43,7 @@ OFFICIAL PLUGINS - elasticsearch-delete-by-query - elasticsearch-lang-javascript - elasticsearch-lang-python + - elasticsearch-mapper-murmur3 - elasticsearch-mapper-size diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java index b376590f68b..743304df941 100644 --- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java +++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java @@ -1116,7 +1116,7 @@ public class GetActionIT extends ESIntegTestCase { @Test public void testGeneratedNumberFieldsUnstored() throws IOException { indexSingleDocumentWithNumericFieldsGeneratedFromText(false, randomBoolean()); - String[] fieldsList = {"token_count", "text.token_count", "murmur", "text.murmur"}; + String[] fieldsList = {"token_count", "text.token_count"}; // before refresh - document is only in translog assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList); refresh(); @@ -1130,7 +1130,7 @@ public class GetActionIT extends ESIntegTestCase { @Test public void testGeneratedNumberFieldsStored() throws IOException { indexSingleDocumentWithNumericFieldsGeneratedFromText(true, randomBoolean()); - String[] fieldsList = {"token_count", "text.token_count", "murmur", "text.murmur"}; + String[] fieldsList = {"token_count", "text.token_count"}; // before refresh - document is only in translog assertGetFieldsNull(indexOrAlias(), "doc", "1", fieldsList); assertGetFieldsException(indexOrAlias(), "doc", "1", fieldsList); @@ -1159,10 +1159,6 @@ public class GetActionIT extends ESIntegTestCase { " \"analyzer\": \"standard\",\n" + " \"store\": \"" + storedString + "\"" + " },\n" + - " \"murmur\": {\n" + - " \"type\": \"murmur3\",\n" + - " \"store\": \"" + storedString + "\"" + - " },\n" + " \"text\": {\n" + " \"type\": \"string\",\n" + " \"fields\": {\n" + @@ -1170,10 +1166,6 @@ public class GetActionIT extends ESIntegTestCase { " \"type\": \"token_count\",\n" + " \"analyzer\": \"standard\",\n" + " \"store\": \"" + storedString + "\"" + - " },\n" + - " \"murmur\": {\n" + - " \"type\": \"murmur3\",\n" + - " \"store\": \"" + storedString + "\"" + " }\n" + " }\n" + " }" + @@ -1185,7 +1177,6 @@ public class GetActionIT extends ESIntegTestCase { assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); ensureGreen(); String doc = "{\n" + - " \"murmur\": \"Some value that can be hashed\",\n" + " \"token_count\": \"A text with five words.\",\n" + " \"text\": \"A text with five words.\"\n" + "}\n"; diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 974a4a21557..8edc47410a3 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -535,7 +535,7 @@ public class InternalEngineTests extends ESTestCase { } @Override - public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { + public IndexSearcher wrap(EngineConfig engineConfig, IndexSearcher searcher) throws EngineException { counter.incrementAndGet(); return searcher; } diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java b/core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java index 08746b2a5e5..07a600f1484 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java @@ -550,6 +550,7 @@ public class PluginManagerIT extends ESIntegTestCase { PluginManager.checkForOfficialPlugins("elasticsearch-delete-by-query"); PluginManager.checkForOfficialPlugins("elasticsearch-lang-javascript"); PluginManager.checkForOfficialPlugins("elasticsearch-lang-python"); + PluginManager.checkForOfficialPlugins("elasticsearch-mapper-murmur3"); try { PluginManager.checkForOfficialPlugins("elasticsearch-mapper-attachment"); diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java index 491e4f694c9..d77e4d1ccd0 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java @@ -61,54 +61,23 @@ public class CardinalityIT extends ESIntegTestCase { jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("str_value") .field("type", "string") - .startObject("fields") - .startObject("hash") - .field("type", "murmur3") - .endObject() - .endObject() .endObject() .startObject("str_values") .field("type", "string") - .startObject("fields") - .startObject("hash") - .field("type", "murmur3") - .endObject() - .endObject() .endObject() .startObject("l_value") .field("type", "long") - .startObject("fields") - .startObject("hash") - .field("type", "murmur3") - .endObject() - .endObject() .endObject() .startObject("l_values") .field("type", "long") - .startObject("fields") - .startObject("hash") - .field("type", "murmur3") - .endObject() - .endObject() .endObject() - .startObject("d_value") - .field("type", "double") - .startObject("fields") - .startObject("hash") - .field("type", "murmur3") - .endObject() - .endObject() - .endObject() - .startObject("d_values") - .field("type", "double") - .startObject("fields") - .startObject("hash") - .field("type", "murmur3") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject().endObject()).execute().actionGet(); + .startObject("d_value") + .field("type", "double") + .endObject() + .startObject("d_values") + .field("type", "double") + .endObject() + .endObject().endObject().endObject()).execute().actionGet(); numDocs = randomIntBetween(2, 100); precisionThreshold = randomIntBetween(0, 1 << randomInt(20)); @@ -145,12 +114,12 @@ public class CardinalityIT extends ESIntegTestCase { assertThat(count.getValue(), greaterThan(0L)); } } - private String singleNumericField(boolean hash) { - return (randomBoolean() ? "l_value" : "d_value") + (hash ? ".hash" : ""); + private String singleNumericField() { + return randomBoolean() ? "l_value" : "d_value"; } private String multiNumericField(boolean hash) { - return (randomBoolean() ? "l_values" : "d_values") + (hash ? ".hash" : ""); + return randomBoolean() ? "l_values" : "d_values"; } @Test @@ -195,24 +164,10 @@ public class CardinalityIT extends ESIntegTestCase { assertCount(count, numDocs); } - @Test - public void singleValuedStringHashed() throws Exception { - SearchResponse response = client().prepareSearch("idx").setTypes("type") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value.hash")) - .execute().actionGet(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs); - } - @Test public void singleValuedNumeric() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField(false))) + .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField())) .execute().actionGet(); assertSearchResponse(response); @@ -229,7 +184,7 @@ public class CardinalityIT extends ESIntegTestCase { SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()) .addAggregation( global("global").subAggregation( - cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField(false)))) + cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField()))) .execute().actionGet(); assertSearchResponse(searchResponse); @@ -254,7 +209,7 @@ public class CardinalityIT extends ESIntegTestCase { @Test public void singleValuedNumericHashed() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField(true))) + .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField())) .execute().actionGet(); assertSearchResponse(response); @@ -279,20 +234,6 @@ public class CardinalityIT extends ESIntegTestCase { assertCount(count, numDocs * 2); } - @Test - public void multiValuedStringHashed() throws Exception { - SearchResponse response = client().prepareSearch("idx").setTypes("type") - .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_values.hash")) - .execute().actionGet(); - - assertSearchResponse(response); - - Cardinality count = response.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, numDocs * 2); - } - @Test public void multiValuedNumeric() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") @@ -356,7 +297,7 @@ public class CardinalityIT extends ESIntegTestCase { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation( cardinality("cardinality").precisionThreshold(precisionThreshold).script( - new Script("doc['" + singleNumericField(false) + "'].value"))) + new Script("doc['" + singleNumericField() + "'].value"))) .execute().actionGet(); assertSearchResponse(response); @@ -417,7 +358,7 @@ public class CardinalityIT extends ESIntegTestCase { public void singleValuedNumericValueScript() throws Exception { SearchResponse response = client().prepareSearch("idx").setTypes("type") .addAggregation( - cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField(false)) + cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField()) .script(new Script("_value"))) .execute().actionGet(); @@ -464,23 +405,4 @@ public class CardinalityIT extends ESIntegTestCase { } } - @Test - public void asSubAggHashed() throws Exception { - SearchResponse response = client().prepareSearch("idx").setTypes("type") - .addAggregation(terms("terms").field("str_value") - .collectMode(randomFrom(SubAggCollectionMode.values())) - .subAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_values.hash"))) - .execute().actionGet(); - - assertSearchResponse(response); - - Terms terms = response.getAggregations().get("terms"); - for (Terms.Bucket bucket : terms.getBuckets()) { - Cardinality count = bucket.getAggregations().get("cardinality"); - assertThat(count, notNullValue()); - assertThat(count.getName(), equalTo("cardinality")); - assertCount(count, 2); - } - } - } diff --git a/docs/plugins/mapper-murmur3.asciidoc b/docs/plugins/mapper-murmur3.asciidoc new file mode 100644 index 00000000000..2889eb4aa68 --- /dev/null +++ b/docs/plugins/mapper-murmur3.asciidoc @@ -0,0 +1,101 @@ +[[mapper-murmur3]] +=== Mapper Murmur3 Plugin + +The mapper-murmur3 plugin provides the ability to compute hash of field values +at index-time and store them in the index. This can sometimes be helpful when +running cardinality aggregations on high-cardinality and large string fields. + +[[mapper-murmur3-install]] +[float] +==== Installation + +This plugin can be installed using the plugin manager: + +[source,sh] +---------------------------------------------------------------- +sudo bin/plugin install mapper-murmur3 +---------------------------------------------------------------- + +The plugin must be installed on every node in the cluster, and each node must +be restarted after installation. + +[[mapper-murmur3-remove]] +[float] +==== Removal + +The plugin can be removed with the following command: + +[source,sh] +---------------------------------------------------------------- +sudo bin/plugin remove mapper-murmur3 +---------------------------------------------------------------- + +The node must be stopped before removing the plugin. + +[[mapper-murmur3-usage]] +==== Using the `murmur3` field + +The `murmur3` is typically used within a multi-field, so that both the original +value and its hash are stored in the index: + +[source,js] +-------------------------- +PUT my_index +{ + "mappings": { + "my_type": { + "properties": { + "my_field": { + "type": "string", + "fields": { + "hash": { + "type": "murmur3" + } + } + } + } + } + } +} +-------------------------- +// AUTOSENSE + +Such a mapping would allow to refer to `my_field.hash` in order to get hashes +of the values of the `my_field` field. This is only useful in order to run +`cardinality` aggregations: + +[source,js] +-------------------------- +# Example documents +PUT my_index/my_type/1 +{ + "my_field": "This is a document" +} + +PUT my_index/my_type/2 +{ + "my_field": "This is another document" +} + +GET my_index/_search +{ + "aggs": { + "my_field_cardinality": { + "cardinality": { + "field": "my_field.hash" <1> + } + } + } +} +-------------------------- +// AUTOSENSE + +<1> Counting unique values on the `my_field.hash` field + +Running a `cardinality` aggregation on the `my_field` field directly would +yield the same result, however using `my_field.hash` instead might result in +a speed-up if the field has a high-cardinality. On the other hand, it is +discouraged to use the `murmur3` field on numeric fields and string fields +that are not almost unique as the use of a `murmur3` field is unlikely to +bring significant speed-ups, while increasing the amount of disk space required +to store the index. diff --git a/docs/plugins/mapper.asciidoc b/docs/plugins/mapper.asciidoc index 1d9996d9818..226fc4e40d0 100644 --- a/docs/plugins/mapper.asciidoc +++ b/docs/plugins/mapper.asciidoc @@ -14,5 +14,10 @@ The mapper-size plugin provides the `_size` meta field which, when enabled, indexes the size in bytes of the original {ref}/mapping-source-field.html[`_source`] field. -include::mapper-size.asciidoc[] +<>:: +The mapper-murmur3 plugin allows hashes to be computed at index-time and stored +in the index for later use with the `cardinality` aggregation. + +include::mapper-size.asciidoc[] +include::mapper-murmur3.asciidoc[] diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc index 0b484288b1c..1eb0c08772f 100644 --- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc @@ -23,9 +23,9 @@ match a query: ==== Precision control -This aggregation also supports the `precision_threshold` and `rehash` options: +This aggregation also supports the `precision_threshold` option: -experimental[The `precision_threshold` and `rehash` options are specific to the current internal implementation of the `cardinality` agg, which may change in the future] +experimental[The `precision_threshold` option is specific to the current internal implementation of the `cardinality` agg, which may change in the future] [source,js] -------------------------------------------------- @@ -34,8 +34,7 @@ experimental[The `precision_threshold` and `rehash` options are specific to the "author_count" : { "cardinality" : { "field" : "author_hash", - "precision_threshold": 100, <1> - "rehash": false <2> + "precision_threshold": 100 <1> } } } @@ -49,11 +48,6 @@ supported value is 40000, thresholds above this number will have the same effect as a threshold of 40000. Default value depends on the number of parent aggregations that multiple create buckets (such as terms or histograms). -<2> If you computed a hash on client-side, stored it into your documents and want -Elasticsearch to use them to compute counts using this hash function without -rehashing values, it is possible to specify `rehash: false`. Default value is -`true`. Please note that the hash must be indexed as a long when `rehash` is -false. ==== Counts are approximate @@ -86,47 +80,11 @@ counting millions of items. ==== Pre-computed hashes -If you don't want Elasticsearch to re-compute hashes on every run of this -aggregation, it is possible to use pre-computed hashes, either by computing a -hash on client-side, indexing it and specifying `rehash: false`, or by using -the special `murmur3` field mapper, typically in the context of a `multi-field` -in the mapping: - -[source,js] --------------------------------------------------- -{ - "author": { - "type": "string", - "fields": { - "hash": { - "type": "murmur3" - } - } - } -} --------------------------------------------------- - -With such a mapping, Elasticsearch is going to compute hashes of the `author` -field at indexing time and store them in the `author.hash` field. This -way, unique counts can be computed using the cardinality aggregation by only -loading the hashes into memory, not the values of the `author` field, and -without computing hashes on the fly: - -[source,js] --------------------------------------------------- -{ - "aggs" : { - "author_count" : { - "cardinality" : { - "field" : "author.hash" - } - } - } -} --------------------------------------------------- - -NOTE: `rehash` is automatically set to `false` when computing unique counts on -a `murmur3` field. +On string fields that have a high cardinality, it might be faster to store the +hash of your field values in your index and then run the cardinality aggregation +on this field. This can either be done by providing hash values from client-side +or by letting elasticsearch compute hash values for you by using the +{plugins}/mapper-size.html[`mapper-murmur3`] plugin. NOTE: Pre-computing hashes is usually only useful on very large and/or high-cardinality fields as it saves CPU and memory. However, on numeric diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index abefda42085..52cd41e37e5 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -33,6 +33,7 @@ document: <>:: `completion` to provide auto-complete suggestions <>:: `token_count` to count the number of tokens in a string +{plugins}/mapper-size.html[`mapper-murmur3`]:: `murmur3` to compute hashes of values at index-time and store them in the index Attachment datatype:: diff --git a/docs/reference/migration/migrate_2_0/removals.asciidoc b/docs/reference/migration/migrate_2_0/removals.asciidoc index 60f7422876c..ab764691042 100644 --- a/docs/reference/migration/migrate_2_0/removals.asciidoc +++ b/docs/reference/migration/migrate_2_0/removals.asciidoc @@ -41,6 +41,16 @@ can install the plugin with: The `_shutdown` API has been removed without a replacement. Nodes should be managed via the operating system and the provided start/stop scripts. +==== `murmur3` is now a plugin + +The `murmur3` field, which indexes hashes of the field values, has been moved +out of core and is available as a plugin. It can be installed as: + +[source,sh] +------------------ +./bin/plugin install mapper-murmur3 +------------------ + ==== `_size` is now a plugin The `_size` meta-data field, which indexes the size in bytes of the original diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc index ea14c58e212..60aab15b54a 100644 --- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc +++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc @@ -59,7 +59,7 @@ standard -180:180 / -90:90 coordinate system. (default is `false`). accept geo points with invalid latitude or longitude (default is `false`). |`type` |Set to one of `indexed` or `memory` to defines whether this filter will -be executed in memory or indexed. See <> below for further details +be executed in memory or indexed. See <> below for further details Default is `memory`. |======================================================================= @@ -214,6 +214,7 @@ a single location / point matches the filter, the document will be included in the filter [float] +[[geo-bbox-type]] ==== Type The type of the bounding box execution by default is set to `memory`, diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc index d778999c251..ff53a351b67 100644 --- a/docs/reference/query-dsl/geo-polygon-query.asciidoc +++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc @@ -39,6 +39,7 @@ standard -180:180 / -90:90 coordinate system. (default is `false`). |`ignore_malformed` |Set to `true` to accept geo points with invalid latitude or longitude (default is `false`). +|======================================================================= [float] ==== Allowed Formats diff --git a/plugins/mapper-murmur3/licenses/no_deps.txt b/plugins/mapper-murmur3/licenses/no_deps.txt new file mode 100644 index 00000000000..8cce254d037 --- /dev/null +++ b/plugins/mapper-murmur3/licenses/no_deps.txt @@ -0,0 +1 @@ +This plugin has no third party dependencies diff --git a/plugins/mapper-murmur3/pom.xml b/plugins/mapper-murmur3/pom.xml new file mode 100644 index 00000000000..9d4c7db0000 --- /dev/null +++ b/plugins/mapper-murmur3/pom.xml @@ -0,0 +1,43 @@ + + + + + 4.0.0 + + + org.elasticsearch.plugin + elasticsearch-plugin + 2.1.0-SNAPSHOT + + + elasticsearch-mapper-murmur3 + Elasticsearch Mapper Murmur3 plugin + The Mapper Murmur3 plugin allows to compute hashes of a field's values at index-time and to store them in the index. + + + org.elasticsearch.plugin.mapper.MapperMurmur3Plugin + mapper_murmur3 + false + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + + + diff --git a/plugins/mapper-murmur3/rest-api-spec/test/mapper_murmur3/10_basic.yaml b/plugins/mapper-murmur3/rest-api-spec/test/mapper_murmur3/10_basic.yaml new file mode 100644 index 00000000000..4ed879c7192 --- /dev/null +++ b/plugins/mapper-murmur3/rest-api-spec/test/mapper_murmur3/10_basic.yaml @@ -0,0 +1,65 @@ +# Integration tests for Mapper Murmur3 components +# + +--- +"Mapper Murmur3": + + - do: + indices.create: + index: test + body: + mappings: + type1: { "properties": { "foo": { "type": "string", "fields": { "hash": { "type": "murmur3" } } } } } + + - do: + index: + index: test + type: type1 + id: 0 + body: { "foo": null } + + - do: + indices.refresh: {} + + - do: + search: + body: { "aggs": { "foo_count": { "cardinality": { "field": "foo.hash" } } } } + + - match: { aggregations.foo_count.value: 0 } + + - do: + index: + index: test + type: type1 + id: 1 + body: { "foo": "bar" } + + - do: + index: + index: test + type: type1 + id: 2 + body: { "foo": "baz" } + + - do: + index: + index: test + type: type1 + id: 3 + body: { "foo": "quux" } + + - do: + index: + index: test + type: type1 + id: 4 + body: { "foo": "bar" } + + - do: + indices.refresh: {} + + - do: + search: + body: { "aggs": { "foo_count": { "cardinality": { "field": "foo.hash" } } } } + + - match: { aggregations.foo_count.value: 3 } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java similarity index 85% rename from core/src/main/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapper.java rename to plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 5e7b664aceb..60c31c3f765 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -17,9 +17,10 @@ * under the License. */ -package org.elasticsearch.index.mapper.core; +package org.elasticsearch.index.mapper.murmur3; import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; @@ -31,12 +32,13 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.core.LongFieldMapper; +import org.elasticsearch.index.mapper.core.NumberFieldMapper; import java.io.IOException; import java.util.List; import java.util.Map; -import static org.elasticsearch.index.mapper.MapperBuilders.murmur3Field; import static org.elasticsearch.index.mapper.core.TypeParsers.parseNumberField; public class Murmur3FieldMapper extends LongFieldMapper { @@ -45,6 +47,9 @@ public class Murmur3FieldMapper extends LongFieldMapper { public static class Defaults extends LongFieldMapper.Defaults { public static final MappedFieldType FIELD_TYPE = new Murmur3FieldType(); + static { + FIELD_TYPE.freeze(); + } } public static class Builder extends NumberFieldMapper.Builder { @@ -65,6 +70,17 @@ public class Murmur3FieldMapper extends LongFieldMapper { return fieldMapper; } + @Override + protected void setupFieldType(BuilderContext context) { + super.setupFieldType(context); + if (context.indexCreatedVersion().onOrAfter(Version.V_2_0_0_beta1)) { + fieldType.setIndexOptions(IndexOptions.NONE); + defaultFieldType.setIndexOptions(IndexOptions.NONE); + fieldType.setHasDocValues(true); + defaultFieldType.setHasDocValues(true); + } + } + @Override protected NamedAnalyzer makeNumberAnalyzer(int precisionStep) { return NumericLongAnalyzer.buildNamedAnalyzer(precisionStep); @@ -80,7 +96,7 @@ public class Murmur3FieldMapper extends LongFieldMapper { @Override @SuppressWarnings("unchecked") public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = murmur3Field(name); + Builder builder = new Builder(name); // tweaking these settings is no longer allowed, the entire purpose of murmur3 fields is to store a hash if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0_beta1)) { @@ -92,6 +108,10 @@ public class Murmur3FieldMapper extends LongFieldMapper { } } + if (parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { + builder.indexOptions(IndexOptions.DOCS); + } + parseNumberField(builder, name, node, parserContext); // Because this mapper extends LongFieldMapper the null_value field will be added to the JSON when transferring cluster state // between nodes so we have to remove the entry here so that the validation doesn't fail @@ -104,7 +124,8 @@ public class Murmur3FieldMapper extends LongFieldMapper { // this only exists so a check can be done to match the field type to using murmur3 hashing... public static class Murmur3FieldType extends LongFieldMapper.LongFieldType { - public Murmur3FieldType() {} + public Murmur3FieldType() { + } protected Murmur3FieldType(Murmur3FieldType ref) { super(ref); diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/RegisterMurmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/RegisterMurmur3FieldMapper.java new file mode 100644 index 00000000000..5a6a71222c0 --- /dev/null +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/RegisterMurmur3FieldMapper.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.murmur3; + +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.AbstractIndexComponent; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.mapper.MapperService; + +public class RegisterMurmur3FieldMapper extends AbstractIndexComponent { + + @Inject + public RegisterMurmur3FieldMapper(Index index, Settings indexSettings, MapperService mapperService) { + super(index, indexSettings); + mapperService.documentMapperParser().putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + } + +} diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3IndexModule.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3IndexModule.java new file mode 100644 index 00000000000..51054d774bd --- /dev/null +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3IndexModule.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.mapper; + +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.index.mapper.murmur3.RegisterMurmur3FieldMapper; + +public class MapperMurmur3IndexModule extends AbstractModule { + + @Override + protected void configure() { + bind(RegisterMurmur3FieldMapper.class).asEagerSingleton(); + } +} diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java new file mode 100644 index 00000000000..9b6611decde --- /dev/null +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.mapper; + +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.plugins.AbstractPlugin; + +import java.util.Collection; +import java.util.Collections; + +public class MapperMurmur3Plugin extends AbstractPlugin { + + @Override + public String name() { + return "mapper-murmur3"; + } + + @Override + public String description() { + return "A mapper that allows to precompute murmur3 hashes of values at index-time and store them in the index"; + } + + @Override + public Collection> indexModules() { + return Collections.>singleton(MapperMurmur3IndexModule.class); + } + +} diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java new file mode 100644 index 00000000000..f440a343bc6 --- /dev/null +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3RestIT.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.murmur3; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; + +import java.io.IOException; + +public class MapperMurmur3RestIT extends ESRestTestCase { + + public MapperMurmur3RestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return createParameters(0, 1); + } +} + diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java similarity index 80% rename from core/src/test/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapperTests.java rename to plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index d4cfcb8ccde..676a5c4c1cb 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -17,9 +17,11 @@ * under the License. */ -package org.elasticsearch.index.mapper.core; +package org.elasticsearch.index.mapper.murmur3; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; @@ -28,9 +30,12 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; +import java.util.Arrays; + public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { IndexService indexService; @@ -40,6 +45,22 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { public void before() { indexService = createIndex("test"); parser = indexService.mapperService().documentMapperParser(); + parser.putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + } + + public void testDefaults() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties").startObject("field") + .field("type", "murmur3") + .endObject().endObject().endObject().endObject().string(); + DocumentMapper mapper = parser.parse(mapping); + ParsedDocument parsedDoc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes()); + IndexableField[] fields = parsedDoc.rootDoc().getFields("field"); + assertNotNull(fields); + assertEquals(Arrays.toString(fields), 1, fields.length); + IndexableField field = fields[0]; + assertEquals(IndexOptions.NONE, field.fieldType().indexOptions()); + assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType()); } public void testDocValuesSettingNotAllowed() throws Exception { @@ -100,6 +121,7 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); indexService = createIndex("test_bwc", settings); parser = indexService.mapperService().documentMapperParser(); + parser.putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field") .field("type", "murmur3") @@ -115,6 +137,7 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); indexService = createIndex("test_bwc", settings); parser = indexService.mapperService().documentMapperParser(); + parser.putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field") .field("type", "murmur3") diff --git a/plugins/pom.xml b/plugins/pom.xml index d89a441e233..fbf88b089fe 100644 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -436,6 +436,7 @@ delete-by-query lang-python lang-javascript + mapper-murmur3 mapper-size jvm-example site-example diff --git a/qa/smoke-test-plugins/pom.xml b/qa/smoke-test-plugins/pom.xml index d479b16b7f0..b56bc35e57b 100644 --- a/qa/smoke-test-plugins/pom.xml +++ b/qa/smoke-test-plugins/pom.xml @@ -333,6 +333,14 @@ true + + org.elasticsearch.plugin + elasticsearch-mapper-murmur3 + ${elasticsearch.version} + zip + true + + org.elasticsearch.plugin elasticsearch-mapper-size