From 17a32fca0381f51f41a4217b55263fa12d300f17 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 6 May 2014 10:01:15 +0200 Subject: [PATCH] [TEST] Random dynamic templates. This change randomly indexes the _id field and randomizes field data formats and loading. Close #5834 --- .../context/GeolocationContextMapping.java | 8 +- .../DisabledFieldDataFormatTests.java | 2 + .../indices/cache/CacheTests.java | 4 +- .../breaker/CircuitBreakerServiceTests.java | 4 +- .../mapping/SimpleGetMappingsTests.java | 2 + .../indices/mapping/UpdateMappingTests.java | 2 + .../routing/SimpleRoutingTests.java | 6 ++ .../child/SimpleChildQuerySearchTests.java | 30 ++++++- .../test/ElasticsearchIntegrationTest.java | 84 ++++++++++++++++++- .../test/ElasticsearchTestCase.java | 13 +-- .../test/rest/ElasticsearchRestTests.java | 2 + 11 files changed, 138 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java b/src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java index 1016d7aa6a2..aabc85a9113 100644 --- a/src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java +++ b/src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java @@ -621,8 +621,12 @@ public class GeolocationContextMapping extends ContextMapping { for (int i = 0 ; i < lonFields.length ; i++) { IndexableField lonField = lonFields[i]; IndexableField latField = latFields[i]; - spare.reset(latField.numericValue().doubleValue(), lonField.numericValue().doubleValue()); - geohashes.add(spare.geohash()); + assert lonField.fieldType().docValueType() == latField.fieldType().docValueType(); + // we write doc values fields differently: one field for all values, so we need to only care about indexed fields + if (lonField.fieldType().docValueType() == null) { + spare.reset(latField.numericValue().doubleValue(), lonField.numericValue().doubleValue()); + geohashes.add(spare.geohash()); + } } } else { geohashes = mapping.defaultLocations; diff --git a/src/test/java/org/elasticsearch/index/fielddata/DisabledFieldDataFormatTests.java b/src/test/java/org/elasticsearch/index/fielddata/DisabledFieldDataFormatTests.java index d4ade8e868f..a18bbfa2a5b 100644 --- a/src/test/java/org/elasticsearch/index/fielddata/DisabledFieldDataFormatTests.java +++ b/src/test/java/org/elasticsearch/index/fielddata/DisabledFieldDataFormatTests.java @@ -24,10 +24,12 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +@ClusterScope(randomDynamicTemplates = false) public class DisabledFieldDataFormatTests extends ElasticsearchIntegrationTest { @Override diff --git a/src/test/java/org/elasticsearch/indices/cache/CacheTests.java b/src/test/java/org/elasticsearch/indices/cache/CacheTests.java index 8435d90eeb2..41da58c7c62 100644 --- a/src/test/java/org/elasticsearch/indices/cache/CacheTests.java +++ b/src/test/java/org/elasticsearch/indices/cache/CacheTests.java @@ -29,16 +29,16 @@ import org.elasticsearch.index.query.FilterBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; +import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; import org.junit.Test; import static org.elasticsearch.index.query.QueryBuilders.filteredQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.ElasticsearchIntegrationTest.*; import static org.hamcrest.Matchers.*; /** */ -@ClusterScope(scope= Scope.SUITE, numDataNodes =1, numClientNodes = 0) +@ClusterScope(scope= Scope.SUITE, numDataNodes =1, numClientNodes = 0, randomDynamicTemplates = false) public class CacheTests extends ElasticsearchIntegrationTest { @Override diff --git a/src/test/java/org/elasticsearch/indices/fielddata/breaker/CircuitBreakerServiceTests.java b/src/test/java/org/elasticsearch/indices/fielddata/breaker/CircuitBreakerServiceTests.java index 41fa51606ba..2300578a541 100644 --- a/src/test/java/org/elasticsearch/indices/fielddata/breaker/CircuitBreakerServiceTests.java +++ b/src/test/java/org/elasticsearch/indices/fielddata/breaker/CircuitBreakerServiceTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.junit.annotations.TestLogging; import org.junit.Test; @@ -31,7 +32,6 @@ import java.util.Arrays; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; -import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope.TEST; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; @@ -40,7 +40,7 @@ import static org.hamcrest.CoreMatchers.containsString; /** * Integration tests for InternalCircuitBreakerService */ -@ClusterScope(scope = TEST) +@ClusterScope(scope = TEST, randomDynamicTemplates = false) public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest { private String randomRidiculouslySmallLimit() { diff --git a/src/test/java/org/elasticsearch/indices/mapping/SimpleGetMappingsTests.java b/src/test/java/org/elasticsearch/indices/mapping/SimpleGetMappingsTests.java index 5352994e7a3..adfd37a0c15 100644 --- a/src/test/java/org/elasticsearch/indices/mapping/SimpleGetMappingsTests.java +++ b/src/test/java/org/elasticsearch/indices/mapping/SimpleGetMappingsTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.common.Priority; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.junit.Test; import java.io.IOException; @@ -35,6 +36,7 @@ import static org.hamcrest.Matchers.notNullValue; /** * */ +@ClusterScope(randomDynamicTemplates = false) public class SimpleGetMappingsTests extends ElasticsearchIntegrationTest { @Test diff --git a/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingTests.java b/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingTests.java index 4af7e7ddeb7..f7485374359 100644 --- a/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingTests.java +++ b/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.hamcrest.Matchers; import org.junit.Test; @@ -55,6 +56,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.*; +@ClusterScope(randomDynamicTemplates = false) public class UpdateMappingTests extends ElasticsearchIntegrationTest { @Test diff --git a/src/test/java/org/elasticsearch/routing/SimpleRoutingTests.java b/src/test/java/org/elasticsearch/routing/SimpleRoutingTests.java index 8c39936c2aa..0eaa2778f9e 100644 --- a/src/test/java/org/elasticsearch/routing/SimpleRoutingTests.java +++ b/src/test/java/org/elasticsearch/routing/SimpleRoutingTests.java @@ -303,6 +303,12 @@ public class SimpleRoutingTests extends ElasticsearchIntegrationTest { client().admin().indices().prepareCreate("test") .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("_routing").field("required", true).field("path", "routing_field").endObject() + .startObject("properties") + .startObject("routing_field") + .field("type", "long") + .field("doc_values", false) // TODO this test fails with doc values https://github.com/elasticsearch/elasticsearch/pull/5858 + .endObject() + .endObject() .endObject().endObject()) .execute().actionGet(); ensureGreen(); diff --git a/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java b/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java index 3b8b593ee32..447c4c4aa07 100644 --- a/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java +++ b/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java @@ -33,6 +33,9 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.fielddata.FieldDataType; +import org.elasticsearch.index.mapper.FieldMapper.Loading; import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.query.*; import org.elasticsearch.index.search.child.ScoreType; @@ -241,8 +244,18 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest { @Test public void testClearIdCacheBug() throws Exception { + // enforce lazy loading to make sure that p/c stats are not counted as part of field data assertAcked(prepareCreate("test") - .addMapping("parent")); + .addMapping("parent", XContentFactory.jsonBuilder().startObject().startObject("parent") + .startObject("properties") + .startObject("p_field") + .field("type", "string") + .startObject("fielddata") + .field(FieldDataType.FORMAT_KEY, Loading.LAZY) + .endObject() + .endObject() + .endObject().endObject().endObject())); + ensureGreen(); client().prepareIndex("test", "parent", "p0").setSource("p_field", "p_value0").get(); @@ -256,7 +269,18 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest { // Now add mapping + children client().admin().indices().preparePutMapping("test").setType("child") - .setSource("_parent", "type=parent") + .setSource(XContentFactory.jsonBuilder().startObject().startObject("child") + .startObject("_parent") + .field("type", "parent") + .endObject() + .startObject("properties") + .startObject("c_field") + .field("type", "string") + .startObject("fielddata") + .field(FieldDataType.FORMAT_KEY, Loading.LAZY) + .endObject() + .endObject() + .endObject().endObject().endObject()) .get(); // index simple data @@ -1460,7 +1484,7 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest { GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("test").get(); Map mapping = getMappingsResponse.getMappings().get("test").get("child").getSourceAsMap(); - assertThat(mapping.size(), equalTo(1)); + assertThat(mapping.size(), greaterThanOrEqualTo(1)); // there are potentially some meta fields configured randomly assertThat(mapping.get("properties"), notNullValue()); try { diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java index ad1c658a258..267dcae23cc 100644 --- a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java +++ b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java @@ -39,6 +39,7 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRespon import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.get.GetResponse; @@ -65,8 +66,12 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.discovery.zen.elect.ElectMasterService; +import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.FieldMapper.Loading; +import org.elasticsearch.index.mapper.internal.IdFieldMapper; import org.elasticsearch.index.merge.policy.*; import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider; import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule; @@ -294,7 +299,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase * Creates a randomized index template. This template is used to pass in randomized settings on a * per index basis. Allows to enable/disable the randomization for number of shards and replicas */ - private void randomIndexTemplate() { + private void randomIndexTemplate() throws IOException { // TODO move settings for random directory etc here into the index based randomized settings. if (immutableCluster().size() > 0) { ImmutableSettings.Builder randomSettingsBuilder = @@ -306,11 +311,72 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase //use either 0 or 1 replica, yet a higher amount when possible, but only rarely .put(SETTING_NUMBER_OF_REPLICAS, between(0, getRandom().nextInt(10) > 0 ? 1 : immutableCluster().numDataNodes() - 1)); } - client().admin().indices().preparePutTemplate("random_index_template") + XContentBuilder mappings = null; + if (frequently() && randomDynamicTemplates()) { + mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_"); + if (randomBoolean()) { + mappings.startObject(IdFieldMapper.NAME) + .field("index", randomFrom("not_analyzed", "no")) + .endObject(); + } + mappings.startArray("dynamic_templates") + .startObject() + .startObject("template-strings") + .field("match_mapping_type", "string") + .startObject("mapping") + .startObject("fielddata") + .field(FieldDataType.FORMAT_KEY, randomFrom("paged_bytes", "fst")) // unfortunately doc values only work on not_analyzed fields + .field(Loading.KEY, randomFrom(Loading.values())) + .endObject() + .endObject() + .endObject() + .endObject() + .startObject() + .startObject("template-longs") + .field("match_mapping_type", "long") + .startObject("mapping") + .startObject("fielddata") + .field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values")) + .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER)) + .endObject() + .endObject() + .endObject() + .endObject() + .startObject() + .startObject("template-doubles") + .field("match_mapping_type", "double") + .startObject("mapping") + .startObject("fielddata") + .field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values")) + .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER)) + .endObject() + .endObject() + .endObject() + .endObject() + .startObject() + .startObject("template-geo_points") + .field("match_mapping_type", "geo_point") + .startObject("mapping") + .startObject("fielddata") + .field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values")) + .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER)) + .endObject() + .endObject() + .endObject() + .endObject() + .endArray(); + mappings.endObject().endObject(); + } + + PutIndexTemplateRequestBuilder putTemplate = client().admin().indices() + .preparePutTemplate("random_index_template") .setTemplate("*") .setOrder(0) - .setSettings(randomSettingsBuilder) - .execute().actionGet(); + .setSettings(randomSettingsBuilder); + if (mappings != null) { + putTemplate.addMapping("_default_", mappings); + } + assertAcked(putTemplate.execute().actionGet()); } } @@ -1057,6 +1123,11 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase * ratio in the interval [0..1] is used. */ double transportClientRatio() default -1; + + /** + * Return whether or not to enable dynamic templates for the mappings. + */ + boolean randomDynamicTemplates() default true; } private class LatchedActionListener implements ActionListener { @@ -1148,6 +1219,11 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase return annotation == null ? TestCluster.DEFAULT_NUM_CLIENT_NODES : annotation.numClientNodes(); } + private boolean randomDynamicTemplates() { + ClusterScope annotation = getAnnotation(this.getClass()); + return annotation == null ? true : annotation.randomDynamicTemplates(); + } + /** * This method is used to obtain settings for the Nth node in the cluster. * Nodes in this cluster are associated with an ordinal number such that nodes can diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java b/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java index a507b56b5f0..0c1fdbc94fe 100644 --- a/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java +++ b/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java @@ -18,11 +18,9 @@ */ package org.elasticsearch.test; -import com.carrotsearch.randomizedtesting.annotations.Listeners; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.*; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; -import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import org.apache.lucene.store.MockDirectoryWrapper; @@ -30,7 +28,6 @@ import org.apache.lucene.util.AbstractRandomizedTest; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TimeUnits; import org.elasticsearch.Version; -import org.elasticsearch.test.cache.recycler.MockPageCacheRecycler; import org.elasticsearch.client.Requests; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.ESLogger; @@ -39,11 +36,11 @@ import org.elasticsearch.common.util.concurrent.EsAbortPolicy; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.cache.recycler.MockBigArrays; +import org.elasticsearch.test.cache.recycler.MockPageCacheRecycler; import org.elasticsearch.test.junit.listeners.LoggingListener; import org.elasticsearch.test.store.MockDirectoryHelper; import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import java.io.Closeable; @@ -293,4 +290,8 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest { return threadGroup.getName(); } } + + public static T randomFrom(T... values) { + return RandomizedTest.randomFrom(values); + } } diff --git a/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java b/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java index 58ce88e8f8c..c4cea6b5de1 100644 --- a/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java +++ b/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java @@ -25,6 +25,7 @@ import com.google.common.collect.Lists; import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.rest.client.RestException; import org.elasticsearch.test.rest.parser.RestTestParseException; import org.elasticsearch.test.rest.parser.RestTestSuiteParser; @@ -51,6 +52,7 @@ import java.util.Set; //tests distribution disabled for now since it causes reporting problems, // due to the non unique suite name //@ReplicateOnEachVm +@ClusterScope(randomDynamicTemplates = false) public class ElasticsearchRestTests extends ElasticsearchIntegrationTest { /**