[TEST] Random dynamic templates.

This change randomly indexes the _id field and randomizes field data formats
and loading.

Close #5834

src/main/java/org/elasticsearch/search/suggest/context/GeolocationContextMapping.java

@@ -621,8 +621,12 @@ public class GeolocationContextMapping extends ContextMapping {
                             for (int i = 0 ; i < lonFields.length ; i++) {
                                 IndexableField lonField = lonFields[i];
                                 IndexableField latField = latFields[i];
-                                spare.reset(latField.numericValue().doubleValue(), lonField.numericValue().doubleValue());
+                                assert lonField.fieldType().docValueType() == latField.fieldType().docValueType();
-                                geohashes.add(spare.geohash());
+                                // we write doc values fields differently: one field for all values, so we need to only care about indexed fields
+                                if (lonField.fieldType().docValueType() == null) {
+                                    spare.reset(latField.numericValue().doubleValue(), lonField.numericValue().doubleValue());
+                                    geohashes.add(spare.geohash());
+                                }
                             }
                         } else {
                             geohashes = mapping.defaultLocations;

src/test/java/org/elasticsearch/index/fielddata/DisabledFieldDataFormatTests.java

@@ -24,10 +24,12 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 
+@ClusterScope(randomDynamicTemplates = false)
 public class DisabledFieldDataFormatTests extends ElasticsearchIntegrationTest {
 
     @Override

src/test/java/org/elasticsearch/indices/cache/CacheTests.java

@@ -29,16 +29,16 @@ import org.elasticsearch.index.query.FilterBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
 import org.junit.Test;
 
 import static org.elasticsearch.index.query.QueryBuilders.filteredQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.test.ElasticsearchIntegrationTest.*;
 import static org.hamcrest.Matchers.*;
 
 /**
  */
-@ClusterScope(scope= Scope.SUITE, numDataNodes =1, numClientNodes = 0)
+@ClusterScope(scope= Scope.SUITE, numDataNodes =1, numClientNodes = 0, randomDynamicTemplates = false)
 public class CacheTests extends ElasticsearchIntegrationTest {
 
     @Override

src/test/java/org/elasticsearch/indices/fielddata/breaker/CircuitBreakerServiceTests.java

@@ -24,6 +24,7 @@ import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.junit.Test;
 
@@ -31,7 +32,6 @@ import java.util.Arrays;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
-import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope.TEST;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
@@ -40,7 +40,7 @@ import static org.hamcrest.CoreMatchers.containsString;
 /**
  * Integration tests for InternalCircuitBreakerService
  */
-@ClusterScope(scope = TEST)
+@ClusterScope(scope = TEST, randomDynamicTemplates = false)
 public class CircuitBreakerServiceTests extends ElasticsearchIntegrationTest {
 
     private String randomRidiculouslySmallLimit() {

src/test/java/org/elasticsearch/indices/mapping/SimpleGetMappingsTests.java

@@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -35,6 +36,7 @@ import static org.hamcrest.Matchers.notNullValue;
 /**
  *
  */
+@ClusterScope(randomDynamicTemplates = false)
 public class SimpleGetMappingsTests extends ElasticsearchIntegrationTest {
 
     @Test

src/test/java/org/elasticsearch/indices/mapping/UpdateMappingTests.java

@@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MergeMappingException;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import org.hamcrest.Matchers;
 import org.junit.Test;
 
@@ -55,6 +56,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.*;
 
+@ClusterScope(randomDynamicTemplates = false)
 public class UpdateMappingTests extends ElasticsearchIntegrationTest {
 
     @Test

src/test/java/org/elasticsearch/routing/SimpleRoutingTests.java

@@ -303,6 +303,12 @@ public class SimpleRoutingTests extends ElasticsearchIntegrationTest {
         client().admin().indices().prepareCreate("test")
                 .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1")
                         .startObject("_routing").field("required", true).field("path", "routing_field").endObject()
+                        .startObject("properties")
+                            .startObject("routing_field")
+                                .field("type", "long")
+                                .field("doc_values", false) // TODO this test fails with doc values https://github.com/elasticsearch/elasticsearch/pull/5858
+                            .endObject()
+                        .endObject()
                         .endObject().endObject())
                 .execute().actionGet();
         ensureGreen();

src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java

@@ -33,6 +33,9 @@ import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.lucene.search.function.CombineFunction;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.mapper.FieldMapper.Loading;
 import org.elasticsearch.index.mapper.MergeMappingException;
 import org.elasticsearch.index.query.*;
 import org.elasticsearch.index.search.child.ScoreType;
@@ -241,8 +244,18 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest {
 
     @Test
     public void testClearIdCacheBug() throws Exception {
+        // enforce lazy loading to make sure that p/c stats are not counted as part of field data
         assertAcked(prepareCreate("test")
-                .addMapping("parent"));
+                .addMapping("parent", XContentFactory.jsonBuilder().startObject().startObject("parent")
+                        .startObject("properties")
+                            .startObject("p_field")
+                                .field("type", "string")
+                                .startObject("fielddata")
+                                    .field(FieldDataType.FORMAT_KEY, Loading.LAZY)
+                                .endObject()
+                            .endObject()
+                        .endObject().endObject().endObject()));
+
         ensureGreen();
 
         client().prepareIndex("test", "parent", "p0").setSource("p_field", "p_value0").get();
@@ -256,7 +269,18 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest {
 
         // Now add mapping + children
         client().admin().indices().preparePutMapping("test").setType("child")
-                .setSource("_parent", "type=parent")
+                .setSource(XContentFactory.jsonBuilder().startObject().startObject("child")
+                        .startObject("_parent")
+                            .field("type", "parent")
+                        .endObject()
+                        .startObject("properties")
+                            .startObject("c_field")
+                                .field("type", "string")
+                                .startObject("fielddata")
+                                    .field(FieldDataType.FORMAT_KEY, Loading.LAZY)
+                                .endObject()
+                            .endObject()
+                        .endObject().endObject().endObject())
                 .get();
 
         // index simple data
@@ -1460,7 +1484,7 @@ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest {
 
         GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("test").get();
         Map<String, Object> mapping = getMappingsResponse.getMappings().get("test").get("child").getSourceAsMap();
-        assertThat(mapping.size(), equalTo(1));
+        assertThat(mapping.size(), greaterThanOrEqualTo(1)); // there are potentially some meta fields configured randomly
         assertThat(mapping.get("properties"), notNullValue());
 
         try {

src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java

@@ -39,6 +39,7 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRespon
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.get.GetResponse;
@@ -65,8 +66,12 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
+import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.FieldMapper.Loading;
+import org.elasticsearch.index.mapper.internal.IdFieldMapper;
 import org.elasticsearch.index.merge.policy.*;
 import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
 import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule;
@@ -294,7 +299,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
      * Creates a randomized index template. This template is used to pass in randomized settings on a
      * per index basis. Allows to enable/disable the randomization for number of shards and replicas
      */
-    private void randomIndexTemplate() {
+    private void randomIndexTemplate() throws IOException {
         // TODO move settings for random directory etc here into the index based randomized settings.
         if (immutableCluster().size() > 0) {
             ImmutableSettings.Builder randomSettingsBuilder =
@@ -306,11 +311,72 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
                     //use either 0 or 1 replica, yet a higher amount when possible, but only rarely
                     .put(SETTING_NUMBER_OF_REPLICAS, between(0, getRandom().nextInt(10) > 0 ? 1 : immutableCluster().numDataNodes() - 1));
             }
-            client().admin().indices().preparePutTemplate("random_index_template")
+            XContentBuilder mappings = null;
+            if (frequently() && randomDynamicTemplates()) {
+                mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_");
+                if (randomBoolean()) {
+                    mappings.startObject(IdFieldMapper.NAME)
+                            .field("index", randomFrom("not_analyzed", "no"))
+                        .endObject();
+                }
+                mappings.startArray("dynamic_templates")
+                        .startObject()
+                            .startObject("template-strings")
+                                .field("match_mapping_type", "string")
+                                .startObject("mapping")
+                                    .startObject("fielddata")
+                                        .field(FieldDataType.FORMAT_KEY, randomFrom("paged_bytes", "fst")) // unfortunately doc values only work on not_analyzed fields
+                                        .field(Loading.KEY, randomFrom(Loading.values()))
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                        .startObject()
+                            .startObject("template-longs")
+                                .field("match_mapping_type", "long")
+                                .startObject("mapping")
+                                    .startObject("fielddata")
+                                        .field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values"))
+                                        .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER))
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                        .startObject()
+                            .startObject("template-doubles")
+                                .field("match_mapping_type", "double")
+                                .startObject("mapping")
+                                    .startObject("fielddata")
+                                        .field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values"))
+                                        .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER))
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                        .startObject()
+                            .startObject("template-geo_points")
+                                .field("match_mapping_type", "geo_point")
+                                .startObject("mapping")
+                                    .startObject("fielddata")
+                                        .field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values"))
+                                        .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER))
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endArray();
+                mappings.endObject().endObject();
+            }
+
+            PutIndexTemplateRequestBuilder putTemplate = client().admin().indices()
+                    .preparePutTemplate("random_index_template")
                     .setTemplate("*")
                     .setOrder(0)
-                    .setSettings(randomSettingsBuilder)
+                    .setSettings(randomSettingsBuilder);
-                    .execute().actionGet();
+            if (mappings != null) {
+                putTemplate.addMapping("_default_", mappings);
+            }
+            assertAcked(putTemplate.execute().actionGet());
         }
     }
 
@@ -1057,6 +1123,11 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
          * ratio in the interval <code>[0..1]</code> is used.
          */
         double transportClientRatio() default -1;
+
+        /**
+         * Return whether or not to enable dynamic templates for the mappings.
+         */
+        boolean randomDynamicTemplates() default true;
     }
 
     private class LatchedActionListener<Response> implements ActionListener<Response> {
@@ -1148,6 +1219,11 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
         return annotation == null ? TestCluster.DEFAULT_NUM_CLIENT_NODES : annotation.numClientNodes();
     }
 
+    private boolean randomDynamicTemplates() {
+        ClusterScope annotation = getAnnotation(this.getClass());
+        return annotation == null ? true : annotation.randomDynamicTemplates();
+    }
+
     /**
      * This method is used to obtain settings for the <tt>Nth</tt> node in the cluster.
      * Nodes in this cluster are associated with an ordinal number such that nodes can

src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java

@@ -18,11 +18,9 @@
  */
 package org.elasticsearch.test;
 
-import com.carrotsearch.randomizedtesting.annotations.Listeners;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import com.carrotsearch.randomizedtesting.annotations.*;
-import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
-import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableList;
 import org.apache.lucene.store.MockDirectoryWrapper;
@@ -30,7 +28,6 @@ import org.apache.lucene.util.AbstractRandomizedTest;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.Version;
-import org.elasticsearch.test.cache.recycler.MockPageCacheRecycler;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.ESLogger;
@@ -39,11 +36,11 @@ import org.elasticsearch.common.util.concurrent.EsAbortPolicy;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.cache.recycler.MockBigArrays;
+import org.elasticsearch.test.cache.recycler.MockPageCacheRecycler;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
 import org.elasticsearch.test.store.MockDirectoryHelper;
 import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.Closeable;
@@ -293,4 +290,8 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest {
             return threadGroup.getName();
         }
     }
+
+    public static <T> T randomFrom(T... values) {
+        return RandomizedTest.randomFrom(values);
+    }
 }

src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTests.java

@@ -25,6 +25,7 @@ import com.google.common.collect.Lists;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import org.elasticsearch.test.rest.client.RestException;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
 import org.elasticsearch.test.rest.parser.RestTestSuiteParser;
@@ -51,6 +52,7 @@ import java.util.Set;
 //tests distribution disabled for now since it causes reporting problems,
 // due to the non unique suite name
 //@ReplicateOnEachVm
+@ClusterScope(randomDynamicTemplates = false)
 public class ElasticsearchRestTests extends ElasticsearchIntegrationTest {
 
     /**