Merge pull request #11576 from bleskes/wait_on_mapping_remove

Test: rename waitForConcreteMappingsOnAll & waitForMappingOnMaster to assertConcreteMappingsOnAll & assertMappingOnMaster

core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java

@@ -917,7 +917,7 @@ public class GetTermVectorsTests extends AbstractTermVectorsTests {
             assertThat(resp.isExists(), equalTo(true));
             checkBrownFoxTermVector(resp.getFields(), "field1", false);
             // we should have created a mapping for this field
-            waitForMappingOnMaster("test", "type1", "non_existing");
+            assertMappingOnMaster("test", "type1", "non_existing");
             // and return the generated term vectors
             checkBrownFoxTermVector(resp.getFields(), "non_existing", false);
         }

core/src/test/java/org/elasticsearch/aliases/IndexAliasesTests.java

@@ -303,13 +303,9 @@ public class IndexAliasesTests extends ElasticsearchIntegrationTest {
         logger.info("--> creating indices");
         createIndex("test1", "test2", "test3");
 
-        client().admin().indices().preparePutMapping("test1", "test2", "test3")
+        assertAcked(client().admin().indices().preparePutMapping("test1", "test2", "test3")
                 .setType("type1")
-                .setSource("name", "type=string")
+                .setSource("name", "type=string"));
-                .get();
-        waitForConcreteMappingsOnAll("test1", "type1", "name");
-        waitForConcreteMappingsOnAll("test2", "type1", "name");
-        waitForConcreteMappingsOnAll("test3", "type1", "name");
 
         ensureGreen();
 
@@ -553,14 +549,8 @@ public class IndexAliasesTests extends ElasticsearchIntegrationTest {
         createIndex("foobarbaz");
         createIndex("bazbar");
 
-        client().admin().indices().preparePutMapping("foobar", "test", "test123", "foobarbaz", "bazbar")
+        assertAcked(client().admin().indices().preparePutMapping("foobar", "test", "test123", "foobarbaz", "bazbar")
-                .setType("type").setSource("field", "type=string").get();
+                .setType("type").setSource("field", "type=string"));
-        waitForConcreteMappingsOnAll("foobar", "type", "field");
-        waitForConcreteMappingsOnAll("test", "type", "field");
-        waitForConcreteMappingsOnAll("test123", "type", "field");
-        waitForConcreteMappingsOnAll("foobarbaz", "type", "field");
-        waitForConcreteMappingsOnAll("bazbar", "type", "field");
-
         ensureGreen();
 
         logger.info("--> creating aliases [alias1, alias2]");

core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateTests.java

@@ -104,14 +104,10 @@ public class GatewayIndexStateTests extends ElasticsearchIntegrationTest {
         assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards));
 
         logger.info("--> indexing a simple document");
-        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get();
-
-        // we need this until we have  https://github.com/elasticsearch/elasticsearch/issues/8688
-        // the test rarely fails else because the master does not apply the new mapping quick enough and it is lost
-        waitForConcreteMappingsOnAll("test", "type1", "field1");
 
         logger.info("--> closing test index...");
-        client().admin().indices().prepareClose("test").execute().actionGet();
+        client().admin().indices().prepareClose("test").get();
 
         stateResponse = client().admin().cluster().prepareState().execute().actionGet();
         assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.CLOSE));

core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java

@@ -55,7 +55,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
         String redNode = startDataNode("red");
         assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.number_of_replicas", 0)));
         index("test", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
-        waitForConcreteMappingsOnAll("test", "doc", "text");
         ensureGreen("test");
         assertIndexInMetaState(redNode, "test");
         assertIndexInMetaState(masterNodeName, "test");
@@ -63,8 +62,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
         ((InternalTestCluster) cluster()).stopCurrentMasterNode();
         String newMasterNode = startMasterNode();
         ensureGreen("test");
-        // wait for mapping also on master becasue then we can be sure the state was written
-        waitForConcreteMappingsOnAll("test", "doc", "text");
         // check for meta data
         assertIndexInMetaState(redNode, "test");
         assertIndexInMetaState(newMasterNode, "test");
@@ -85,8 +82,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
         assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")));
         index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
         ensureGreen();
-        waitForConcreteMappingsOnAll("blue_index", "doc", "text");
-        waitForConcreteMappingsOnAll("red_index", "doc", "text");
         assertIndexNotInMetaState(blueNode, "red_index");
         assertIndexNotInMetaState(redNode, "blue_index");
         assertIndexInMetaState(blueNode, "blue_index");
@@ -151,8 +146,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
         assertIndexInMetaState(blueNode, "red_index");
         assertIndexInMetaState(masterNode, "red_index");
         assertIndexInMetaState(masterNode, "blue_index");
-        waitForConcreteMappingsOnAll("blue_index", "doc", "text");
-        waitForConcreteMappingsOnAll("red_index", "doc", "text");
 
         //at this point the blue_index is on red node and the red_index on blue node
         // now, when we start red and master node again but without data folder, the red index should be gone but the blue index should initialize fine
@@ -188,7 +181,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
         assertIndexInMetaState(redNode, "red_index");
         assertIndexInMetaState(masterNode, "red_index");
 
-        waitForConcreteMappingsOnAll("red_index", "doc", "text");
         client().admin().indices().prepareClose("red_index").get();
         // close the index
         ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
@@ -252,8 +244,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
         assertIndexInMetaState(redNode, "red_index");
         assertIndexInMetaState(masterNode, "red_index");
 
-        waitForConcreteMappingsOnAll("red_index", "doc", "text");
-
         logger.info("--> close red_index");
         client().admin().indices().prepareClose("red_index").get();
         // close the index

core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateTests.java

@@ -89,29 +89,4 @@ public class ConcurrentDynamicTemplateTests extends ElasticsearchIntegrationTest
 
         }
     }
-
-    @Test
-    public void testDynamicMappingIntroductionPropagatesToAll() throws Exception {
-        int numDocs = randomIntBetween(100, 1000);
-        int numberOfFields = scaledRandomIntBetween(1, 50);
-        Set<Integer> fieldsIdx = Sets.newHashSet();
-        IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
-
-        createIndex("idx");
-        ensureGreen("idx");
-        for (int i = 0; i < numDocs; ++i) {
-            int fieldIdx = i % numberOfFields;
-            fieldsIdx.add(fieldIdx);
-            builders[i] = client().prepareIndex("idx", "type").setSource(jsonBuilder()
-                    .startObject()
-                    .field("str_value_" + fieldIdx, "s" + i)
-                    .field("l_value_" + fieldIdx, i)
-                    .field("d_value_" + fieldIdx, (double)i + 0.01)
-                    .endObject());
-        }
-        indexRandom(false, builders);
-        for (Integer fieldIdx : fieldsIdx) {
-            waitForConcreteMappingsOnAll("idx", "type", "str_value_" + fieldIdx, "l_value_" + fieldIdx, "d_value_" + fieldIdx);
-        }
-    }
 }

core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsTests.java

@@ -133,7 +133,6 @@ public class SimpleGetFieldMappingsTests extends ElasticsearchIntegrationTest {
 
         client().prepareIndex("test", "type", "1").setSource("num", 1).get();
         ensureYellow();
-        waitForConcreteMappingsOnAll("test", "type", "num"); // for num, we need to wait...
 
         GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings().setFields("num", "field1", "obj.subfield").includeDefaults(true).get();
 

core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationTests.java

@@ -39,10 +39,7 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
 import org.hamcrest.Matchers;
 import org.junit.Test;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -86,7 +83,7 @@ public class UpdateMappingIntegrationTests extends ElasticsearchIntegrationTest
         for (int rec = 0; rec < recCount; rec++) {
             String type = "type" + (rec % numberOfTypes);
             String fieldName = "field_" + type + "_" + rec;
-            waitForConcreteMappingsOnAll("test", type, fieldName);
+            assertConcreteMappingsOnAll("test", type, fieldName);
         }
     }
 

core/src/test/java/org/elasticsearch/percolator/PercolatorTests.java

@@ -109,7 +109,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
 
         logger.info("--> Add dummy doc");
         client().prepareIndex("test", "type", "1").setSource("field1", "value").execute().actionGet();
-        waitForConcreteMappingsOnAll("test", "type", "field1");
 
         logger.info("--> register a queries");
         client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
@@ -198,7 +197,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
                 .execute().actionGet();
         assertMatchCount(response, 0l);
         assertThat(response.getMatches(), emptyArray());
-        waitForConcreteMappingsOnAll("test", "type1", "field1", "field2");
 
         // add first query...
         client().prepareIndex("test", PercolatorService.TYPE_NAME, "test1")
@@ -278,12 +276,11 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
     }
 
     @Test
-    public void percolateOnRecreatedIndex() throws Exception {
+    public void storePeroclateQueriesOnRecreatedIndex() throws Exception {
         createIndex("test");
         ensureGreen();
 
         client().prepareIndex("my-queries-index", "test", "1").setSource("field1", "value1").execute().actionGet();
-        waitForConcreteMappingsOnAll("my-queries-index", "test", "field1");
         logger.info("--> register a query");
         client().prepareIndex("my-queries-index", PercolatorService.TYPE_NAME, "kuku1")
                 .setSource(jsonBuilder().startObject()
@@ -298,7 +295,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
         ensureGreen();
 
         client().prepareIndex("my-queries-index", "test", "1").setSource("field1", "value1").execute().actionGet();
-        waitForConcreteMappingsOnAll("my-queries-index", "test", "field1");
         logger.info("--> register a query");
         client().prepareIndex("my-queries-index", PercolatorService.TYPE_NAME, "kuku2")
                 .setSource(jsonBuilder().startObject()
@@ -995,7 +991,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
 
         logger.info("--> Add dummy doc");
         client().prepareIndex("test", "type", "1").setSource("field1", "value").execute().actionGet();
-        waitForConcreteMappingsOnAll("test", "type", "field1");
 
         logger.info("--> register a queries");
         client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
@@ -1724,7 +1719,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
 
         assertMatchCount(percolateResponse, 0l);
         assertThat(percolateResponse.getMatches(), arrayWithSize(0));
-        waitForConcreteMappingsOnAll("idx", "type", "custom.color");
 
         // The previous percolate request introduced the custom.color field, so now we register the query again
         // and the field name `color` will be resolved to `custom.color` field in mapping via smart field mapping resolving.
@@ -1762,7 +1756,7 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
         assertMatchCount(response, 0l);
         assertThat(response.getMatches(), arrayWithSize(0));
 
-        waitForMappingOnMaster("test", "type1");
+        assertMappingOnMaster("test", "type1");
 
         GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings("test").get();
         assertThat(mappingsResponse.getMappings().get("test"), notNullValue());

core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorTests.java

@@ -186,7 +186,6 @@ public class RecoveryPercolatorTests extends ElasticsearchIntegrationTest {
         logger.info("--> Add dummy docs");
         client().prepareIndex("test", "type1", "1").setSource("field1", 0).get();
         client().prepareIndex("test", "type2", "1").setSource("field1", "0").get();
-        waitForConcreteMappingsOnAll("test", "type1", "field1");
 
         logger.info("--> register a queries");
         for (int i = 1; i <= 100; i++) {
@@ -199,7 +198,6 @@ public class RecoveryPercolatorTests extends ElasticsearchIntegrationTest {
                             .endObject())
                     .get();
         }
-        waitForConcreteMappingsOnAll("test", PercolatorService.TYPE_NAME);
 
         logger.info("--> Percolate doc with field1=95");
         PercolateResponse response = client().preparePercolate()

core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java

@@ -196,9 +196,6 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
         String docId = Integer.toString(randomInt());
         index(indexName, typeName, docId, "value", expectedValue);
 
-        // TODO: Remove after dynamic mapping flushing is implemented
-        waitForConcreteMappingsOnAll(indexName, typeName, "value");
-
         logger.info("-->  creating repository");
         assertAcked(client.admin().cluster().preparePutRepository(repoName)
                 .setType("fs").setSettings(Settings.settingsBuilder()

core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java

@@ -97,7 +97,6 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Loading;
 import org.elasticsearch.index.mapper.internal.SizeFieldMapper;
@@ -866,10 +865,7 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
      * Waits till a (pattern) field name mappings concretely exists on all nodes. Note, this waits for the current
      * started shards and checks for concrete mappings.
      */
-    public void waitForConcreteMappingsOnAll(final String index, final String type, final String... fieldNames) throws Exception {
+    public void assertConcreteMappingsOnAll(final String index, final String type, final String... fieldNames) throws Exception {
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
         Set<String> nodes = internalCluster().nodesInclude(index);
         assertThat(nodes, Matchers.not(Matchers.emptyIterable()));
         for (String node : nodes) {
@@ -883,18 +879,13 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
                 assertThat("field " + fieldName + " doesn't exists on " + node, matches, Matchers.not(emptyIterable()));
             }
         }
-            }
+        assertMappingOnMaster(index, type, fieldNames);
-        });
-        waitForMappingOnMaster(index, type, fieldNames);
     }
 
     /**
      * Waits for the given mapping type to exists on the master node.
      */
-    public void waitForMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception {
+    public void assertMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception {
-        assertBusy(new Callable() {
-            @Override
-            public Object call() throws Exception {
         GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get();
         ImmutableOpenMap<String, MappingMetaData> mappings = response.getMappings().get(index);
         assertThat(mappings, notNullValue());
@@ -912,10 +903,6 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
             }
             assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(), XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue());
         }
-
-                return null;
-            }
-        });
     }
 
     /**