From 356f0ffde2c24b38a3e5f5c1ee67d470f024d932 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Wed, 10 Jun 2015 13:27:57 +0200 Subject: [PATCH] Test: rename waitForConcreteMappingsOnAll & waitForMappingOnMaster to assertConcreteMappingsOnAll & assertMappingOnMaster Now that mapping updates are sync and done before indexing we don't really need the waiting component. Also, removed many places were they were used as safe guard against delayed mapping updates, which are now not needed. --- .../termvectors/GetTermVectorsTests.java | 2 +- .../aliases/IndexAliasesTests.java | 18 ++--- .../gateway/GatewayIndexStateTests.java | 8 +-- .../gateway/MetaDataWriteDataNodesTests.java | 10 --- .../ConcurrentDynamicTemplateTests.java | 25 ------- .../mapping/SimpleGetFieldMappingsTests.java | 1 - .../UpdateMappingIntegrationTests.java | 7 +- .../percolator/PercolatorTests.java | 10 +-- .../percolator/RecoveryPercolatorTests.java | 2 - .../SharedClusterSnapshotRestoreTests.java | 3 - .../test/ElasticsearchIntegrationTest.java | 71 ++++++++----------- 11 files changed, 40 insertions(+), 117 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java index 7eec3a65d22..12467ecef59 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java @@ -917,7 +917,7 @@ public class GetTermVectorsTests extends AbstractTermVectorsTests { assertThat(resp.isExists(), equalTo(true)); checkBrownFoxTermVector(resp.getFields(), "field1", false); // we should have created a mapping for this field - waitForMappingOnMaster("test", "type1", "non_existing"); + assertMappingOnMaster("test", "type1", "non_existing"); // and return the generated term vectors checkBrownFoxTermVector(resp.getFields(), "non_existing", false); } diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesTests.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesTests.java index 461c3272b34..63e6771b01a 100644 --- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesTests.java +++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesTests.java @@ -303,13 +303,9 @@ public class IndexAliasesTests extends ElasticsearchIntegrationTest { logger.info("--> creating indices"); createIndex("test1", "test2", "test3"); - client().admin().indices().preparePutMapping("test1", "test2", "test3") + assertAcked(client().admin().indices().preparePutMapping("test1", "test2", "test3") .setType("type1") - .setSource("name", "type=string") - .get(); - waitForConcreteMappingsOnAll("test1", "type1", "name"); - waitForConcreteMappingsOnAll("test2", "type1", "name"); - waitForConcreteMappingsOnAll("test3", "type1", "name"); + .setSource("name", "type=string")); ensureGreen(); @@ -553,14 +549,8 @@ public class IndexAliasesTests extends ElasticsearchIntegrationTest { createIndex("foobarbaz"); createIndex("bazbar"); - client().admin().indices().preparePutMapping("foobar", "test", "test123", "foobarbaz", "bazbar") - .setType("type").setSource("field", "type=string").get(); - waitForConcreteMappingsOnAll("foobar", "type", "field"); - waitForConcreteMappingsOnAll("test", "type", "field"); - waitForConcreteMappingsOnAll("test123", "type", "field"); - waitForConcreteMappingsOnAll("foobarbaz", "type", "field"); - waitForConcreteMappingsOnAll("bazbar", "type", "field"); - + assertAcked(client().admin().indices().preparePutMapping("foobar", "test", "test123", "foobarbaz", "bazbar") + .setType("type").setSource("field", "type=string")); ensureGreen(); logger.info("--> creating aliases [alias1, alias2]"); diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateTests.java index 3123edb4c5f..186e05da604 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateTests.java @@ -104,14 +104,10 @@ public class GatewayIndexStateTests extends ElasticsearchIntegrationTest { assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards)); logger.info("--> indexing a simple document"); - client().prepareIndex("test", "type1", "1").setSource("field1", "value1").execute().actionGet(); - - // we need this until we have https://github.com/elasticsearch/elasticsearch/issues/8688 - // the test rarely fails else because the master does not apply the new mapping quick enough and it is lost - waitForConcreteMappingsOnAll("test", "type1", "field1"); + client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get(); logger.info("--> closing test index..."); - client().admin().indices().prepareClose("test").execute().actionGet(); + client().admin().indices().prepareClose("test").get(); stateResponse = client().admin().cluster().prepareState().execute().actionGet(); assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.CLOSE)); diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java index 71143159cc4..30608cf1a8d 100644 --- a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java @@ -55,7 +55,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { String redNode = startDataNode("red"); assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.number_of_replicas", 0))); index("test", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); - waitForConcreteMappingsOnAll("test", "doc", "text"); ensureGreen("test"); assertIndexInMetaState(redNode, "test"); assertIndexInMetaState(masterNodeName, "test"); @@ -63,8 +62,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { ((InternalTestCluster) cluster()).stopCurrentMasterNode(); String newMasterNode = startMasterNode(); ensureGreen("test"); - // wait for mapping also on master becasue then we can be sure the state was written - waitForConcreteMappingsOnAll("test", "doc", "text"); // check for meta data assertIndexInMetaState(redNode, "test"); assertIndexInMetaState(newMasterNode, "test"); @@ -85,8 +82,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red"))); index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); ensureGreen(); - waitForConcreteMappingsOnAll("blue_index", "doc", "text"); - waitForConcreteMappingsOnAll("red_index", "doc", "text"); assertIndexNotInMetaState(blueNode, "red_index"); assertIndexNotInMetaState(redNode, "blue_index"); assertIndexInMetaState(blueNode, "blue_index"); @@ -151,8 +146,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { assertIndexInMetaState(blueNode, "red_index"); assertIndexInMetaState(masterNode, "red_index"); assertIndexInMetaState(masterNode, "blue_index"); - waitForConcreteMappingsOnAll("blue_index", "doc", "text"); - waitForConcreteMappingsOnAll("red_index", "doc", "text"); //at this point the blue_index is on red node and the red_index on blue node // now, when we start red and master node again but without data folder, the red index should be gone but the blue index should initialize fine @@ -188,7 +181,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { assertIndexInMetaState(redNode, "red_index"); assertIndexInMetaState(masterNode, "red_index"); - waitForConcreteMappingsOnAll("red_index", "doc", "text"); client().admin().indices().prepareClose("red_index").get(); // close the index ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get(); @@ -252,8 +244,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { assertIndexInMetaState(redNode, "red_index"); assertIndexInMetaState(masterNode, "red_index"); - waitForConcreteMappingsOnAll("red_index", "doc", "text"); - logger.info("--> close red_index"); client().admin().indices().prepareClose("red_index").get(); // close the index diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateTests.java b/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateTests.java index 28bcde323d2..673bd140d01 100644 --- a/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateTests.java +++ b/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateTests.java @@ -89,29 +89,4 @@ public class ConcurrentDynamicTemplateTests extends ElasticsearchIntegrationTest } } - - @Test - public void testDynamicMappingIntroductionPropagatesToAll() throws Exception { - int numDocs = randomIntBetween(100, 1000); - int numberOfFields = scaledRandomIntBetween(1, 50); - Set fieldsIdx = Sets.newHashSet(); - IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; - - createIndex("idx"); - ensureGreen("idx"); - for (int i = 0; i < numDocs; ++i) { - int fieldIdx = i % numberOfFields; - fieldsIdx.add(fieldIdx); - builders[i] = client().prepareIndex("idx", "type").setSource(jsonBuilder() - .startObject() - .field("str_value_" + fieldIdx, "s" + i) - .field("l_value_" + fieldIdx, i) - .field("d_value_" + fieldIdx, (double)i + 0.01) - .endObject()); - } - indexRandom(false, builders); - for (Integer fieldIdx : fieldsIdx) { - waitForConcreteMappingsOnAll("idx", "type", "str_value_" + fieldIdx, "l_value_" + fieldIdx, "d_value_" + fieldIdx); - } - } } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsTests.java b/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsTests.java index 119157bcfc1..0e160189bfb 100644 --- a/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsTests.java +++ b/core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsTests.java @@ -133,7 +133,6 @@ public class SimpleGetFieldMappingsTests extends ElasticsearchIntegrationTest { client().prepareIndex("test", "type", "1").setSource("num", 1).get(); ensureYellow(); - waitForConcreteMappingsOnAll("test", "type", "num"); // for num, we need to wait... GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings().setFields("num", "field1", "obj.subfield").includeDefaults(true).get(); diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationTests.java b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationTests.java index 36026e95b52..e9aef353582 100644 --- a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationTests.java +++ b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationTests.java @@ -39,10 +39,7 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.hamcrest.Matchers; import org.junit.Test; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; @@ -86,7 +83,7 @@ public class UpdateMappingIntegrationTests extends ElasticsearchIntegrationTest for (int rec = 0; rec < recCount; rec++) { String type = "type" + (rec % numberOfTypes); String fieldName = "field_" + type + "_" + rec; - waitForConcreteMappingsOnAll("test", type, fieldName); + assertConcreteMappingsOnAll("test", type, fieldName); } } diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorTests.java b/core/src/test/java/org/elasticsearch/percolator/PercolatorTests.java index 07ef51f32c1..1168642a273 100644 --- a/core/src/test/java/org/elasticsearch/percolator/PercolatorTests.java +++ b/core/src/test/java/org/elasticsearch/percolator/PercolatorTests.java @@ -109,7 +109,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest { logger.info("--> Add dummy doc"); client().prepareIndex("test", "type", "1").setSource("field1", "value").execute().actionGet(); - waitForConcreteMappingsOnAll("test", "type", "field1"); logger.info("--> register a queries"); client().prepareIndex("test", PercolatorService.TYPE_NAME, "1") @@ -198,7 +197,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest { .execute().actionGet(); assertMatchCount(response, 0l); assertThat(response.getMatches(), emptyArray()); - waitForConcreteMappingsOnAll("test", "type1", "field1", "field2"); // add first query... client().prepareIndex("test", PercolatorService.TYPE_NAME, "test1") @@ -278,12 +276,11 @@ public class PercolatorTests extends ElasticsearchIntegrationTest { } @Test - public void percolateOnRecreatedIndex() throws Exception { + public void storePeroclateQueriesOnRecreatedIndex() throws Exception { createIndex("test"); ensureGreen(); client().prepareIndex("my-queries-index", "test", "1").setSource("field1", "value1").execute().actionGet(); - waitForConcreteMappingsOnAll("my-queries-index", "test", "field1"); logger.info("--> register a query"); client().prepareIndex("my-queries-index", PercolatorService.TYPE_NAME, "kuku1") .setSource(jsonBuilder().startObject() @@ -298,7 +295,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest { ensureGreen(); client().prepareIndex("my-queries-index", "test", "1").setSource("field1", "value1").execute().actionGet(); - waitForConcreteMappingsOnAll("my-queries-index", "test", "field1"); logger.info("--> register a query"); client().prepareIndex("my-queries-index", PercolatorService.TYPE_NAME, "kuku2") .setSource(jsonBuilder().startObject() @@ -995,7 +991,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest { logger.info("--> Add dummy doc"); client().prepareIndex("test", "type", "1").setSource("field1", "value").execute().actionGet(); - waitForConcreteMappingsOnAll("test", "type", "field1"); logger.info("--> register a queries"); client().prepareIndex("test", PercolatorService.TYPE_NAME, "1") @@ -1724,7 +1719,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest { assertMatchCount(percolateResponse, 0l); assertThat(percolateResponse.getMatches(), arrayWithSize(0)); - waitForConcreteMappingsOnAll("idx", "type", "custom.color"); // The previous percolate request introduced the custom.color field, so now we register the query again // and the field name `color` will be resolved to `custom.color` field in mapping via smart field mapping resolving. @@ -1762,7 +1756,7 @@ public class PercolatorTests extends ElasticsearchIntegrationTest { assertMatchCount(response, 0l); assertThat(response.getMatches(), arrayWithSize(0)); - waitForMappingOnMaster("test", "type1"); + assertMappingOnMaster("test", "type1"); GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings("test").get(); assertThat(mappingsResponse.getMappings().get("test"), notNullValue()); diff --git a/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorTests.java b/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorTests.java index 7679ff6724b..131f72853d6 100644 --- a/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorTests.java +++ b/core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorTests.java @@ -186,7 +186,6 @@ public class RecoveryPercolatorTests extends ElasticsearchIntegrationTest { logger.info("--> Add dummy docs"); client().prepareIndex("test", "type1", "1").setSource("field1", 0).get(); client().prepareIndex("test", "type2", "1").setSource("field1", "0").get(); - waitForConcreteMappingsOnAll("test", "type1", "field1"); logger.info("--> register a queries"); for (int i = 1; i <= 100; i++) { @@ -199,7 +198,6 @@ public class RecoveryPercolatorTests extends ElasticsearchIntegrationTest { .endObject()) .get(); } - waitForConcreteMappingsOnAll("test", PercolatorService.TYPE_NAME); logger.info("--> Percolate doc with field1=95"); PercolateResponse response = client().preparePercolate() diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java index b9fb2bf1fe4..243c06c9d76 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java @@ -196,9 +196,6 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests { String docId = Integer.toString(randomInt()); index(indexName, typeName, docId, "value", expectedValue); - // TODO: Remove after dynamic mapping flushing is implemented - waitForConcreteMappingsOnAll(indexName, typeName, "value"); - logger.info("--> creating repository"); assertAcked(client.admin().cluster().preparePutRepository(repoName) .setType("fs").setSettings(Settings.settingsBuilder() diff --git a/core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java b/core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java index 621d4fc4d01..28965cf217c 100644 --- a/core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java +++ b/core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java @@ -97,7 +97,6 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappedFieldType.Loading; import org.elasticsearch.index.mapper.internal.SizeFieldMapper; @@ -866,56 +865,44 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase * Waits till a (pattern) field name mappings concretely exists on all nodes. Note, this waits for the current * started shards and checks for concrete mappings. */ - public void waitForConcreteMappingsOnAll(final String index, final String type, final String... fieldNames) throws Exception { - assertBusy(new Runnable() { - @Override - public void run() { - Set nodes = internalCluster().nodesInclude(index); - assertThat(nodes, Matchers.not(Matchers.emptyIterable())); - for (String node : nodes) { - IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); - IndexService indexService = indicesService.indexService(index); - assertThat("index service doesn't exists on " + node, indexService, notNullValue()); - DocumentMapper documentMapper = indexService.mapperService().documentMapper(type); - assertThat("document mapper doesn't exists on " + node, documentMapper, notNullValue()); - for (String fieldName : fieldNames) { - Collection matches = documentMapper.mappers().simpleMatchToFullName(fieldName); - assertThat("field " + fieldName + " doesn't exists on " + node, matches, Matchers.not(emptyIterable())); - } - } + public void assertConcreteMappingsOnAll(final String index, final String type, final String... fieldNames) throws Exception { + Set nodes = internalCluster().nodesInclude(index); + assertThat(nodes, Matchers.not(Matchers.emptyIterable())); + for (String node : nodes) { + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + IndexService indexService = indicesService.indexService(index); + assertThat("index service doesn't exists on " + node, indexService, notNullValue()); + DocumentMapper documentMapper = indexService.mapperService().documentMapper(type); + assertThat("document mapper doesn't exists on " + node, documentMapper, notNullValue()); + for (String fieldName : fieldNames) { + Collection matches = documentMapper.mappers().simpleMatchToFullName(fieldName); + assertThat("field " + fieldName + " doesn't exists on " + node, matches, Matchers.not(emptyIterable())); } - }); - waitForMappingOnMaster(index, type, fieldNames); + } + assertMappingOnMaster(index, type, fieldNames); } /** * Waits for the given mapping type to exists on the master node. */ - public void waitForMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception { - assertBusy(new Callable() { - @Override - public Object call() throws Exception { - GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get(); - ImmutableOpenMap mappings = response.getMappings().get(index); - assertThat(mappings, notNullValue()); - MappingMetaData mappingMetaData = mappings.get(type); - assertThat(mappingMetaData, notNullValue()); + public void assertMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception { + GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get(); + ImmutableOpenMap mappings = response.getMappings().get(index); + assertThat(mappings, notNullValue()); + MappingMetaData mappingMetaData = mappings.get(type); + assertThat(mappingMetaData, notNullValue()); - Map mappingSource = mappingMetaData.getSourceAsMap(); - assertFalse(mappingSource.isEmpty()); - assertTrue(mappingSource.containsKey("properties")); + Map mappingSource = mappingMetaData.getSourceAsMap(); + assertFalse(mappingSource.isEmpty()); + assertTrue(mappingSource.containsKey("properties")); - for (String fieldName : fieldNames) { - Map mappingProperties = (Map) mappingSource.get("properties"); - if (fieldName.indexOf('.') != -1) { - fieldName = fieldName.replace(".", ".properties."); - } - assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(), XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue()); - } - - return null; + for (String fieldName : fieldNames) { + Map mappingProperties = (Map) mappingSource.get("properties"); + if (fieldName.indexOf('.') != -1) { + fieldName = fieldName.replace(".", ".properties."); } - }); + assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(), XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue()); + } } /**