honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11576 from bleskes/wait_on_mapping_remove 

Test: rename waitForConcreteMappingsOnAll & waitForMappingOnMaster to assertConcreteMappingsOnAll & assertMappingOnMaster
This commit is contained in:
Boaz Leskes 2015-06-10 14:37:59 +02:00
parent 199c9002e5 356f0ffde2
commit 5951f2580d
11 changed files with 40 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsTests.java
View File

@ -917,7 +917,7 @@ public class GetTermVectorsTests extends AbstractTermVectorsTests {
assertThat(resp.isExists(), equalTo(true));
checkBrownFoxTermVector(resp.getFields(), "field1", false);
// we should have created a mapping for this field
waitForMappingOnMaster("test", "type1", "non_existing");
assertMappingOnMaster("test", "type1", "non_existing");
// and return the generated term vectors
checkBrownFoxTermVector(resp.getFields(), "non_existing", false);
}

18
core/src/test/java/org/elasticsearch/aliases/IndexAliasesTests.java
View File

@ -303,13 +303,9 @@ public class IndexAliasesTests extends ElasticsearchIntegrationTest {
logger.info("--> creating indices");
createIndex("test1", "test2", "test3");
client().admin().indices().preparePutMapping("test1", "test2", "test3")
assertAcked(client().admin().indices().preparePutMapping("test1", "test2", "test3")
.setType("type1")
.setSource("name", "type=string")
.get();
waitForConcreteMappingsOnAll("test1", "type1", "name");
waitForConcreteMappingsOnAll("test2", "type1", "name");
waitForConcreteMappingsOnAll("test3", "type1", "name");
.setSource("name", "type=string"));
ensureGreen();
@ -553,14 +549,8 @@ public class IndexAliasesTests extends ElasticsearchIntegrationTest {
createIndex("foobarbaz");
createIndex("bazbar");
client().admin().indices().preparePutMapping("foobar", "test", "test123", "foobarbaz", "bazbar")
.setType("type").setSource("field", "type=string").get();
waitForConcreteMappingsOnAll("foobar", "type", "field");
waitForConcreteMappingsOnAll("test", "type", "field");
waitForConcreteMappingsOnAll("test123", "type", "field");
waitForConcreteMappingsOnAll("foobarbaz", "type", "field");
waitForConcreteMappingsOnAll("bazbar", "type", "field");
assertAcked(client().admin().indices().preparePutMapping("foobar", "test", "test123", "foobarbaz", "bazbar")
.setType("type").setSource("field", "type=string"));
ensureGreen();
logger.info("--> creating aliases [alias1, alias2]");

8
core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateTests.java
View File

@ -104,14 +104,10 @@ public class GatewayIndexStateTests extends ElasticsearchIntegrationTest {
assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(test.totalNumShards));
logger.info("--> indexing a simple document");
client().prepareIndex("test", "type1", "1").setSource("field1", "value1").execute().actionGet();
// we need this until we have https://github.com/elasticsearch/elasticsearch/issues/8688
// the test rarely fails else because the master does not apply the new mapping quick enough and it is lost
waitForConcreteMappingsOnAll("test", "type1", "field1");
client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get();
logger.info("--> closing test index...");
client().admin().indices().prepareClose("test").execute().actionGet();
client().admin().indices().prepareClose("test").get();
stateResponse = client().admin().cluster().prepareState().execute().actionGet();
assertThat(stateResponse.getState().metaData().index("test").state(), equalTo(IndexMetaData.State.CLOSE));

10
core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java
View File

@ -55,7 +55,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
String redNode = startDataNode("red");
assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.number_of_replicas", 0)));
index("test", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
waitForConcreteMappingsOnAll("test", "doc", "text");
ensureGreen("test");
assertIndexInMetaState(redNode, "test");
assertIndexInMetaState(masterNodeName, "test");
@ -63,8 +62,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
((InternalTestCluster) cluster()).stopCurrentMasterNode();
String newMasterNode = startMasterNode();
ensureGreen("test");
// wait for mapping also on master becasue then we can be sure the state was written
waitForConcreteMappingsOnAll("test", "doc", "text");
// check for meta data
assertIndexInMetaState(redNode, "test");
assertIndexInMetaState(newMasterNode, "test");
@ -85,8 +82,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")));
index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
ensureGreen();
waitForConcreteMappingsOnAll("blue_index", "doc", "text");
waitForConcreteMappingsOnAll("red_index", "doc", "text");
assertIndexNotInMetaState(blueNode, "red_index");
assertIndexNotInMetaState(redNode, "blue_index");
assertIndexInMetaState(blueNode, "blue_index");
@ -151,8 +146,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
assertIndexInMetaState(blueNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
assertIndexInMetaState(masterNode, "blue_index");
waitForConcreteMappingsOnAll("blue_index", "doc", "text");
waitForConcreteMappingsOnAll("red_index", "doc", "text");
//at this point the blue_index is on red node and the red_index on blue node
// now, when we start red and master node again but without data folder, the red index should be gone but the blue index should initialize fine
@ -188,7 +181,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
waitForConcreteMappingsOnAll("red_index", "doc", "text");
client().admin().indices().prepareClose("red_index").get();
// close the index
ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
@ -252,8 +244,6 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest {
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
waitForConcreteMappingsOnAll("red_index", "doc", "text");
logger.info("--> close red_index");
client().admin().indices().prepareClose("red_index").get();
// close the index

25
core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateTests.java
View File

@ -89,29 +89,4 @@ public class ConcurrentDynamicTemplateTests extends ElasticsearchIntegrationTest
}
}
@Test
public void testDynamicMappingIntroductionPropagatesToAll() throws Exception {
int numDocs = randomIntBetween(100, 1000);
int numberOfFields = scaledRandomIntBetween(1, 50);
Set<Integer> fieldsIdx = Sets.newHashSet();
IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
createIndex("idx");
ensureGreen("idx");
for (int i = 0; i < numDocs; ++i) {
int fieldIdx = i % numberOfFields;
fieldsIdx.add(fieldIdx);
builders[i] = client().prepareIndex("idx", "type").setSource(jsonBuilder()
.startObject()
.field("str_value_" + fieldIdx, "s" + i)
.field("l_value_" + fieldIdx, i)
.field("d_value_" + fieldIdx, (double)i + 0.01)
.endObject());
}
indexRandom(false, builders);
for (Integer fieldIdx : fieldsIdx) {
waitForConcreteMappingsOnAll("idx", "type", "str_value_" + fieldIdx, "l_value_" + fieldIdx, "d_value_" + fieldIdx);
}
}
}

1
core/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsTests.java
View File

@ -133,7 +133,6 @@ public class SimpleGetFieldMappingsTests extends ElasticsearchIntegrationTest {
client().prepareIndex("test", "type", "1").setSource("num", 1).get();
ensureYellow();
waitForConcreteMappingsOnAll("test", "type", "num"); // for num, we need to wait...
GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings().setFields("num", "field1", "obj.subfield").includeDefaults(true).get();

7
core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationTests.java
View File

@ -39,10 +39,7 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.hamcrest.Matchers;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
@ -86,7 +83,7 @@ public class UpdateMappingIntegrationTests extends ElasticsearchIntegrationTest
for (int rec = 0; rec < recCount; rec++) {
String type = "type" + (rec % numberOfTypes);
String fieldName = "field_" + type + "_" + rec;
waitForConcreteMappingsOnAll("test", type, fieldName);
assertConcreteMappingsOnAll("test", type, fieldName);
}
}

10
core/src/test/java/org/elasticsearch/percolator/PercolatorTests.java
View File

@ -109,7 +109,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
logger.info("--> Add dummy doc");
client().prepareIndex("test", "type", "1").setSource("field1", "value").execute().actionGet();
waitForConcreteMappingsOnAll("test", "type", "field1");
logger.info("--> register a queries");
client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
@ -198,7 +197,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
.execute().actionGet();
assertMatchCount(response, 0l);
assertThat(response.getMatches(), emptyArray());
waitForConcreteMappingsOnAll("test", "type1", "field1", "field2");
// add first query...
client().prepareIndex("test", PercolatorService.TYPE_NAME, "test1")
@ -278,12 +276,11 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
}
@Test
public void percolateOnRecreatedIndex() throws Exception {
public void storePeroclateQueriesOnRecreatedIndex() throws Exception {
createIndex("test");
ensureGreen();
client().prepareIndex("my-queries-index", "test", "1").setSource("field1", "value1").execute().actionGet();
waitForConcreteMappingsOnAll("my-queries-index", "test", "field1");
logger.info("--> register a query");
client().prepareIndex("my-queries-index", PercolatorService.TYPE_NAME, "kuku1")
.setSource(jsonBuilder().startObject()
@ -298,7 +295,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
ensureGreen();
client().prepareIndex("my-queries-index", "test", "1").setSource("field1", "value1").execute().actionGet();
waitForConcreteMappingsOnAll("my-queries-index", "test", "field1");
logger.info("--> register a query");
client().prepareIndex("my-queries-index", PercolatorService.TYPE_NAME, "kuku2")
.setSource(jsonBuilder().startObject()
@ -995,7 +991,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
logger.info("--> Add dummy doc");
client().prepareIndex("test", "type", "1").setSource("field1", "value").execute().actionGet();
waitForConcreteMappingsOnAll("test", "type", "field1");
logger.info("--> register a queries");
client().prepareIndex("test", PercolatorService.TYPE_NAME, "1")
@ -1724,7 +1719,6 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
assertMatchCount(percolateResponse, 0l);
assertThat(percolateResponse.getMatches(), arrayWithSize(0));
waitForConcreteMappingsOnAll("idx", "type", "custom.color");
// The previous percolate request introduced the custom.color field, so now we register the query again
// and the field name `color` will be resolved to `custom.color` field in mapping via smart field mapping resolving.
@ -1762,7 +1756,7 @@ public class PercolatorTests extends ElasticsearchIntegrationTest {
assertMatchCount(response, 0l);
assertThat(response.getMatches(), arrayWithSize(0));
waitForMappingOnMaster("test", "type1");
assertMappingOnMaster("test", "type1");
GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings("test").get();
assertThat(mappingsResponse.getMappings().get("test"), notNullValue());

2
core/src/test/java/org/elasticsearch/percolator/RecoveryPercolatorTests.java
View File

@ -186,7 +186,6 @@ public class RecoveryPercolatorTests extends ElasticsearchIntegrationTest {
logger.info("--> Add dummy docs");
client().prepareIndex("test", "type1", "1").setSource("field1", 0).get();
client().prepareIndex("test", "type2", "1").setSource("field1", "0").get();
waitForConcreteMappingsOnAll("test", "type1", "field1");
logger.info("--> register a queries");
for (int i = 1; i <= 100; i++) {
@ -199,7 +198,6 @@ public class RecoveryPercolatorTests extends ElasticsearchIntegrationTest {
.endObject())
.get();
}
waitForConcreteMappingsOnAll("test", PercolatorService.TYPE_NAME);
logger.info("--> Percolate doc with field1=95");
PercolateResponse response = client().preparePercolate()

3
core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java
View File

@ -196,9 +196,6 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
String docId = Integer.toString(randomInt());
index(indexName, typeName, docId, "value", expectedValue);
// TODO: Remove after dynamic mapping flushing is implemented
waitForConcreteMappingsOnAll(indexName, typeName, "value");
logger.info("--> creating repository");
assertAcked(client.admin().cluster().preparePutRepository(repoName)
.setType("fs").setSettings(Settings.settingsBuilder()

71
core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java
View File

@ -97,7 +97,6 @@ import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.codec.CodecService;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MappedFieldType.Loading;
import org.elasticsearch.index.mapper.internal.SizeFieldMapper;
@ -866,56 +865,44 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
* Waits till a (pattern) field name mappings concretely exists on all nodes. Note, this waits for the current
* started shards and checks for concrete mappings.
*/
public void waitForConcreteMappingsOnAll(final String index, final String type, final String... fieldNames) throws Exception {
assertBusy(new Runnable() {
@Override
public void run() {
Set<String> nodes = internalCluster().nodesInclude(index);
assertThat(nodes, Matchers.not(Matchers.emptyIterable()));
for (String node : nodes) {
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
IndexService indexService = indicesService.indexService(index);
assertThat("index service doesn't exists on " + node, indexService, notNullValue());
DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
assertThat("document mapper doesn't exists on " + node, documentMapper, notNullValue());
for (String fieldName : fieldNames) {
Collection<String> matches = documentMapper.mappers().simpleMatchToFullName(fieldName);
assertThat("field " + fieldName + " doesn't exists on " + node, matches, Matchers.not(emptyIterable()));
}
}
public void assertConcreteMappingsOnAll(final String index, final String type, final String... fieldNames) throws Exception {
Set<String> nodes = internalCluster().nodesInclude(index);
assertThat(nodes, Matchers.not(Matchers.emptyIterable()));
for (String node : nodes) {
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
IndexService indexService = indicesService.indexService(index);
assertThat("index service doesn't exists on " + node, indexService, notNullValue());
DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
assertThat("document mapper doesn't exists on " + node, documentMapper, notNullValue());
for (String fieldName : fieldNames) {
Collection<String> matches = documentMapper.mappers().simpleMatchToFullName(fieldName);
assertThat("field " + fieldName + " doesn't exists on " + node, matches, Matchers.not(emptyIterable()));
}
});
waitForMappingOnMaster(index, type, fieldNames);
}
assertMappingOnMaster(index, type, fieldNames);
}
/**
* Waits for the given mapping type to exists on the master node.
*/
public void waitForMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception {
assertBusy(new Callable() {
@Override
public Object call() throws Exception {
GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get();
ImmutableOpenMap<String, MappingMetaData> mappings = response.getMappings().get(index);
assertThat(mappings, notNullValue());
MappingMetaData mappingMetaData = mappings.get(type);
assertThat(mappingMetaData, notNullValue());
public void assertMappingOnMaster(final String index, final String type, final String... fieldNames) throws Exception {
GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setTypes(type).get();
ImmutableOpenMap<String, MappingMetaData> mappings = response.getMappings().get(index);
assertThat(mappings, notNullValue());
MappingMetaData mappingMetaData = mappings.get(type);
assertThat(mappingMetaData, notNullValue());
Map<String, Object> mappingSource = mappingMetaData.getSourceAsMap();
assertFalse(mappingSource.isEmpty());
assertTrue(mappingSource.containsKey("properties"));
Map<String, Object> mappingSource = mappingMetaData.getSourceAsMap();
assertFalse(mappingSource.isEmpty());
assertTrue(mappingSource.containsKey("properties"));
for (String fieldName : fieldNames) {
Map<String, Object> mappingProperties = (Map<String, Object>) mappingSource.get("properties");
if (fieldName.indexOf('.') != -1) {
fieldName = fieldName.replace(".", ".properties.");
}
assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(), XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue());
}
return null;
for (String fieldName : fieldNames) {
Map<String, Object> mappingProperties = (Map<String, Object>) mappingSource.get("properties");
if (fieldName.indexOf('.') != -1) {
fieldName = fieldName.replace(".", ".properties.");
}
});
assertThat("field " + fieldName + " doesn't exists in mapping " + mappingMetaData.source().string(), XContentMapValues.extractValue(fieldName, mappingProperties), notNullValue());
}
}
/**