From 2713e903ab140629020e26ee43415fd30ba23741 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Mon, 15 Jun 2015 16:16:18 +0200 Subject: [PATCH 1/2] [TEST] remove redundant tests and move to different suite Some of the test for meta data are redundant. Also, since they somewhat test service disruptions (start master with empty data folder) we might move them to DiscoveryWithServiceDisruptionsTests. Also, this commit adds a test for https://github.com/elastic/elasticsearch/issues/11665 --- .../cluster/ClusterChangedEvent.java | 7 +- .../DiscoveryWithServiceDisruptionsTests.java | 52 ++- .../gateway/MetaDataWriteDataNodesTests.java | 312 +++++------------- .../test/InternalTestCluster.java | 62 +++- 4 files changed, 205 insertions(+), 228 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java index e3469abde3e..1ceb822abfd 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java @@ -110,8 +110,11 @@ public class ClusterChangedEvent { // is actually supposed to be deleted or imported as dangling instead. for example a new master might not have // the index in its cluster state because it was started with an empty data folder and in this case we want to // import as dangling. we check here for new master too to be on the safe side in this case. - // norelease because we are not sure this is actually a good solution - // See discussion on https://github.com/elastic/elasticsearch/pull/9952 + // This means that under certain conditions deleted indices might be reimported if a master fails while the deletion + // request is issued and a node receives the cluster state that would trigger the deletion from the new master. + // See test MetaDataWriteDataNodesTests.testIndicesDeleted() + // See discussion on https://github.com/elastic/elasticsearch/pull/9952 and + // https://github.com/elastic/elasticsearch/issues/11665 if (hasNewMaster() || previousState == null) { return ImmutableList.of(); } diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsTests.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsTests.java index 0755b9ab6fd..8a758fe83d6 100644 --- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsTests.java @@ -56,6 +56,7 @@ import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.*; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -65,6 +66,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -237,7 +239,9 @@ public class DiscoveryWithServiceDisruptionsTests extends ElasticsearchIntegrati } - /** Verify that nodes fault detection works after master (re) election */ + /** + * Verify that nodes fault detection works after master (re) election + */ @Test public void testNodesFDAfterMasterReelection() throws Exception { startCluster(4); @@ -414,7 +418,7 @@ public class DiscoveryWithServiceDisruptionsTests extends ElasticsearchIntegrati /** * Test that we do not loose document whose indexing request was successful, under a randomly selected disruption scheme * We also collect & report the type of indexing failures that occur. - * + *

* This test is a superset of tests run in the Jepsen test suite, with the exception of versioned updates */ @Test @@ -948,6 +952,50 @@ public class DiscoveryWithServiceDisruptionsTests extends ElasticsearchIntegrati ensureStableCluster(3); } + @Test + public void testIndexImportedFromDataOnlyNodesIfMasterLostDataFolder() throws Exception { + // test for https://github.com/elastic/elasticsearch/issues/8823 + configureCluster(2, 1); + String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY); + internalCluster().startDataOnlyNode(Settings.EMPTY); + + ensureStableCluster(2); + assertAcked(prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 0))); + index("index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + ensureGreen(); + + internalCluster().restartNode(masterNode, new InternalTestCluster.RestartCallback() { + public boolean clearData(String nodeName) { + return true; + } + }); + + ensureGreen("index"); + assertTrue(client().prepareGet("index", "doc", "1").get().isExists()); + } + + // tests if indices are really deleted even if a master transition inbetween + @Ignore("https://github.com/elastic/elasticsearch/issues/11665") + @Test + public void testIndicesDeleted() throws Exception { + configureCluster(3, 2); + Future> masterNodes= internalCluster().startMasterOnlyNodesAsync(2); + Future dataNode = internalCluster().startDataOnlyNodeAsync(); + dataNode.get(); + masterNodes.get(); + ensureStableCluster(3); + assertAcked(prepareCreate("test")); + ensureYellow(); + + String masterNode1 = internalCluster().getMasterName(); + NetworkPartition networkPartition = new NetworkUnresponsivePartition(masterNode1, dataNode.get(), getRandom()); + internalCluster().setDisruptionScheme(networkPartition); + networkPartition.startDisrupting(); + internalCluster().client(masterNode1).admin().indices().prepareDelete("test").setTimeout("1s").get(); + internalCluster().restartNode(masterNode1, InternalTestCluster.EMPTY_CALLBACK); + ensureYellow(); + assertFalse(client().admin().indices().prepareExists("test").get().isExists()); + } protected NetworkPartition addRandomPartition() { NetworkPartition partition; diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java index 30608cf1a8d..f4fcc088710 100644 --- a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.gateway; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.base.Predicate; +import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -33,227 +34,81 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.InternalTestCluster; import org.junit.Test; -import java.io.IOException; import java.util.LinkedHashMap; +import java.util.concurrent.Future; -import static org.elasticsearch.client.Requests.clusterHealthRequest; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; +import static org.elasticsearch.test.InternalTestCluster.RestartCallback; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; /** * */ +@LuceneTestCase.Slow @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { @Test public void testMetaWrittenAlsoOnDataNode() throws Exception { - // this test checks that index state is written on data only nodes - String masterNodeName = startMasterNode(); - String redNode = startDataNode("red"); - assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.number_of_replicas", 0))); + // this test checks that index state is written on data only nodes if they have a shard allocated + String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY); + String dataNode = internalCluster().startDataOnlyNode(Settings.EMPTY); + assertAcked(prepareCreate("test").setSettings("index.number_of_replicas", 0)); index("test", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); ensureGreen("test"); - assertIndexInMetaState(redNode, "test"); - assertIndexInMetaState(masterNodeName, "test"); - //stop master node and start again with an empty data folder - ((InternalTestCluster) cluster()).stopCurrentMasterNode(); - String newMasterNode = startMasterNode(); - ensureGreen("test"); - // check for meta data - assertIndexInMetaState(redNode, "test"); - assertIndexInMetaState(newMasterNode, "test"); - // check if index and doc is still there - ensureGreen("test"); - assertTrue(client().prepareGet("test", "doc", "1").get().isExists()); - } - - @Test - public void testMetaWrittenOnlyForIndicesOnNodesThatHaveAShard() throws Exception { - // this test checks that the index state is only written to a data only node if they have a shard of that index allocated on the node - String masterNode = startMasterNode(); - String blueNode = startDataNode("blue"); - String redNode = startDataNode("red"); - - assertAcked(prepareCreate("blue_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "blue"))); - index("blue_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); - assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red"))); - index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); - ensureGreen(); - assertIndexNotInMetaState(blueNode, "red_index"); - assertIndexNotInMetaState(redNode, "blue_index"); - assertIndexInMetaState(blueNode, "blue_index"); - assertIndexInMetaState(redNode, "red_index"); - assertIndexInMetaState(masterNode, "red_index"); - assertIndexInMetaState(masterNode, "blue_index"); - - // not the index state for blue_index should only be written on blue_node and the for red_index only on red_node - // we restart red node and master but with empty data folders - stopNode(redNode); - ((InternalTestCluster) cluster()).stopCurrentMasterNode(); - masterNode = startMasterNode(); - redNode = startDataNode("red"); - - ensureGreen(); - assertIndexNotInMetaState(blueNode, "red_index"); - assertIndexInMetaState(blueNode, "blue_index"); - assertIndexNotInMetaState(redNode, "red_index"); - assertIndexNotInMetaState(redNode, "blue_index"); - assertIndexNotInMetaState(masterNode, "red_index"); - assertIndexInMetaState(masterNode, "blue_index"); - // check that blue index is still there - assertFalse(client().admin().indices().prepareExists("red_index").get().isExists()); - assertTrue(client().prepareGet("blue_index", "doc", "1").get().isExists()); - // red index should be gone - // if the blue node had stored the index state then cluster health would be red and red_index would exist - assertFalse(client().admin().indices().prepareExists("red_index").get().isExists()); - + assertIndexInMetaState(dataNode, "test"); + assertIndexInMetaState(masterNode, "test"); } @Test public void testMetaIsRemovedIfAllShardsFromIndexRemoved() throws Exception { // this test checks that the index state is removed from a data only node once all shards have been allocated away from it - String masterNode = startMasterNode(); - String blueNode = startDataNode("blue"); - String redNode = startDataNode("red"); - - // create blue_index on blue_node and same for red - client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("3")).get(); - assertAcked(prepareCreate("blue_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "blue"))); - index("blue_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); - assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red"))); - index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY); + Future nodeName1 = internalCluster().startDataOnlyNodeAsync(); + Future nodeName2 = internalCluster().startDataOnlyNodeAsync(); + String node1 = nodeName1.get(); + String node2 = nodeName2.get(); + String index = "index"; + assertAcked(prepareCreate(index).setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "_name", node1))); + index(index, "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); ensureGreen(); - assertIndexNotInMetaState(redNode, "blue_index"); - assertIndexNotInMetaState(blueNode, "red_index"); - assertIndexInMetaState(redNode, "red_index"); - assertIndexInMetaState(blueNode, "blue_index"); - assertIndexInMetaState(masterNode, "red_index"); - assertIndexInMetaState(masterNode, "blue_index"); + assertIndexInMetaState(node1, index); + assertIndexNotInMetaState(node2, index); + assertIndexInMetaState(masterNode, index); - // now relocate blue_index to red_node and red_index to blue_node - logger.debug("relocating indices..."); - client().admin().indices().prepareUpdateSettings("blue_index").setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")).get(); - client().admin().indices().prepareUpdateSettings("red_index").setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "blue")).get(); + logger.debug("relocating index..."); + client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder().put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "_name", node2)).get(); client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get(); ensureGreen(); - assertIndexNotInMetaState(redNode, "red_index"); - assertIndexNotInMetaState(blueNode, "blue_index"); - assertIndexInMetaState(redNode, "blue_index"); - assertIndexInMetaState(blueNode, "red_index"); - assertIndexInMetaState(masterNode, "red_index"); - assertIndexInMetaState(masterNode, "blue_index"); - - //at this point the blue_index is on red node and the red_index on blue node - // now, when we start red and master node again but without data folder, the red index should be gone but the blue index should initialize fine - stopNode(redNode); - ((InternalTestCluster) cluster()).stopCurrentMasterNode(); - masterNode = startMasterNode(); - redNode = startDataNode("red"); - ensureGreen(); - assertIndexNotInMetaState(redNode, "blue_index"); - assertIndexNotInMetaState(blueNode, "blue_index"); - assertIndexNotInMetaState(redNode, "red_index"); - assertIndexInMetaState(blueNode, "red_index"); - assertIndexInMetaState(masterNode, "red_index"); - assertIndexNotInMetaState(masterNode, "blue_index"); - assertTrue(client().prepareGet("red_index", "doc", "1").get().isExists()); - // if the red_node had stored the index state then cluster health would be red and blue_index would exist - assertFalse(client().admin().indices().prepareExists("blue_index").get().isExists()); - } - - @Test - public void testMetaWrittenWhenIndexIsClosed() throws Exception { - String masterNode = startMasterNode(); - String redNodeDataPath = createTempDir().toString(); - String redNode = startDataNode("red", redNodeDataPath); - String blueNode = startDataNode("blue"); - // create red_index on red_node and same for red - client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("3")).get(); - assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red"))); - index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); - - ensureGreen(); - assertIndexNotInMetaState(blueNode, "red_index"); - assertIndexInMetaState(redNode, "red_index"); - assertIndexInMetaState(masterNode, "red_index"); - - client().admin().indices().prepareClose("red_index").get(); - // close the index - ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get(); - assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.CLOSE.name())); - - // restart master with empty data folder and maybe red node - boolean restartRedNode = randomBoolean(); - //at this point the red_index on red node - if (restartRedNode) { - stopNode(redNode); - } - ((InternalTestCluster) cluster()).stopCurrentMasterNode(); - masterNode = startMasterNode(); - if (restartRedNode) { - redNode = startDataNode("red", redNodeDataPath); - } - - ensureGreen("red_index"); - assertIndexNotInMetaState(blueNode, "red_index"); - assertIndexInMetaState(redNode, "red_index"); - assertIndexInMetaState(masterNode, "red_index"); - clusterStateResponse = client().admin().cluster().prepareState().get(); - assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.CLOSE.name())); - - // open the index again - client().admin().indices().prepareOpen("red_index").get(); - clusterStateResponse = client().admin().cluster().prepareState().get(); - assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.OPEN.name())); - // restart again - ensureGreen(); - if (restartRedNode) { - stopNode(redNode); - } - ((InternalTestCluster) cluster()).stopCurrentMasterNode(); - masterNode = startMasterNode(); - if (restartRedNode) { - redNode = startDataNode("red", redNodeDataPath); - } - ensureGreen("red_index"); - assertIndexNotInMetaState(blueNode, "red_index"); - assertIndexInMetaState(redNode, "red_index"); - assertIndexInMetaState(masterNode, "red_index"); - clusterStateResponse = client().admin().cluster().prepareState().get(); - assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.OPEN.name())); - assertTrue(client().prepareGet("red_index", "doc", "1").get().isExists()); + assertIndexNotInMetaState(node1, index); + assertIndexInMetaState(node2, index); + assertIndexInMetaState(masterNode, index); } @Test public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception { - String masterNode = startMasterNode(); - String redNodeDataPath = createTempDir().toString(); - String redNode = startDataNode("red", redNodeDataPath); - // create red_index on red_node and same for red - client().admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("2")).get(); - assertAcked(prepareCreate("red_index").setSettings(Settings.builder().put("index.number_of_replicas", 0).put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red"))); - index("red_index", "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); + String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY); + final String dataNode = internalCluster().startDataOnlyNode(Settings.EMPTY); - logger.info("--> wait for green red_index"); + final String index = "index"; + assertAcked(prepareCreate(index).setSettings(Settings.builder().put("index.number_of_replicas", 0))); + logger.info("--> wait for green index"); ensureGreen(); - logger.info("--> wait for meta state written for red_index"); - assertIndexInMetaState(redNode, "red_index"); - assertIndexInMetaState(masterNode, "red_index"); + logger.info("--> wait for meta state written for index"); + assertIndexInMetaState(dataNode, index); + assertIndexInMetaState(masterNode, index); - logger.info("--> close red_index"); - client().admin().indices().prepareClose("red_index").get(); + logger.info("--> close index"); + client().admin().indices().prepareClose(index).get(); // close the index ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get(); - assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.CLOSE.name())); + assertThat(clusterStateResponse.getState().getMetaData().index(index).getState().name(), equalTo(IndexMetaData.State.CLOSE.name())); - logger.info("--> restart red node"); - stopNode(redNode); - redNode = startDataNode("red", redNodeDataPath); - client().admin().indices().preparePutMapping("red_index").setType("doc").setSource(jsonBuilder().startObject() + // update the mapping. this should cause the new meta data to be written although index is closed + client().admin().indices().preparePutMapping(index).setType("doc").setSource(jsonBuilder().startObject() .startObject("properties") .startObject("integer_field") .field("type", "integer") @@ -261,45 +116,54 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { .endObject() .endObject()).get(); - GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("red_index").addTypes("doc").get(); - assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get("red_index").get("doc").getSourceAsMap().get("properties"))).get("integer_field")); - // restart master with empty data folder and maybe red node - ((InternalTestCluster) cluster()).stopCurrentMasterNode(); - masterNode = startMasterNode(); + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).addTypes("doc").get(); + assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get(index).get("doc").getSourceAsMap().get("properties"))).get("integer_field")); - ensureGreen("red_index"); - assertIndexInMetaState(redNode, "red_index"); - assertIndexInMetaState(masterNode, "red_index"); - clusterStateResponse = client().admin().cluster().prepareState().get(); - assertThat(clusterStateResponse.getState().getMetaData().index("red_index").getState().name(), equalTo(IndexMetaData.State.CLOSE.name())); - getMappingsResponse = client().admin().indices().prepareGetMappings("red_index").addTypes("doc").get(); - assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get("red_index").get("doc").getSourceAsMap().get("properties"))).get("integer_field")); + // make sure it was also written on red node although index is closed + ImmutableOpenMap indicesMetaData = getIndicesMetaDataOnNode(dataNode); + assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("doc").getSourceAsMap().get("properties"))).get("integer_field")); + assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.CLOSE)); - } + /** + * Try the same and see if this also works if node was just restarted. + * Each node holds an array of indices it knows of and checks if it should + * write new meta data by looking up in this array. We need it because if an + * index is closed it will not appear in the shard routing and we therefore + * need to keep track of what we wrote before. However, when the node is + * restarted this array is empty and we have to fill it before we decide + * what we write. This is why I explicitly test for it. + */ + internalCluster().restartNode(dataNode, new RestartCallback()); + client().admin().indices().preparePutMapping(index).setType("doc").setSource(jsonBuilder().startObject() + .startObject("properties") + .startObject("float_field") + .field("type", "float") + .endObject() + .endObject() + .endObject()).get(); - private String startDataNode(String color) { - return startDataNode(color, createTempDir().toString()); - } + getMappingsResponse = client().admin().indices().prepareGetMappings(index).addTypes("doc").get(); + assertNotNull(((LinkedHashMap) (getMappingsResponse.getMappings().get(index).get("doc").getSourceAsMap().get("properties"))).get("float_field")); - private String startDataNode(String color, String newDataPath) { - Settings.Builder settingsBuilder = Settings.builder() - .put("node.data", true) - .put("node.master", false) - .put("node.color", color) - .put("path.data", newDataPath); - return internalCluster().startNode(settingsBuilder.build()); - } + // make sure it was also written on red node although index is closed + indicesMetaData = getIndicesMetaDataOnNode(dataNode); + assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("doc").getSourceAsMap().get("properties"))).get("float_field")); + assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.CLOSE)); - private String startMasterNode() { - Settings.Builder settingsBuilder = Settings.builder() - .put("node.data", false) - .put("node.master", true) - .put("path.data", createTempDir().toString()); - return internalCluster().startNode(settingsBuilder.build()); - } - - private void stopNode(String name) throws IOException { - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(name)); + // finally check that meta data is also written of index opened again + client().admin().indices().prepareOpen(index).get(); + assertBusy(new Runnable() { + @Override + public void run() { + try { + ImmutableOpenMap indicesMetaData = getIndicesMetaDataOnNode(dataNode); + assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.OPEN)); + } catch (Exception e) { + logger.info("caught exception while reading meta state: ", e); + fail(); + } + } + }); } protected void assertIndexNotInMetaState(String nodeName, String indexName) throws Exception { @@ -335,14 +199,18 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { } private boolean metaStateExists(String nodeName, String indexName) throws Exception { - GatewayMetaState nodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName); - MetaData nodeMetaData = null; - nodeMetaData = nodeMetaState.loadMetaState(); - ImmutableOpenMap indices = nodeMetaData.getIndices(); + ImmutableOpenMap indices = getIndicesMetaDataOnNode(nodeName); boolean inMetaSate = false; for (ObjectObjectCursor index : indices) { inMetaSate = inMetaSate || index.key.equals(indexName); } return inMetaSate; } + + private ImmutableOpenMap getIndicesMetaDataOnNode(String nodeName) throws Exception { + GatewayMetaState nodeMetaState = ((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName); + MetaData nodeMetaData = null; + nodeMetaData = nodeMetaState.loadMetaState(); + return nodeMetaData.getIndices(); + } } diff --git a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java b/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java index ac785c1601d..5204f0b456d 100644 --- a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java +++ b/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java @@ -1286,6 +1286,18 @@ public final class InternalTestCluster extends TestCluster { } } + /** + * Restarts a node and calls the callback during restart. + */ + public void restartNode(String nodeName, RestartCallback callback) throws Exception { + ensureOpen(); + NodeAndClient nodeAndClient = nodes.get(nodeName); + if (nodeAndClient != null) { + logger.info("Restarting node [{}] ", nodeAndClient.name); + nodeAndClient.restart(callback); + } + } + private void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception { ensureOpen(); List toRemove = new ArrayList<>(); @@ -1343,7 +1355,7 @@ public final class InternalTestCluster extends TestCluster { } - private static final RestartCallback EMPTY_CALLBACK = new RestartCallback() { + public static final RestartCallback EMPTY_CALLBACK = new RestartCallback() { @Override public Settings onNodeStopped(String node) { return null; @@ -1468,6 +1480,52 @@ public final class InternalTestCluster extends TestCluster { return buildNode.name; } + public synchronized ListenableFuture> startMasterOnlyNodesAsync(int numNodes) { + return startMasterOnlyNodesAsync(numNodes, Settings.EMPTY); + } + + public synchronized ListenableFuture> startMasterOnlyNodesAsync(int numNodes, Settings settings) { + Settings settings1 = Settings.builder().put(settings).put("node.master", true).put("node.data", false).build(); + return startNodesAsync(numNodes, settings1, Version.CURRENT); + } + + public synchronized ListenableFuture> startDataOnlyNodesAsync(int numNodes) { + return startDataOnlyNodesAsync(numNodes, Settings.EMPTY); + } + + public synchronized ListenableFuture> startDataOnlyNodesAsync(int numNodes, Settings settings) { + Settings settings1 = Settings.builder().put(settings).put("node.master", false).put("node.data", true).build(); + return startNodesAsync(numNodes, settings1, Version.CURRENT); + } + + public synchronized ListenableFuture startMasterOnlyNodeAsync() { + return startMasterOnlyNodeAsync(Settings.EMPTY); + } + + public synchronized ListenableFuture startMasterOnlyNodeAsync(Settings settings) { + Settings settings1 = Settings.builder().put(settings).put("node.master", true).put("node.data", false).build(); + return startNodeAsync(settings1, Version.CURRENT); + } + + public synchronized String startMasterOnlyNode(Settings settings) { + Settings settings1 = Settings.builder().put(settings).put("node.master", true).put("node.data", false).build(); + return startNode(settings1, Version.CURRENT); + } + + public synchronized ListenableFuture startDataOnlyNodeAsync() { + return startDataOnlyNodeAsync(Settings.EMPTY); + } + + public synchronized ListenableFuture startDataOnlyNodeAsync(Settings settings) { + Settings settings1 = Settings.builder().put(settings).put("node.master", false).put("node.data", true).build(); + return startNodeAsync(settings1, Version.CURRENT); + } + + public synchronized String startDataOnlyNode(Settings settings) { + Settings settings1 = Settings.builder().put(settings).put("node.master", false).put("node.data", true).build(); + return startNode(settings1, Version.CURRENT); + } + /** * Starts a node in an async manner with the given settings and returns future with its name. */ @@ -1726,7 +1784,7 @@ public final class InternalTestCluster extends TestCluster { * and / or {@link #fullRestart(InternalTestCluster.RestartCallback)} to execute actions at certain * stages of the restart. */ - public static abstract class RestartCallback { + public static class RestartCallback { /** * Executed once the give node name has been stopped. From e44c5ff70384751bfa9bd5224183c1bc8c68daa0 Mon Sep 17 00:00:00 2001 From: Britta Weber Date: Thu, 23 Jul 2015 16:39:21 +0200 Subject: [PATCH 2/2] check if index is closed or was previously closed when gathering relevant indices to write meta state When an index is opened it will not be assigned to a node but also not have closed state anymore. Before we only checked if an index either is closed or assigned to the data node and therefore the change from close->open was not written. --- .../gateway/GatewayMetaState.java | 21 +++++++++++++------ .../gateway/GatewayMetaStateTests.java | 4 ++-- .../gateway/MetaDataWriteDataNodesTests.java | 21 +++++-------------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index 86d23ee4416..b927cbbde84 100644 --- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -28,7 +28,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; @@ -98,6 +99,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL @Override public void clusterChanged(ClusterChangedEvent event) { + Set relevantIndices = new HashSet<>(); final ClusterState state = event.state(); if (state.blocks().disableStatePersistence()) { @@ -148,7 +150,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL } Iterable writeInfo; - relevantIndices = getRelevantIndices(event.state(), previouslyWrittenIndices); + relevantIndices = getRelevantIndices(event.state(), event.previousState(), previouslyWrittenIndices); writeInfo = resolveStatesToBeWritten(previouslyWrittenIndices, relevantIndices, previousMetaData, event.state().metaData()); // check and write changes in indices for (IndexMetaWriteInfo indexMetaWrite : writeInfo) { @@ -169,10 +171,10 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL } } - public static Set getRelevantIndices(ClusterState state, ImmutableSet previouslyWrittenIndices) { + public static Set getRelevantIndices(ClusterState state, ClusterState previousState,ImmutableSet previouslyWrittenIndices) { Set relevantIndices; if (isDataOnlyNode(state)) { - relevantIndices = getRelevantIndicesOnDataOnlyNode(state, previouslyWrittenIndices); + relevantIndices = getRelevantIndicesOnDataOnlyNode(state, previousState, previouslyWrittenIndices); } else if (state.nodes().localNode().masterNode() == true) { relevantIndices = getRelevantIndicesForMasterEligibleNode(state); } else { @@ -278,7 +280,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL return indicesToWrite; } - public static Set getRelevantIndicesOnDataOnlyNode(ClusterState state, ImmutableSet previouslyWrittenIndices) { + public static Set getRelevantIndicesOnDataOnlyNode(ClusterState state, ClusterState previousState, ImmutableSet previouslyWrittenIndices) { RoutingNode newRoutingNode = state.getRoutingNodes().node(state.nodes().localNodeId()); if (newRoutingNode == null) { throw new IllegalStateException("cluster state does not contain this node - cannot write index meta state"); @@ -289,7 +291,14 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL } // we have to check the meta data also: closed indices will not appear in the routing table, but we must still write the state if we have it written on disk previously for (IndexMetaData indexMetaData : state.metaData()) { - if (previouslyWrittenIndices.contains(indexMetaData.getIndex()) && state.metaData().getIndices().get(indexMetaData.getIndex()).state().equals(IndexMetaData.State.CLOSE)) { + boolean isOrWasClosed = indexMetaData.state().equals(IndexMetaData.State.CLOSE); + // if the index is open we might still have to write the state if it just transitioned from closed to open + // so we have to check for that as well. + IndexMetaData previousMetaData = previousState.metaData().getIndices().get(indexMetaData.getIndex()); + if (previousMetaData != null) { + isOrWasClosed = isOrWasClosed || previousMetaData.state().equals(IndexMetaData.State.CLOSE); + } + if (previouslyWrittenIndices.contains(indexMetaData.getIndex()) && isOrWasClosed) { indices.add(indexMetaData.getIndex()); } } diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index ffe4e716b5f..594dee7c57f 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -174,9 +174,9 @@ public class GatewayMetaStateTests extends ElasticsearchAllocationTestCase { if (stateInMemory) { inMemoryMetaData = event.previousState().metaData(); ImmutableSet.Builder relevantIndices = ImmutableSet.builder(); - oldIndicesList = relevantIndices.addAll(GatewayMetaState.getRelevantIndices(event.previousState(), oldIndicesList)).build(); + oldIndicesList = relevantIndices.addAll(GatewayMetaState.getRelevantIndices(event.previousState(), event.previousState(), oldIndicesList)).build(); } - Set newIndicesList = GatewayMetaState.getRelevantIndices(event.state(), oldIndicesList); + Set newIndicesList = GatewayMetaState.getRelevantIndices(event.state(),event.previousState(), oldIndicesList); // third, get the actual write info Iterator indices = GatewayMetaState.resolveStatesToBeWritten(oldIndicesList, newIndicesList, inMemoryMetaData, event.state().metaData()).iterator(); diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java index f4fcc088710..afa3c5b44c1 100644 --- a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesTests.java @@ -124,14 +124,13 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { assertNotNull(((LinkedHashMap) (indicesMetaData.get(index).getMappings().get("doc").getSourceAsMap().get("properties"))).get("integer_field")); assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.CLOSE)); - /** - * Try the same and see if this also works if node was just restarted. + /* Try the same and see if this also works if node was just restarted. * Each node holds an array of indices it knows of and checks if it should * write new meta data by looking up in this array. We need it because if an * index is closed it will not appear in the shard routing and we therefore * need to keep track of what we wrote before. However, when the node is * restarted this array is empty and we have to fill it before we decide - * what we write. This is why I explicitly test for it. + * what we write. This is why we explicitly test for it. */ internalCluster().restartNode(dataNode, new RestartCallback()); client().admin().indices().preparePutMapping(index).setType("doc").setSource(jsonBuilder().startObject() @@ -151,19 +150,9 @@ public class MetaDataWriteDataNodesTests extends ElasticsearchIntegrationTest { assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.CLOSE)); // finally check that meta data is also written of index opened again - client().admin().indices().prepareOpen(index).get(); - assertBusy(new Runnable() { - @Override - public void run() { - try { - ImmutableOpenMap indicesMetaData = getIndicesMetaDataOnNode(dataNode); - assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.OPEN)); - } catch (Exception e) { - logger.info("caught exception while reading meta state: ", e); - fail(); - } - } - }); + assertAcked(client().admin().indices().prepareOpen(index).get()); + indicesMetaData = getIndicesMetaDataOnNode(dataNode); + assertThat(indicesMetaData.get(index).state(), equalTo(IndexMetaData.State.OPEN)); } protected void assertIndexNotInMetaState(String nodeName, String indexName) throws Exception {