From 35f9ee7a6288da829ccd579365dda7f4d76247bf Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 28 Aug 2015 10:52:04 +0200 Subject: [PATCH] Tests: better isolation of cluster ports Previously multiple clusters in the same JVM reused the same port ranges, leading to potential big gaps in port selection, which in turns causes unicast based discovery to fail, missing to find another node in the default 5 port range. Also the previous logic had http use a range that is assigned to another JVMs. --- .../test/InternalTestCluster.java | 23 ++++++++++++-- .../ClusterDiscoveryConfiguration.java | 2 +- .../test/test/InternalTestClusterTests.java | 30 +++++++++++++------ .../java/org/elasticsearch/tribe/TribeIT.java | 16 ++-------- 4 files changed, 44 insertions(+), 27 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java b/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java index 217e8da3340..8916d00554b 100644 --- a/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java +++ b/core/src/test/java/org/elasticsearch/test/InternalTestCluster.java @@ -171,8 +171,25 @@ public final class InternalTestCluster extends TestCluster { */ public static final int PORTS_PER_JVM = 100; + /** + * The number of ports in the range used for this cluster + */ + public static final int PORTS_PER_CLUSTER = 20; + + private static final int GLOBAL_TRANSPORT_BASE_PORT = 9300; + private static final int GLOBAL_HTTP_BASE_PORT = 19200; + private static final int JVM_ORDINAL = Integer.parseInt(System.getProperty(SysGlobals.CHILDVM_SYSPROP_JVM_ID, "0")); - public static final int BASE_PORT = 9300 + PORTS_PER_JVM * (JVM_ORDINAL + 1); + + /** a per-JVM unique offset to be used for calculating unique port ranges. */ + public static final int JVM_BASE_PORT_OFFEST = PORTS_PER_JVM * (JVM_ORDINAL + 1); + + private static final AtomicInteger clusterOrdinal = new AtomicInteger(); + private final int CLUSTER_BASE_PORT_OFFSET = JVM_BASE_PORT_OFFEST + (clusterOrdinal.getAndIncrement() * PORTS_PER_CLUSTER) % PORTS_PER_JVM; + + public final int TRANSPORT_BASE_PORT = GLOBAL_TRANSPORT_BASE_PORT + CLUSTER_BASE_PORT_OFFSET; + public final int HTTP_BASE_PORT = GLOBAL_HTTP_BASE_PORT + CLUSTER_BASE_PORT_OFFSET; + private static final boolean ENABLE_MOCK_MODULES = RandomizedTest.systemPropertyAsBoolean(TESTS_ENABLE_MOCK_MODULES, true); @@ -288,8 +305,8 @@ public final class InternalTestCluster extends TestCluster { builder.put("path.shared_data", baseDir.resolve("custom")); builder.put("path.home", baseDir); builder.put("path.repo", baseDir.resolve("repos")); - builder.put("transport.tcp.port", BASE_PORT + "-" + (BASE_PORT + 100)); - builder.put("http.port", BASE_PORT + 101 + "-" + (BASE_PORT + 200)); + builder.put("transport.tcp.port", TRANSPORT_BASE_PORT + "-" + (TRANSPORT_BASE_PORT + PORTS_PER_CLUSTER)); + builder.put("http.port", HTTP_BASE_PORT + "-" + (HTTP_BASE_PORT + PORTS_PER_CLUSTER)); builder.put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true); builder.put("node.mode", nodeMode); builder.put("http.pipelining", enableHttpPipelining); diff --git a/core/src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java b/core/src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java index b7d0c2dd930..4c426e70c48 100644 --- a/core/src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java +++ b/core/src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java @@ -103,7 +103,7 @@ public class ClusterDiscoveryConfiguration extends SettingsSource { } private static int calcBasePort() { - return 30000 + InternalTestCluster.BASE_PORT; + return 30000 + InternalTestCluster.JVM_BASE_PORT_OFFEST; } @Override diff --git a/core/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/core/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index cf5c00e1575..a3f4a9921b0 100644 --- a/core/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/core/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -30,9 +30,7 @@ import org.elasticsearch.test.SettingsSource; import java.io.IOException; import java.nio.file.Path; -import java.util.Iterator; -import java.util.Map; -import java.util.Random; +import java.util.*; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; @@ -57,26 +55,40 @@ public class InternalTestClusterTests extends ESTestCase { Path baseDir = createTempDir(); InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, settingsSource, numClientNodes, enableHttpPipelining, nodePrefix); InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, settingsSource, numClientNodes, enableHttpPipelining, nodePrefix); - assertClusters(cluster0, cluster1, true); + // TODO: this is not ideal - we should have a way to make sure ports are initialized in the same way + assertClusters(cluster0, cluster1, false); } - public static void assertClusters(InternalTestCluster cluster0, InternalTestCluster cluster1, boolean assertClusterName) { + /** + * a set of settings that are expected to have different values betweem clusters, even they have been initialized with the same + * base settins. + */ + final static Set clusterUniqueSettings = new HashSet<>(); + + static { + clusterUniqueSettings.add(ClusterName.SETTING); + clusterUniqueSettings.add("transport.tcp.port"); + clusterUniqueSettings.add("http.port"); + clusterUniqueSettings.add("http.port"); + } + + public static void assertClusters(InternalTestCluster cluster0, InternalTestCluster cluster1, boolean checkClusterUniqueSettings) { Settings defaultSettings0 = cluster0.getDefaultSettings(); Settings defaultSettings1 = cluster1.getDefaultSettings(); - assertSettings(defaultSettings0, defaultSettings1, assertClusterName); + assertSettings(defaultSettings0, defaultSettings1, checkClusterUniqueSettings); assertThat(cluster0.numDataNodes(), equalTo(cluster1.numDataNodes())); - if (assertClusterName) { + if (checkClusterUniqueSettings) { assertThat(cluster0.getClusterName(), equalTo(cluster1.getClusterName())); } } - public static void assertSettings(Settings left, Settings right, boolean compareClusterName) { + public static void assertSettings(Settings left, Settings right, boolean checkClusterUniqueSettings) { ImmutableSet> entries0 = left.getAsMap().entrySet(); Map entries1 = right.getAsMap(); assertThat(entries0.size(), equalTo(entries1.size())); for (Map.Entry entry : entries0) { - if(entry.getKey().equals(ClusterName.SETTING) && compareClusterName == false) { + if (clusterUniqueSettings.contains(entry.getKey()) && checkClusterUniqueSettings == false) { continue; } assertThat(entries1, hasEntry(entry.getKey(), entry.getValue())); diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java index 4847d2b528e..9a633e4ae3d 100644 --- a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java +++ b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java @@ -76,20 +76,8 @@ public class TribeIT extends ESIntegTestCase { @BeforeClass public static void setupSecondCluster() throws Exception { ESIntegTestCase.beforeClass(); - SettingsSource source = new SettingsSource() { - @Override - public Settings node(int nodeOrdinal) { - final int base = InternalTestCluster.BASE_PORT + 1000; - return Settings.builder().put("transport.tcp.port", base + "-" + (base + 100)).build(); - } - - @Override - public Settings transportClient() { - return node(0); - } - }; - // create another cluster - cluster2 = new InternalTestCluster(InternalTestCluster.configuredNodeMode(), randomLong(), createTempDir(), 2, 2, Strings.randomBase64UUID(getRandom()), source, 0, false, SECOND_CLUSTER_NODE_PREFIX); + cluster2 = new InternalTestCluster(InternalTestCluster.configuredNodeMode(), randomLong(), createTempDir(), 2, 2, + Strings.randomBase64UUID(getRandom()), SettingsSource.EMPTY, 0, false, SECOND_CLUSTER_NODE_PREFIX); cluster2.beforeTest(getRandom(), 0.1); cluster2.ensureAtLeastNumDataNodes(2);