From 7949f92becde2045864ae92e1860bfe31369f1fa Mon Sep 17 00:00:00 2001 From: Rabi Panda Date: Fri, 9 Apr 2021 10:24:41 -0700 Subject: [PATCH] Cleanup `default` flavor stragglers from docker distributions. (#481) This commit removes the references for the `default` docker distributions which were originally part of the different flavors of distributions. This also fixes some of the failing docker compose tests under `qa`. Signed-off-by: Rabi Panda --- distribution/build.gradle | 40 ---------- distribution/docker/build.gradle | 3 +- distribution/docker/docker-compose.yml | 4 +- distribution/docker/docker-test-entrypoint.sh | 2 +- .../docker/test/DockerYmlTestSuiteIT.java | 32 +------- qa/remote-clusters/build.gradle | 14 +--- qa/remote-clusters/docker-compose-oss.yml | 74 ------------------- qa/remote-clusters/docker-compose.yml | 47 +++++------- .../AbstractMultiClusterRemoteTestCase.java | 29 +------- qa/wildfly/build.gradle | 10 +-- qa/wildfly/docker-compose-oss.yml | 35 --------- qa/wildfly/docker-compose.yml | 4 +- 12 files changed, 34 insertions(+), 260 deletions(-) delete mode 100644 qa/remote-clusters/docker-compose-oss.yml delete mode 100644 qa/wildfly/docker-compose-oss.yml diff --git a/distribution/build.gradle b/distribution/build.gradle index 5880d3d085c..c7cdb5fc540 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -58,17 +58,10 @@ tasks.register("generateDependenciesReport", ConcatFilesTask) { // integ test zip only uses server, so a different notice file is needed there def buildServerNoticeTaskProvider = tasks.register("buildServerNotice", NoticeTask) -// other distributions include notices from modules as well, which are added below later -def buildDefaultNoticeTaskProvider = tasks.register("buildDefaultNotice", NoticeTask) { - licensesDir new File(project(':distribution').projectDir, 'licenses') -} - def buildOssNoticeTaskProvider = tasks.register("buildOssNotice", NoticeTask) { licensesDir new File(project(':distribution').projectDir, 'licenses') } -def buildDefaultNoJdkNoticeTaskProvider = tasks.register("buildDefaultNoJdkNotice", NoticeTask) - def buildOssNoJdkNoticeTaskProvider = tasks.register("buildOssNoJdkNotice", NoticeTask) // The :server and :libs projects belong to all distributions @@ -85,7 +78,6 @@ tasks.withType(NoticeTask).configureEach { * Modules * *****************************************************************************/ String ossOutputs = 'build/outputs/oss' -String defaultOutputs = 'build/outputs/default' String systemdOutputs = 'build/outputs/systemd' String transportOutputs = 'build/outputs/transport-only' String externalTestOutputs = 'build/outputs/external-test' @@ -94,11 +86,6 @@ def processOssOutputsTaskProvider = tasks.register("processOssOutputs", Sync) { into ossOutputs } -def processDefaultOutputsTaskProvider = tasks.register("processDefaultOutputs", Sync) { - into defaultOutputs - from processOssOutputsTaskProvider -} - def processSystemdOutputsTaskProvider = tasks.register("processSystemdOutputs", Sync) { into systemdOutputs } @@ -126,18 +113,6 @@ tasks.register("buildOssConfig") { dependsOn "processOssOutputs" outputs.dir "${ossOutputs}/config" } -def buildDefaultModulesTaskProvider = tasks.register("buildDefaultModules") { - dependsOn processDefaultOutputsTaskProvider - outputs.dir "${defaultOutputs}/modules" -} -tasks.register("buildDefaultBin") { - dependsOn processDefaultOutputsTaskProvider - outputs.dir "${defaultOutputs}/bin" -} -def buildDefaultConfigTaskProvider = tasks.register("buildDefaultConfig") { - dependsOn processOssOutputsTaskProvider - outputs.dir "${defaultOutputs}/config" -} def buildSystemdModuleTaskProvider = tasks.register("buildSystemdModule") { dependsOn "processSystemdOutputs" outputs.dir "${systemdOutputs}/modules" @@ -192,12 +167,6 @@ def buildOssLog4jConfigTaskProvider = tasks.register("buildOssLog4jConfig") { ext.log4jFile = file("${ossOutputs}/log4j2.properties") outputs.file log4jFile } -def buildDefaultLog4jConfigTaskProvider = tasks.register("buildDefaultLog4jConfig") { - dependsOn processDefaultOutputsTaskProvider - ext.contents = [] - ext.log4jFile = file("${defaultOutputs}/log4j2.properties") - outputs.file log4jFile -} Closure writeLog4jProperties = { String mainLog4jProperties = file('src/config/log4j2.properties').getText('UTF-8') @@ -210,10 +179,6 @@ buildOssLog4jConfigTaskProvider.configure { doLast(writeLog4jProperties) } -buildDefaultLog4jConfigTaskProvider.configure { - doLast(writeLog4jProperties) -} - // copy log4j2.properties from modules that have it void copyLog4jProperties(TaskProvider buildTask, Project module) { buildTask.configure { @@ -246,10 +211,6 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { } File licenses = new File(module.projectDir, 'licenses') if (licenses.exists()) { - buildDefaultNoticeTaskProvider.configure { - licensesDir licenses - source module.file('src/main/java') - } buildOssNotice.configure { licensesDir licenses source module.file('src/main/java') @@ -262,7 +223,6 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { } copyLog4jProperties(buildOssLog4jConfigTaskProvider, module) - copyLog4jProperties(buildDefaultLog4jConfigTaskProvider, module) restTestExpansions['expected.modules.count'] += 1 } diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index aac3e138444..947394b835b 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -182,8 +182,7 @@ void addBuildDockerImage(Architecture architecture, DockerBase base) { String version = VersionProperties.getOpenSearch() tags = [ - //TODO remove the tag and replace with OpenSearch docker tag - "docker.elastic.co/elasticsearch/elasticsearch-oss:${version}", + "docker.opensearch.org/opensearch/opensearch-oss:${version}", "opensearch-oss:test" ] } diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml index f9e6741f9ca..c5ecf823c38 100644 --- a/distribution/docker/docker-compose.yml +++ b/distribution/docker/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: opensearch-oss-1: - image: opensearch:test + image: opensearch-oss:test environment: - node.name=opensearch-oss-1 - cluster.initial_master_nodes=opensearch-oss-1,opensearch-oss-2 @@ -26,7 +26,7 @@ services: soft: -1 hard: -1 opensearch-oss-2: - image: opensearch:test + image: opensearch-oss:test environment: - node.name=opensearch-oss-2 - cluster.initial_master_nodes=opensearch-oss-1,opensearch-oss-2 diff --git a/distribution/docker/docker-test-entrypoint.sh b/distribution/docker/docker-test-entrypoint.sh index 5171bbcb4cf..38e8bac3cb7 100755 --- a/distribution/docker/docker-test-entrypoint.sh +++ b/distribution/docker/docker-test-entrypoint.sh @@ -1,4 +1,4 @@ #!/bin/bash cd /usr/share/opensearch/bin/ -echo "testnode" > /tmp/password + /usr/local/bin/docker-entrypoint.sh | tee > /usr/share/opensearch/logs/console.log diff --git a/distribution/docker/src/test/java/org/opensearch/docker/test/DockerYmlTestSuiteIT.java b/distribution/docker/src/test/java/org/opensearch/docker/test/DockerYmlTestSuiteIT.java index 291a8e2cd44..af14a6a611d 100644 --- a/distribution/docker/src/test/java/org/opensearch/docker/test/DockerYmlTestSuiteIT.java +++ b/distribution/docker/src/test/java/org/opensearch/docker/test/DockerYmlTestSuiteIT.java @@ -19,27 +19,16 @@ package org.opensearch.docker.test; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.opensearch.OpenSearchException; +import org.junit.Before; import org.opensearch.client.Request; -import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; public class DockerYmlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { - private static final String USER = "rest_user"; - private static final String PASS = "test-password"; - private static final String KEYSTORE_PASS = "testnode"; - public DockerYmlTestSuiteIT(ClientYamlTestCandidate testCandidate) { super(testCandidate); } @@ -83,25 +72,6 @@ public class DockerYmlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { client().performRequest(health); } - static Path keyStore; - - @BeforeClass - public static void getKeyStore() { - try { - keyStore = PathUtils.get(DockerYmlTestSuiteIT.class.getResource("/testnode.jks").toURI()); - } catch (URISyntaxException e) { - throw new OpenSearchException("exception while reading the store", e); - } - if (Files.exists(keyStore) == false) { - throw new IllegalStateException("Keystore file [" + keyStore + "] does not exist."); - } - } - - @AfterClass - public static void clearKeyStore() { - keyStore = null; - } - @Override protected Settings restClientSettings() { return super.restClientSettings(); diff --git a/qa/remote-clusters/build.gradle b/qa/remote-clusters/build.gradle index fb052b44bc7..5fb763e0603 100644 --- a/qa/remote-clusters/build.gradle +++ b/qa/remote-clusters/build.gradle @@ -31,15 +31,11 @@ dependencies { testImplementation project(':client:rest-high-level') } -/** - * TODO OpenSearch - this task is currently broken and temporarily commented out to - * unblock the pre-commit. - * Issue URL - https://github.com/opensearch-project/OpenSearch/issues/400 opensearch_distributions { docker { type = 'docker' architecture = Architecture.current() - version = VersionProperties.getElasticsearch() + version = VersionProperties.getOpenSearch() failIfUnavailable = false // This ensures we skip this testing if Docker is unavailable } } @@ -49,23 +45,19 @@ preProcessFixture { doLast { // tests expect to have an empty repo project.delete( - "${buildDir}/repo", "${buildDir}/oss-repo" ) createAndSetWritable( - "${buildDir}/repo", "${buildDir}/oss-repo", - "${buildDir}/logs/default-1", - "${buildDir}/logs/default-2", "${buildDir}/logs/oss-1", "${buildDir}/logs/oss-2" ) } } -*/ + dockerCompose { tcpPortsToIgnoreWhenWaiting = [9600, 9601] - useComposeFiles = ['docker-compose-oss.yml'] + useComposeFiles = ['docker-compose.yml'] } def createAndSetWritable(Object... locations) { diff --git a/qa/remote-clusters/docker-compose-oss.yml b/qa/remote-clusters/docker-compose-oss.yml deleted file mode 100644 index e484a5758f6..00000000000 --- a/qa/remote-clusters/docker-compose-oss.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Only used for testing the docker images -version: '3.7' -services: - opensearch-oss-1: - image: opensearch:test - environment: - - node.name=opensearch-oss-1 - - cluster.initial_master_nodes=opensearch-oss-1 - - cluster.name=opensearch-oss-1 - - bootstrap.memory_lock=true - - network.publish_host=127.0.0.1 - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - - path.repo=/tmp/opensearch-repo - - node.attr.testattr=test - - cluster.routing.allocation.disk.watermark.low=1b - - cluster.routing.allocation.disk.watermark.high=1b - - cluster.routing.allocation.disk.watermark.flood_stage=1b - - node.store.allow_mmap=false - volumes: - - ./build/oss-repo:/tmp/opensearch-repo - - ./build/logs/oss-1:/usr/share/opensearch/logs - ports: - - "9200" - - "9300" - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - healthcheck: - start_period: 15s - test: ["CMD", "curl", "-f", "http://localhost:9200"] - interval: 10s - timeout: 2s - retries: 5 - opensearch-oss-2: - image: opensearch:test - environment: - - node.name=opensearch-oss-2 - - cluster.initial_master_nodes=opensearch-oss-2 - - cluster.name=opensearch-oss-2 - - bootstrap.memory_lock=true - - network.publish_host=127.0.0.1 - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - - path.repo=/tmp/opensearch-repo - - node.attr.testattr=test - - cluster.routing.allocation.disk.watermark.low=1b - - cluster.routing.allocation.disk.watermark.high=1b - - cluster.routing.allocation.disk.watermark.flood_stage=1b - - node.store.allow_mmap=false - volumes: - - ./build/oss-repo:/tmp/opensearch-repo - - ./build/logs/oss-2:/usr/share/opensearch/logs - ports: - - "9200" - - "9300" - ulimits: - memlock: - soft: -1 - hard: -1 - healthcheck: - start_period: 15s - test: ["CMD", "curl", "-f", "http://localhost:9200"] - interval: 10s - timeout: 2s - retries: 5 - haproxy: - image: haproxy:2.1.2 - ports: - - "9600" - volumes: - - ./haproxy-oss.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro diff --git a/qa/remote-clusters/docker-compose.yml b/qa/remote-clusters/docker-compose.yml index 2076a0c5e5a..66136bd8e2d 100644 --- a/qa/remote-clusters/docker-compose.yml +++ b/qa/remote-clusters/docker-compose.yml @@ -1,15 +1,15 @@ # Only used for testing the docker images version: '3.7' services: - opensearch-default-1: - image: opensearch:test + opensearch-oss-1: + image: opensearch-oss:test environment: - - node.name=opensearch-default-1 - - cluster.initial_master_nodes=opensearch-default-1 - - cluster.name=opensearch-default-1 + - node.name=opensearch-oss-1 + - cluster.initial_master_nodes=opensearch-oss-1 + - cluster.name=opensearch-oss-1 - bootstrap.memory_lock=true - network.publish_host=127.0.0.1 - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - path.repo=/tmp/opensearch-repo - node.attr.testattr=test - cluster.routing.allocation.disk.watermark.low=1b @@ -17,10 +17,8 @@ services: - cluster.routing.allocation.disk.watermark.flood_stage=1b - node.store.allow_mmap=false volumes: - - ./build/repo:/tmp/opensearch-repo - - ./build/certs/testnode.jks:/usr/share/opensearch/config/testnode.jks - - ./build/logs/default-1:/usr/share/opensearch/logs - - ./docker-test-entrypoint.sh:/docker-test-entrypoint.sh + - ./build/oss-repo:/tmp/opensearch-repo + - ./build/logs/oss-1:/usr/share/opensearch/logs ports: - "9200" - "9300" @@ -31,22 +29,21 @@ services: nofile: soft: 65536 hard: 65536 - entrypoint: /docker-test-entrypoint.sh healthcheck: start_period: 15s - test: ["CMD", "curl", "-f", "-u", "rest_user:test-password", "-k", "https://localhost:9200"] + test: ["CMD", "curl", "-f", "http://localhost:9200"] interval: 10s timeout: 2s retries: 5 - opensearch-default-2: - image: opensearch:test + opensearch-oss-2: + image: opensearch-oss:test environment: - - node.name=opensearch-default-2 - - cluster.initial_master_nodes=opensearch-default-2 - - cluster.name=opensearch-default-2 + - node.name=opensearch-oss-2 + - cluster.initial_master_nodes=opensearch-oss-2 + - cluster.name=opensearch-oss-2 - bootstrap.memory_lock=true - network.publish_host=127.0.0.1 - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - path.repo=/tmp/opensearch-repo - node.attr.testattr=test - cluster.routing.allocation.disk.watermark.low=1b @@ -54,10 +51,8 @@ services: - cluster.routing.allocation.disk.watermark.flood_stage=1b - node.store.allow_mmap=false volumes: - - ./build/repo:/tmp/opensearch-repo - - ./build/certs/testnode.jks:/usr/share/opensearch/config/testnode.jks - - ./build/logs/default-2:/usr/share/opensearch/logs - - ./docker-test-entrypoint.sh:/docker-test-entrypoint.sh + - ./build/oss-repo:/tmp/opensearch-repo + - ./build/logs/oss-2:/usr/share/opensearch/logs ports: - "9200" - "9300" @@ -65,13 +60,9 @@ services: memlock: soft: -1 hard: -1 - nofile: - soft: 65536 - hard: 65536 - entrypoint: /docker-test-entrypoint.sh healthcheck: start_period: 15s - test: ["CMD", "curl", "-f", "-u", "rest_user:test-password", "-k", "https://localhost:9200"] + test: ["CMD", "curl", "-f", "http://localhost:9200"] interval: 10s timeout: 2s retries: 5 @@ -80,4 +71,4 @@ services: ports: - "9600" volumes: - - ./haproxy-default.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./haproxy-oss.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro diff --git a/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java b/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java index a53d0a5356d..0c9a65b6725 100644 --- a/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java +++ b/qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java @@ -19,23 +19,17 @@ package org.opensearch.cluster.remote.test; import org.apache.http.HttpHost; -import org.opensearch.OpenSearchException; +import org.junit.AfterClass; +import org.junit.Before; import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestClient; import org.opensearch.client.RestHighLevelClient; -import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.core.internal.io.IOUtils; import org.opensearch.test.rest.OpenSearchRestTestCase; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Collections; public abstract class AbstractMultiClusterRemoteTestCase extends OpenSearchRestTestCase { @@ -102,25 +96,6 @@ public abstract class AbstractMultiClusterRemoteTestCase extends OpenSearchRestT return new HighLevelClient(buildClient(restAdminSettings(), new HttpHost[]{httpHost})); } - static Path keyStore; - - @BeforeClass - public static void getKeyStore() { - try { - keyStore = PathUtils.get(AbstractMultiClusterRemoteTestCase.class.getResource("/testnode.jks").toURI()); - } catch (URISyntaxException e) { - throw new OpenSearchException("exception while reading the store", e); - } - if (Files.exists(keyStore) == false) { - throw new IllegalStateException("Keystore file [" + keyStore + "] does not exist."); - } - } - - @AfterClass - public static void clearKeyStore() { - keyStore = null; - } - @Override protected Settings restClientSettings() { return super.restClientSettings(); diff --git a/qa/wildfly/build.gradle b/qa/wildfly/build.gradle index 5c3d6090a27..a8504f68f00 100644 --- a/qa/wildfly/build.gradle +++ b/qa/wildfly/build.gradle @@ -53,15 +53,11 @@ war { archiveFileName = 'example-app.war' } -/** - * TODO OpenSearch - this task is currently broken and temporarily commented out to - * unblock the pre-commit. - * Issue URL - https://github.com/opensearch-project/OpenSearch/issues/400 opensearch_distributions { docker { type = 'docker' architecture = Architecture.current() - version = VersionProperties.getElasticsearch() + version = VersionProperties.getOpenSearch() failIfUnavailable = false // This ensures we skip this testing if Docker is unavailable } } @@ -69,9 +65,9 @@ opensearch_distributions { preProcessFixture { dependsOn war, opensearch_distributions.docker } -*/ + dockerCompose { - useComposeFiles = ['docker-compose-oss.yml'] + useComposeFiles = ['docker-compose.yml'] } tasks.register("integTest", Test) { diff --git a/qa/wildfly/docker-compose-oss.yml b/qa/wildfly/docker-compose-oss.yml deleted file mode 100644 index 4d7f3f57932..00000000000 --- a/qa/wildfly/docker-compose-oss.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: '3.7' -services: - - wildfly: - image: jboss/wildfly:18.0.1.Final - environment: - JAVA_OPTS: -Dopensearch.uri=opensearch:9200 -Djboss.http.port=8080 -Djava.net.preferIPv4Stack=true - volumes: - - ./build/distributions/example-app.war:/opt/jboss/wildfly/standalone/deployments/example-app.war - ports: - - "8080" - healthcheck: - start_period: 5s - test: ["CMD", "grep", "Admin console listening on", "/opt/jboss/wildfly/standalone/log/server.log"] - interval: 2s - timeout: 1s - retries: 5 - - opensearch: - image: opensearch-oss:test - environment: - discovery.type: single-node - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - healthcheck: - start_period: 15s - test: ["CMD", "curl", "-f", "-k", "http://localhost:9200"] - interval: 10s - timeout: 2s - retries: 5 diff --git a/qa/wildfly/docker-compose.yml b/qa/wildfly/docker-compose.yml index 170741b7349..4d7f3f57932 100644 --- a/qa/wildfly/docker-compose.yml +++ b/qa/wildfly/docker-compose.yml @@ -4,7 +4,7 @@ services: wildfly: image: jboss/wildfly:18.0.1.Final environment: - JAVA_OPTS: -Delasticsearch.uri=opensearch:9200 -Djboss.http.port=8080 -Djava.net.preferIPv4Stack=true + JAVA_OPTS: -Dopensearch.uri=opensearch:9200 -Djboss.http.port=8080 -Djava.net.preferIPv4Stack=true volumes: - ./build/distributions/example-app.war:/opt/jboss/wildfly/standalone/deployments/example-app.war ports: @@ -17,7 +17,7 @@ services: retries: 5 opensearch: - image: opensearch:test + image: opensearch-oss:test environment: discovery.type: single-node ulimits: