honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup `default` flavor stragglers from docker distributions. (#481) 

This commit removes the references for the `default` docker distributions which were originally part of the different flavors of distributions. This also fixes some of the failing docker compose tests under `qa`.

Signed-off-by: Rabi Panda <adnapibar@gmail.com>
This commit is contained in:
Rabi Panda 2021-04-09 10:24:41 -07:00 committed by GitHub
parent cea21610bb
commit 7949f92bec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 34 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
distribution/build.gradle
View File

@ -58,17 +58,10 @@ tasks.register("generateDependenciesReport", ConcatFilesTask) {
// integ test zip only uses server, so a different notice file is needed there
def buildServerNoticeTaskProvider = tasks.register("buildServerNotice", NoticeTask)
// other distributions include notices from modules as well, which are added below later
def buildDefaultNoticeTaskProvider = tasks.register("buildDefaultNotice", NoticeTask) {
licensesDir new File(project(':distribution').projectDir, 'licenses')
}
def buildOssNoticeTaskProvider = tasks.register("buildOssNotice", NoticeTask) {
licensesDir new File(project(':distribution').projectDir, 'licenses')
}
def buildDefaultNoJdkNoticeTaskProvider = tasks.register("buildDefaultNoJdkNotice", NoticeTask)
def buildOssNoJdkNoticeTaskProvider = tasks.register("buildOssNoJdkNotice", NoticeTask)
// The :server and :libs projects belong to all distributions
@ -85,7 +78,6 @@ tasks.withType(NoticeTask).configureEach {
* Modules *
*****************************************************************************/
String ossOutputs = 'build/outputs/oss'
String defaultOutputs = 'build/outputs/default'
String systemdOutputs = 'build/outputs/systemd'
String transportOutputs = 'build/outputs/transport-only'
String externalTestOutputs = 'build/outputs/external-test'
@ -94,11 +86,6 @@ def processOssOutputsTaskProvider = tasks.register("processOssOutputs", Sync) {
into ossOutputs
}
def processDefaultOutputsTaskProvider = tasks.register("processDefaultOutputs", Sync) {
into defaultOutputs
from processOssOutputsTaskProvider
}
def processSystemdOutputsTaskProvider = tasks.register("processSystemdOutputs", Sync) {
into systemdOutputs
}
@ -126,18 +113,6 @@ tasks.register("buildOssConfig") {
dependsOn "processOssOutputs"
outputs.dir "${ossOutputs}/config"
}
def buildDefaultModulesTaskProvider = tasks.register("buildDefaultModules") {
dependsOn processDefaultOutputsTaskProvider
outputs.dir "${defaultOutputs}/modules"
}
tasks.register("buildDefaultBin") {
dependsOn processDefaultOutputsTaskProvider
outputs.dir "${defaultOutputs}/bin"
}
def buildDefaultConfigTaskProvider = tasks.register("buildDefaultConfig") {
dependsOn processOssOutputsTaskProvider
outputs.dir "${defaultOutputs}/config"
}
def buildSystemdModuleTaskProvider = tasks.register("buildSystemdModule") {
dependsOn "processSystemdOutputs"
outputs.dir "${systemdOutputs}/modules"
@ -192,12 +167,6 @@ def buildOssLog4jConfigTaskProvider = tasks.register("buildOssLog4jConfig") {
ext.log4jFile = file("${ossOutputs}/log4j2.properties")
outputs.file log4jFile
}
def buildDefaultLog4jConfigTaskProvider = tasks.register("buildDefaultLog4jConfig") {
dependsOn processDefaultOutputsTaskProvider
ext.contents = []
ext.log4jFile = file("${defaultOutputs}/log4j2.properties")
outputs.file log4jFile
}
Closure writeLog4jProperties = {
String mainLog4jProperties = file('src/config/log4j2.properties').getText('UTF-8')
@ -210,10 +179,6 @@ buildOssLog4jConfigTaskProvider.configure {
doLast(writeLog4jProperties)
}
buildDefaultLog4jConfigTaskProvider.configure {
doLast(writeLog4jProperties)
}
// copy log4j2.properties from modules that have it
void copyLog4jProperties(TaskProvider buildTask, Project module) {
buildTask.configure {
@ -246,10 +211,6 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each {
}
File licenses = new File(module.projectDir, 'licenses')
if (licenses.exists()) {
buildDefaultNoticeTaskProvider.configure {
licensesDir licenses
source module.file('src/main/java')
}
buildOssNotice.configure {
licensesDir licenses
source module.file('src/main/java')
@ -262,7 +223,6 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each {
}
copyLog4jProperties(buildOssLog4jConfigTaskProvider, module)
copyLog4jProperties(buildDefaultLog4jConfigTaskProvider, module)
restTestExpansions['expected.modules.count'] += 1
}

3
distribution/docker/build.gradle
View File

@ -182,8 +182,7 @@ void addBuildDockerImage(Architecture architecture, DockerBase base) {
String version = VersionProperties.getOpenSearch()
tags = [
//TODO remove the tag and replace with OpenSearch docker tag
"docker.elastic.co/elasticsearch/elasticsearch-oss:${version}",
"docker.opensearch.org/opensearch/opensearch-oss:${version}",
"opensearch-oss:test"
]
}

4
distribution/docker/docker-compose.yml
View File

@ -2,7 +2,7 @@
version: '3'
services:
opensearch-oss-1:
image: opensearch:test
image: opensearch-oss:test
environment:
- node.name=opensearch-oss-1
- cluster.initial_master_nodes=opensearch-oss-1,opensearch-oss-2
@ -26,7 +26,7 @@ services:
soft: -1
hard: -1
opensearch-oss-2:
image: opensearch:test
image: opensearch-oss:test
environment:
- node.name=opensearch-oss-2
- cluster.initial_master_nodes=opensearch-oss-1,opensearch-oss-2

2
distribution/docker/docker-test-entrypoint.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
cd /usr/share/opensearch/bin/
echo "testnode" > /tmp/password
/usr/local/bin/docker-entrypoint.sh | tee > /usr/share/opensearch/logs/console.log

32
distribution/docker/src/test/java/org/opensearch/docker/test/DockerYmlTestSuiteIT.java
View File

@ -19,27 +19,16 @@
package org.opensearch.docker.test;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.opensearch.OpenSearchException;
import org.junit.Before;
import org.opensearch.client.Request;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
public class DockerYmlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {
private static final String USER = "rest_user";
private static final String PASS = "test-password";
private static final String KEYSTORE_PASS = "testnode";
public DockerYmlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ -83,25 +72,6 @@ public class DockerYmlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {
client().performRequest(health);
}
static Path keyStore;
@BeforeClass
public static void getKeyStore() {
try {
keyStore = PathUtils.get(DockerYmlTestSuiteIT.class.getResource("/testnode.jks").toURI());
} catch (URISyntaxException e) {
throw new OpenSearchException("exception while reading the store", e);
}
if (Files.exists(keyStore) == false) {
throw new IllegalStateException("Keystore file [" + keyStore + "] does not exist.");
}
}
@AfterClass
public static void clearKeyStore() {
keyStore = null;
}
@Override
protected Settings restClientSettings() {
return super.restClientSettings();

14
qa/remote-clusters/build.gradle
View File

@ -31,15 +31,11 @@ dependencies {
testImplementation project(':client:rest-high-level')
}
/**
* TODO OpenSearch - this task is currently broken and temporarily commented out to
* unblock the pre-commit.
* Issue URL - https://github.com/opensearch-project/OpenSearch/issues/400
opensearch_distributions {
docker {
type = 'docker'
architecture = Architecture.current()
version = VersionProperties.getElasticsearch()
version = VersionProperties.getOpenSearch()
failIfUnavailable = false // This ensures we skip this testing if Docker is unavailable
}
}
@ -49,23 +45,19 @@ preProcessFixture {
doLast {
// tests expect to have an empty repo
project.delete(
"${buildDir}/repo",
"${buildDir}/oss-repo"
)
createAndSetWritable(
"${buildDir}/repo",
"${buildDir}/oss-repo",
"${buildDir}/logs/default-1",
"${buildDir}/logs/default-2",
"${buildDir}/logs/oss-1",
"${buildDir}/logs/oss-2"
)
}
}
*/
dockerCompose {
tcpPortsToIgnoreWhenWaiting = [9600, 9601]
useComposeFiles = ['docker-compose-oss.yml']
useComposeFiles = ['docker-compose.yml']
}
def createAndSetWritable(Object... locations) {

74
qa/remote-clusters/docker-compose-oss.yml
View File

@ -1,74 +0,0 @@
# Only used for testing the docker images
version: '3.7'
services:
opensearch-oss-1:
image: opensearch:test
environment:
- node.name=opensearch-oss-1
- cluster.initial_master_nodes=opensearch-oss-1
- cluster.name=opensearch-oss-1
- bootstrap.memory_lock=true
- network.publish_host=127.0.0.1
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
- path.repo=/tmp/opensearch-repo
- node.attr.testattr=test
- cluster.routing.allocation.disk.watermark.low=1b
- cluster.routing.allocation.disk.watermark.high=1b
- cluster.routing.allocation.disk.watermark.flood_stage=1b
- node.store.allow_mmap=false
volumes:
- ./build/oss-repo:/tmp/opensearch-repo
- ./build/logs/oss-1:/usr/share/opensearch/logs
ports:
- "9200"
- "9300"
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
healthcheck:
start_period: 15s
test: ["CMD", "curl", "-f", "http://localhost:9200"]
interval: 10s
timeout: 2s
retries: 5
opensearch-oss-2:
image: opensearch:test
environment:
- node.name=opensearch-oss-2
- cluster.initial_master_nodes=opensearch-oss-2
- cluster.name=opensearch-oss-2
- bootstrap.memory_lock=true
- network.publish_host=127.0.0.1
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
- path.repo=/tmp/opensearch-repo
- node.attr.testattr=test
- cluster.routing.allocation.disk.watermark.low=1b
- cluster.routing.allocation.disk.watermark.high=1b
- cluster.routing.allocation.disk.watermark.flood_stage=1b
- node.store.allow_mmap=false
volumes:
- ./build/oss-repo:/tmp/opensearch-repo
- ./build/logs/oss-2:/usr/share/opensearch/logs
ports:
- "9200"
- "9300"
ulimits:
memlock:
soft: -1
hard: -1
healthcheck:
start_period: 15s
test: ["CMD", "curl", "-f", "http://localhost:9200"]
interval: 10s
timeout: 2s
retries: 5
haproxy:
image: haproxy:2.1.2
ports:
- "9600"
volumes:
- ./haproxy-oss.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro

47
qa/remote-clusters/docker-compose.yml
View File

@ -1,15 +1,15 @@
# Only used for testing the docker images
version: '3.7'
services:
opensearch-default-1:
image: opensearch:test
opensearch-oss-1:
image: opensearch-oss:test
environment:
- node.name=opensearch-default-1
- cluster.initial_master_nodes=opensearch-default-1
- cluster.name=opensearch-default-1
- node.name=opensearch-oss-1
- cluster.initial_master_nodes=opensearch-oss-1
- cluster.name=opensearch-oss-1
- bootstrap.memory_lock=true
- network.publish_host=127.0.0.1
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
- path.repo=/tmp/opensearch-repo
- node.attr.testattr=test
- cluster.routing.allocation.disk.watermark.low=1b
@ -17,10 +17,8 @@ services:
- cluster.routing.allocation.disk.watermark.flood_stage=1b
- node.store.allow_mmap=false
volumes:
- ./build/repo:/tmp/opensearch-repo
- ./build/certs/testnode.jks:/usr/share/opensearch/config/testnode.jks
- ./build/logs/default-1:/usr/share/opensearch/logs
- ./docker-test-entrypoint.sh:/docker-test-entrypoint.sh
- ./build/oss-repo:/tmp/opensearch-repo
- ./build/logs/oss-1:/usr/share/opensearch/logs
ports:
- "9200"
- "9300"
@ -31,22 +29,21 @@ services:
nofile:
soft: 65536
hard: 65536
entrypoint: /docker-test-entrypoint.sh
healthcheck:
start_period: 15s
test: ["CMD", "curl", "-f", "-u", "rest_user:test-password", "-k", "https://localhost:9200"]
test: ["CMD", "curl", "-f", "http://localhost:9200"]
interval: 10s
timeout: 2s
retries: 5
opensearch-default-2:
image: opensearch:test
opensearch-oss-2:
image: opensearch-oss:test
environment:
- node.name=opensearch-default-2
- cluster.initial_master_nodes=opensearch-default-2
- cluster.name=opensearch-default-2
- node.name=opensearch-oss-2
- cluster.initial_master_nodes=opensearch-oss-2
- cluster.name=opensearch-oss-2
- bootstrap.memory_lock=true
- network.publish_host=127.0.0.1
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
- path.repo=/tmp/opensearch-repo
- node.attr.testattr=test
- cluster.routing.allocation.disk.watermark.low=1b
@ -54,10 +51,8 @@ services:
- cluster.routing.allocation.disk.watermark.flood_stage=1b
- node.store.allow_mmap=false
volumes:
- ./build/repo:/tmp/opensearch-repo
- ./build/certs/testnode.jks:/usr/share/opensearch/config/testnode.jks
- ./build/logs/default-2:/usr/share/opensearch/logs
- ./docker-test-entrypoint.sh:/docker-test-entrypoint.sh
- ./build/oss-repo:/tmp/opensearch-repo
- ./build/logs/oss-2:/usr/share/opensearch/logs
ports:
- "9200"
- "9300"
@ -65,13 +60,9 @@ services:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
entrypoint: /docker-test-entrypoint.sh
healthcheck:
start_period: 15s
test: ["CMD", "curl", "-f", "-u", "rest_user:test-password", "-k", "https://localhost:9200"]
test: ["CMD", "curl", "-f", "http://localhost:9200"]
interval: 10s
timeout: 2s
retries: 5
@ -80,4 +71,4 @@ services:
ports:
- "9600"
volumes:
- ./haproxy-default.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
- ./haproxy-oss.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro

29
qa/remote-clusters/src/test/java/org/opensearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java
View File

@ -19,23 +19,17 @@
package org.opensearch.cluster.remote.test;
import org.apache.http.HttpHost;
import org.opensearch.OpenSearchException;
import org.junit.AfterClass;
import org.junit.Before;
import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
import org.opensearch.client.RequestOptions;
import org.opensearch.client.RestClient;
import org.opensearch.client.RestHighLevelClient;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.internal.io.IOUtils;
import org.opensearch.test.rest.OpenSearchRestTestCase;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
public abstract class AbstractMultiClusterRemoteTestCase extends OpenSearchRestTestCase {
@ -102,25 +96,6 @@ public abstract class AbstractMultiClusterRemoteTestCase extends OpenSearchRestT
return new HighLevelClient(buildClient(restAdminSettings(), new HttpHost[]{httpHost}));
}
static Path keyStore;
@BeforeClass
public static void getKeyStore() {
try {
keyStore = PathUtils.get(AbstractMultiClusterRemoteTestCase.class.getResource("/testnode.jks").toURI());
} catch (URISyntaxException e) {
throw new OpenSearchException("exception while reading the store", e);
}
if (Files.exists(keyStore) == false) {
throw new IllegalStateException("Keystore file [" + keyStore + "] does not exist.");
}
}
@AfterClass
public static void clearKeyStore() {
keyStore = null;
}
@Override
protected Settings restClientSettings() {
return super.restClientSettings();

10
qa/wildfly/build.gradle
View File

@ -53,15 +53,11 @@ war {
archiveFileName = 'example-app.war'
}
/**
* TODO OpenSearch - this task is currently broken and temporarily commented out to
* unblock the pre-commit.
* Issue URL - https://github.com/opensearch-project/OpenSearch/issues/400
opensearch_distributions {
docker {
type = 'docker'
architecture = Architecture.current()
version = VersionProperties.getElasticsearch()
version = VersionProperties.getOpenSearch()
failIfUnavailable = false // This ensures we skip this testing if Docker is unavailable
}
}
@ -69,9 +65,9 @@ opensearch_distributions {
preProcessFixture {
dependsOn war, opensearch_distributions.docker
}
*/
dockerCompose {
useComposeFiles = ['docker-compose-oss.yml']
useComposeFiles = ['docker-compose.yml']
}
tasks.register("integTest", Test) {

35
qa/wildfly/docker-compose-oss.yml
View File

@ -1,35 +0,0 @@
version: '3.7'
services:
wildfly:
image: jboss/wildfly:18.0.1.Final
environment:
JAVA_OPTS: -Dopensearch.uri=opensearch:9200 -Djboss.http.port=8080 -Djava.net.preferIPv4Stack=true
volumes:
- ./build/distributions/example-app.war:/opt/jboss/wildfly/standalone/deployments/example-app.war
ports:
- "8080"
healthcheck:
start_period: 5s
test: ["CMD", "grep", "Admin console listening on", "/opt/jboss/wildfly/standalone/log/server.log"]
interval: 2s
timeout: 1s
retries: 5
opensearch:
image: opensearch-oss:test
environment:
discovery.type: single-node
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
healthcheck:
start_period: 15s
test: ["CMD", "curl", "-f", "-k", "http://localhost:9200"]
interval: 10s
timeout: 2s
retries: 5

4
qa/wildfly/docker-compose.yml
View File

@ -4,7 +4,7 @@ services:
wildfly:
image: jboss/wildfly:18.0.1.Final
environment:
JAVA_OPTS: -Delasticsearch.uri=opensearch:9200 -Djboss.http.port=8080 -Djava.net.preferIPv4Stack=true
JAVA_OPTS: -Dopensearch.uri=opensearch:9200 -Djboss.http.port=8080 -Djava.net.preferIPv4Stack=true
volumes:
- ./build/distributions/example-app.war:/opt/jboss/wildfly/standalone/deployments/example-app.war
ports:
@ -17,7 +17,7 @@ services:
retries: 5
opensearch:
image: opensearch:test
image: opensearch-oss:test
environment:
discovery.type: single-node
ulimits: