Remove old ES libraries used in reindex due to CVEs (#1359)
This commit removes old ES libraries version 090 and 176 due to CVE Signed-off-by: Xue Zhou <xuezhou@amazon.com>
This commit is contained in:
Xue Zhou
GitHub
parent
ecac8d3c38
commit
574e42c31b
|
|
@ -92,88 +92,3 @@ forbiddenPatterns {
|
||||||
exclude '**/*.p12'
|
exclude '**/*.p12'
|
||||||
}
|
}
|
||||||
|
|
||||||
// Support for testing reindex-from-remote against old Elasticsearch versions
|
|
||||||
configurations {
|
|
||||||
oldesFixture
|
|
||||||
es2
|
|
||||||
es1
|
|
||||||
es090
|
|
||||||
}
|
|
||||||
|
|
||||||
dependencies {
|
|
||||||
oldesFixture project(':test:fixtures:old-elasticsearch')
|
|
||||||
/* Right now we just test against the latest version of each major we expect
|
|
||||||
* reindex-from-remote to work against. We could randomize the versions but
|
|
||||||
* that doesn't seem worth it at this point. */
|
|
||||||
es2 'org.elasticsearch.distribution.zip:elasticsearch:2.4.5@zip'
|
|
||||||
es1 'org.elasticsearch:elasticsearch:1.7.6@zip'
|
|
||||||
es090 'org.elasticsearch:elasticsearch:0.90.13@zip'
|
|
||||||
}
|
|
||||||
|
|
||||||
jdks {
|
|
||||||
legacy {
|
|
||||||
vendor = 'adoptopenjdk'
|
|
||||||
version = '8u242+b08'
|
|
||||||
platform = OS.current().name().toLowerCase()
|
|
||||||
architecture = Architecture.current().name().toLowerCase()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
|
|
||||||
logger.warn("Disabling reindex-from-old tests because we can't get the pid file on windows")
|
|
||||||
javaRestTest {
|
|
||||||
systemProperty "tests.fromOld", "false"
|
|
||||||
}
|
|
||||||
} else if (rootProject.rootDir.toString().contains(" ")) {
|
|
||||||
logger.warn("Disabling reindex-from-old tests because Elasticsearch 1.7 won't start with spaces in the path")
|
|
||||||
javaRestTest {
|
|
||||||
systemProperty "tests.fromOld", "false"
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
/* Set up tasks to unzip and run the old versions of ES before running the
|
|
||||||
* integration tests. */
|
|
||||||
def versions = ['2', '1', '090']
|
|
||||||
if (Os.isFamily(Os.FAMILY_MAC)) {
|
|
||||||
// 0.90 fails sometimes on mac, given that it is so old, let us disable it
|
|
||||||
// see: https://github.com/elastic/elasticsearch/issues/51202
|
|
||||||
versions = ['2', '1']
|
|
||||||
}
|
|
||||||
for (String version : versions) {
|
|
||||||
Task unzip = task("unzipEs${version}", type: Sync) {
|
|
||||||
Configuration oldEsDependency = configurations['es' + version]
|
|
||||||
dependsOn oldEsDependency
|
|
||||||
/* Use a closure here to delay resolution of the dependency until we need
|
|
||||||
* it */
|
|
||||||
from {
|
|
||||||
oldEsDependency.collect { zipTree(it) }
|
|
||||||
}
|
|
||||||
into temporaryDir
|
|
||||||
}
|
|
||||||
|
|
||||||
Task fixture = task("oldEs${version}Fixture", type: org.opensearch.gradle.test.AntFixture) {
|
|
||||||
dependsOn project.configurations.oldesFixture, jdks.legacy
|
|
||||||
dependsOn unzip
|
|
||||||
executable = "${BuildParams.runtimeJavaHome}/bin/java"
|
|
||||||
env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}"
|
|
||||||
env 'JAVA_HOME', jdks.legacy.javaHomePath
|
|
||||||
args 'oldes.OldElasticsearch',
|
|
||||||
baseDir,
|
|
||||||
unzip.temporaryDir,
|
|
||||||
version == '090'
|
|
||||||
waitCondition = { fixture, ant ->
|
|
||||||
// the fixture writes the ports file when Elasticsearch's HTTP service
|
|
||||||
// is ready, so we can just wait for the file to exist
|
|
||||||
return fixture.portsFile.exists()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
javaRestTest {
|
|
||||||
dependsOn fixture
|
|
||||||
systemProperty "tests.fromOld", "true"
|
|
||||||
/* Use a closure on the string to delay evaluation until right before we
|
|
||||||
* run the integration tests so that we can be sure that the file is
|
|
||||||
* ready. */
|
|
||||||
nonInputProperties.systemProperty "es${version}.port", "${-> fixture.addressAndPort}"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,151 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-License-Identifier: Apache-2.0
|
|
||||||
*
|
|
||||||
* The OpenSearch Contributors require contributions made to
|
|
||||||
* this file be licensed under the Apache-2.0 license or a
|
|
||||||
* compatible open source license.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Licensed to Elasticsearch under one or more contributor
|
|
||||||
* license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright
|
|
||||||
* ownership. Elasticsearch licenses this file to you under
|
|
||||||
* the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
* not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing,
|
|
||||||
* software distributed under the License is distributed on an
|
|
||||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
||||||
* KIND, either express or implied. See the License for the
|
|
||||||
* specific language governing permissions and limitations
|
|
||||||
* under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Modifications Copyright OpenSearch Contributors. See
|
|
||||||
* GitHub history for details.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.opensearch.index.reindex.remote;
|
|
||||||
|
|
||||||
import org.apache.http.HttpHost;
|
|
||||||
import org.apache.http.util.EntityUtils;
|
|
||||||
import org.apache.lucene.util.Constants;
|
|
||||||
import org.opensearch.client.Request;
|
|
||||||
import org.opensearch.client.Response;
|
|
||||||
import org.opensearch.client.RestClient;
|
|
||||||
import org.opensearch.common.Booleans;
|
|
||||||
import org.opensearch.test.rest.OpenSearchRestTestCase;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import static org.hamcrest.Matchers.containsString;
|
|
||||||
|
|
||||||
public class ReindexFromOldRemoteIT extends OpenSearchRestTestCase {
|
|
||||||
/**
|
|
||||||
* Number of documents to test when reindexing from an old version.
|
|
||||||
*/
|
|
||||||
private static final int DOCS = 5;
|
|
||||||
|
|
||||||
private void oldEsTestCase(String portPropertyName, String requestsPerSecond) throws IOException {
|
|
||||||
boolean enabled = Booleans.parseBoolean(System.getProperty("tests.fromOld"));
|
|
||||||
assumeTrue("test is disabled, probably because this is windows", enabled);
|
|
||||||
|
|
||||||
int oldEsPort = Integer.parseInt(System.getProperty(portPropertyName));
|
|
||||||
try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) {
|
|
||||||
try {
|
|
||||||
Request createIndex = new Request("PUT", "/test");
|
|
||||||
createIndex.setJsonEntity("{\"settings\":{\"number_of_shards\": 1}}");
|
|
||||||
oldEs.performRequest(createIndex);
|
|
||||||
|
|
||||||
for (int i = 0; i < DOCS; i++) {
|
|
||||||
Request doc = new Request("PUT", "/test/doc/testdoc" + i);
|
|
||||||
doc.addParameter("refresh", "true");
|
|
||||||
doc.setJsonEntity("{\"test\":\"test\"}");
|
|
||||||
oldEs.performRequest(doc);
|
|
||||||
}
|
|
||||||
|
|
||||||
Request reindex = new Request("POST", "/_reindex");
|
|
||||||
if (randomBoolean()) {
|
|
||||||
// Reindex using the external version_type
|
|
||||||
reindex.setJsonEntity(
|
|
||||||
"{\n"
|
|
||||||
+ " \"source\":{\n"
|
|
||||||
+ " \"index\": \"test\",\n"
|
|
||||||
+ " \"size\": 1,\n"
|
|
||||||
+ " \"remote\": {\n"
|
|
||||||
+ " \"host\": \"http://127.0.0.1:" + oldEsPort + "\"\n"
|
|
||||||
+ " }\n"
|
|
||||||
+ " },\n"
|
|
||||||
+ " \"dest\": {\n"
|
|
||||||
+ " \"index\": \"test\",\n"
|
|
||||||
+ " \"version_type\": \"external\"\n"
|
|
||||||
+ " }\n"
|
|
||||||
+ "}");
|
|
||||||
} else {
|
|
||||||
// Reindex using the default internal version_type
|
|
||||||
reindex.setJsonEntity(
|
|
||||||
"{\n"
|
|
||||||
+ " \"source\":{\n"
|
|
||||||
+ " \"index\": \"test\",\n"
|
|
||||||
+ " \"size\": 1,\n"
|
|
||||||
+ " \"remote\": {\n"
|
|
||||||
+ " \"host\": \"http://127.0.0.1:" + oldEsPort + "\"\n"
|
|
||||||
+ " }\n"
|
|
||||||
+ " },\n"
|
|
||||||
+ " \"dest\": {\n"
|
|
||||||
+ " \"index\": \"test\"\n"
|
|
||||||
+ " }\n"
|
|
||||||
+ "}");
|
|
||||||
}
|
|
||||||
reindex.addParameter("refresh", "true");
|
|
||||||
reindex.addParameter("pretty", "true");
|
|
||||||
if (requestsPerSecond != null) {
|
|
||||||
reindex.addParameter("requests_per_second", requestsPerSecond);
|
|
||||||
}
|
|
||||||
client().performRequest(reindex);
|
|
||||||
|
|
||||||
Request search = new Request("POST", "/test/_search");
|
|
||||||
search.addParameter("pretty", "true");
|
|
||||||
Response response = client().performRequest(search);
|
|
||||||
String result = EntityUtils.toString(response.getEntity());
|
|
||||||
for (int i = 0; i < DOCS; i++) {
|
|
||||||
assertThat(result, containsString("\"_id\" : \"testdoc" + i + "\""));
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
oldEs.performRequest(new Request("DELETE", "/test"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testEs2() throws IOException {
|
|
||||||
oldEsTestCase("es2.port", null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testEs1() throws IOException {
|
|
||||||
oldEsTestCase("es1.port", null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testEs090() throws IOException {
|
|
||||||
assumeFalse("No longer works on Mac", Constants.MAC_OS_X);
|
|
||||||
oldEsTestCase("es090.port", null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testEs2WithFunnyThrottle() throws IOException {
|
|
||||||
oldEsTestCase("es2.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testEs1WithFunnyThrottle() throws IOException {
|
|
||||||
oldEsTestCase("es1.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testEs090WithFunnyThrottle() throws IOException {
|
|
||||||
assumeFalse("No longer works on Mac", Constants.MAC_OS_X);
|
|
||||||
oldEsTestCase("es090.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue