Remove old ES libraries used in reindex due to CVEs (#1359)
This commit removes old ES libraries version 090 and 176 due to CVE Signed-off-by: Xue Zhou <xuezhou@amazon.com>
This commit is contained in:
Xue Zhou
GitHub
parent
ecac8d3c38
commit
574e42c31b
|
|
@ -92,88 +92,3 @@ forbiddenPatterns {
|
|||
exclude '**/*.p12'
|
||||
}
|
||||
|
||||
// Support for testing reindex-from-remote against old Elasticsearch versions
|
||||
configurations {
|
||||
oldesFixture
|
||||
es2
|
||||
es1
|
||||
es090
|
||||
}
|
||||
|
||||
dependencies {
|
||||
oldesFixture project(':test:fixtures:old-elasticsearch')
|
||||
/* Right now we just test against the latest version of each major we expect
|
||||
* reindex-from-remote to work against. We could randomize the versions but
|
||||
* that doesn't seem worth it at this point. */
|
||||
es2 'org.elasticsearch.distribution.zip:elasticsearch:2.4.5@zip'
|
||||
es1 'org.elasticsearch:elasticsearch:1.7.6@zip'
|
||||
es090 'org.elasticsearch:elasticsearch:0.90.13@zip'
|
||||
}
|
||||
|
||||
jdks {
|
||||
legacy {
|
||||
vendor = 'adoptopenjdk'
|
||||
version = '8u242+b08'
|
||||
platform = OS.current().name().toLowerCase()
|
||||
architecture = Architecture.current().name().toLowerCase()
|
||||
}
|
||||
}
|
||||
|
||||
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
|
||||
logger.warn("Disabling reindex-from-old tests because we can't get the pid file on windows")
|
||||
javaRestTest {
|
||||
systemProperty "tests.fromOld", "false"
|
||||
}
|
||||
} else if (rootProject.rootDir.toString().contains(" ")) {
|
||||
logger.warn("Disabling reindex-from-old tests because Elasticsearch 1.7 won't start with spaces in the path")
|
||||
javaRestTest {
|
||||
systemProperty "tests.fromOld", "false"
|
||||
}
|
||||
} else {
|
||||
/* Set up tasks to unzip and run the old versions of ES before running the
|
||||
* integration tests. */
|
||||
def versions = ['2', '1', '090']
|
||||
if (Os.isFamily(Os.FAMILY_MAC)) {
|
||||
// 0.90 fails sometimes on mac, given that it is so old, let us disable it
|
||||
// see: https://github.com/elastic/elasticsearch/issues/51202
|
||||
versions = ['2', '1']
|
||||
}
|
||||
for (String version : versions) {
|
||||
Task unzip = task("unzipEs${version}", type: Sync) {
|
||||
Configuration oldEsDependency = configurations['es' + version]
|
||||
dependsOn oldEsDependency
|
||||
/* Use a closure here to delay resolution of the dependency until we need
|
||||
* it */
|
||||
from {
|
||||
oldEsDependency.collect { zipTree(it) }
|
||||
}
|
||||
into temporaryDir
|
||||
}
|
||||
|
||||
Task fixture = task("oldEs${version}Fixture", type: org.opensearch.gradle.test.AntFixture) {
|
||||
dependsOn project.configurations.oldesFixture, jdks.legacy
|
||||
dependsOn unzip
|
||||
executable = "${BuildParams.runtimeJavaHome}/bin/java"
|
||||
env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}"
|
||||
env 'JAVA_HOME', jdks.legacy.javaHomePath
|
||||
args 'oldes.OldElasticsearch',
|
||||
baseDir,
|
||||
unzip.temporaryDir,
|
||||
version == '090'
|
||||
waitCondition = { fixture, ant ->
|
||||
// the fixture writes the ports file when Elasticsearch's HTTP service
|
||||
// is ready, so we can just wait for the file to exist
|
||||
return fixture.portsFile.exists()
|
||||
}
|
||||
}
|
||||
|
||||
javaRestTest {
|
||||
dependsOn fixture
|
||||
systemProperty "tests.fromOld", "true"
|
||||
/* Use a closure on the string to delay evaluation until right before we
|
||||
* run the integration tests so that we can be sure that the file is
|
||||
* ready. */
|
||||
nonInputProperties.systemProperty "es${version}.port", "${-> fixture.addressAndPort}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,151 +0,0 @@
|
|||
/*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* The OpenSearch Contributors require contributions made to
|
||||
* this file be licensed under the Apache-2.0 license or a
|
||||
* compatible open source license.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Modifications Copyright OpenSearch Contributors. See
|
||||
* GitHub history for details.
|
||||
*/
|
||||
|
||||
package org.opensearch.index.reindex.remote;
|
||||
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.apache.lucene.util.Constants;
|
||||
import org.opensearch.client.Request;
|
||||
import org.opensearch.client.Response;
|
||||
import org.opensearch.client.RestClient;
|
||||
import org.opensearch.common.Booleans;
|
||||
import org.opensearch.test.rest.OpenSearchRestTestCase;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
|
||||
public class ReindexFromOldRemoteIT extends OpenSearchRestTestCase {
|
||||
/**
|
||||
* Number of documents to test when reindexing from an old version.
|
||||
*/
|
||||
private static final int DOCS = 5;
|
||||
|
||||
private void oldEsTestCase(String portPropertyName, String requestsPerSecond) throws IOException {
|
||||
boolean enabled = Booleans.parseBoolean(System.getProperty("tests.fromOld"));
|
||||
assumeTrue("test is disabled, probably because this is windows", enabled);
|
||||
|
||||
int oldEsPort = Integer.parseInt(System.getProperty(portPropertyName));
|
||||
try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) {
|
||||
try {
|
||||
Request createIndex = new Request("PUT", "/test");
|
||||
createIndex.setJsonEntity("{\"settings\":{\"number_of_shards\": 1}}");
|
||||
oldEs.performRequest(createIndex);
|
||||
|
||||
for (int i = 0; i < DOCS; i++) {
|
||||
Request doc = new Request("PUT", "/test/doc/testdoc" + i);
|
||||
doc.addParameter("refresh", "true");
|
||||
doc.setJsonEntity("{\"test\":\"test\"}");
|
||||
oldEs.performRequest(doc);
|
||||
}
|
||||
|
||||
Request reindex = new Request("POST", "/_reindex");
|
||||
if (randomBoolean()) {
|
||||
// Reindex using the external version_type
|
||||
reindex.setJsonEntity(
|
||||
"{\n"
|
||||
+ " \"source\":{\n"
|
||||
+ " \"index\": \"test\",\n"
|
||||
+ " \"size\": 1,\n"
|
||||
+ " \"remote\": {\n"
|
||||
+ " \"host\": \"http://127.0.0.1:" + oldEsPort + "\"\n"
|
||||
+ " }\n"
|
||||
+ " },\n"
|
||||
+ " \"dest\": {\n"
|
||||
+ " \"index\": \"test\",\n"
|
||||
+ " \"version_type\": \"external\"\n"
|
||||
+ " }\n"
|
||||
+ "}");
|
||||
} else {
|
||||
// Reindex using the default internal version_type
|
||||
reindex.setJsonEntity(
|
||||
"{\n"
|
||||
+ " \"source\":{\n"
|
||||
+ " \"index\": \"test\",\n"
|
||||
+ " \"size\": 1,\n"
|
||||
+ " \"remote\": {\n"
|
||||
+ " \"host\": \"http://127.0.0.1:" + oldEsPort + "\"\n"
|
||||
+ " }\n"
|
||||
+ " },\n"
|
||||
+ " \"dest\": {\n"
|
||||
+ " \"index\": \"test\"\n"
|
||||
+ " }\n"
|
||||
+ "}");
|
||||
}
|
||||
reindex.addParameter("refresh", "true");
|
||||
reindex.addParameter("pretty", "true");
|
||||
if (requestsPerSecond != null) {
|
||||
reindex.addParameter("requests_per_second", requestsPerSecond);
|
||||
}
|
||||
client().performRequest(reindex);
|
||||
|
||||
Request search = new Request("POST", "/test/_search");
|
||||
search.addParameter("pretty", "true");
|
||||
Response response = client().performRequest(search);
|
||||
String result = EntityUtils.toString(response.getEntity());
|
||||
for (int i = 0; i < DOCS; i++) {
|
||||
assertThat(result, containsString("\"_id\" : \"testdoc" + i + "\""));
|
||||
}
|
||||
} finally {
|
||||
oldEs.performRequest(new Request("DELETE", "/test"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void testEs2() throws IOException {
|
||||
oldEsTestCase("es2.port", null);
|
||||
}
|
||||
|
||||
public void testEs1() throws IOException {
|
||||
oldEsTestCase("es1.port", null);
|
||||
}
|
||||
|
||||
public void testEs090() throws IOException {
|
||||
assumeFalse("No longer works on Mac", Constants.MAC_OS_X);
|
||||
oldEsTestCase("es090.port", null);
|
||||
}
|
||||
|
||||
public void testEs2WithFunnyThrottle() throws IOException {
|
||||
oldEsTestCase("es2.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
|
||||
}
|
||||
|
||||
public void testEs1WithFunnyThrottle() throws IOException {
|
||||
oldEsTestCase("es1.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
|
||||
}
|
||||
|
||||
public void testEs090WithFunnyThrottle() throws IOException {
|
||||
assumeFalse("No longer works on Mac", Constants.MAC_OS_X);
|
||||
oldEsTestCase("es090.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue