honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use fixture to test repository-url module (#29355) 

This commit adds a YAML integration test for the repository-url module
that uses a fixture to test URL based repositories on both http:// and
file:// prefixes.
This commit is contained in:
Tanguy Leroux 2018-04-04 15:55:26 +02:00 committed by GitHub
parent 25d411eb32
commit 08abbdf129
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 511 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
modules/repository-url/build.gradle
View File

@ -16,12 +16,36 @@
* specific language governing permissions and limitations
* under the License.
*/
import org.elasticsearch.gradle.test.AntFixture
esplugin {
description 'Module for URL repository'
classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
}
integTestCluster {
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
forbiddenApisTest {
// we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
bundledSignatures -= 'jdk-non-portable'
bundledSignatures += 'jdk-internal'
}
// This directory is shared between two URL repositories and one FS repository in YAML integration tests
File repositoryDir = new File(project.buildDir, "shared-repository")
/** A task to start the URLFixture which exposes the repositoryDir over HTTP **/
task urlFixture(type: AntFixture) {
doFirst {
repositoryDir.mkdirs()
}
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
executable = new File(project.runtimeJavaHome, 'bin/java')
args 'org.elasticsearch.repositories.url.URLFixture', baseDir, "${repositoryDir.absolutePath}"
}
integTestCluster {
dependsOn urlFixture
// repositoryDir is used by a FS repository to create snapshots
setting 'path.repo', "${repositoryDir.absolutePath}"
// repositoryDir is used by two URL repositories to restore snapshots
setting 'repositories.url.allowed_urls', "http://snapshot.test*,http://${ -> urlFixture.addressAndPort }"
}

85
modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java
View File

@ -21,9 +21,31 @@ package org.elasticsearch.repositories.url;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.apache.http.HttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.nio.entity.NStringEntity;
import org.elasticsearch.client.Response;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import org.junit.Before;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URL;
import java.util.List;
import java.util.Map;
import static java.util.Collections.emptyMap;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@ -35,5 +57,66 @@ public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCas
public static Iterable<Object[]> parameters() throws Exception {
return ESClientYamlSuiteTestCase.createParameters();
}
/**
* This method registers 3 snapshot/restore repositories:
* - repository-fs: this FS repository is used to create snapshots.
* - repository-url: this URL repository is used to restore snapshots created using the previous repository. It uses
* the URLFixture to restore snapshots over HTTP.
* - repository-file: similar as the previous repository but using a file:// prefix instead of http://.
**/
@Before
public void registerRepositories() throws IOException {
Response clusterSettingsResponse = client().performRequest("GET", "/_cluster/settings?include_defaults=true" +
"&filter_path=defaults.path.repo,defaults.repositories.url.allowed_urls");
Map<String, Object> clusterSettings = entityAsMap(clusterSettingsResponse);
@SuppressWarnings("unchecked")
List<String> pathRepo = (List<String>) XContentMapValues.extractValue("defaults.path.repo", clusterSettings);
assertThat(pathRepo, hasSize(1));
// Create a FS repository using the path.repo location
Response createFsRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-fs", emptyMap(),
buildRepositorySettings(FsRepository.TYPE, Settings.builder().put("location", pathRepo.get(0)).build()));
assertThat(createFsRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
// Create a URL repository using the file://{path.repo} URL
Response createFileRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-file", emptyMap(),
buildRepositorySettings(URLRepository.TYPE, Settings.builder().put("url", "file://" + pathRepo.get(0)).build()));
assertThat(createFileRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
// Create a URL repository using the http://{fixture} URL
@SuppressWarnings("unchecked")
List<String> allowedUrls = (List<String>) XContentMapValues.extractValue("defaults.repositories.url.allowed_urls", clusterSettings);
for (String allowedUrl : allowedUrls) {
try {
InetAddress inetAddress = InetAddress.getByName(new URL(allowedUrl).getHost());
if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress()) {
Response createUrlRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-url", emptyMap(),
buildRepositorySettings(URLRepository.TYPE, Settings.builder().put("url", allowedUrl).build()));
assertThat(createUrlRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
break;
}
} catch (Exception e) {
logger.debug("Failed to resolve inet address for allowed URL [{}], skipping", allowedUrl);
}
}
}
private static HttpEntity buildRepositorySettings(final String type, final Settings settings) throws IOException {
try (XContentBuilder builder = jsonBuilder()) {
builder.startObject();
{
builder.field("type", type);
builder.startObject("settings");
{
settings.toXContent(builder, ToXContent.EMPTY_PARAMS);
}
builder.endObject();
}
builder.endObject();
return new NStringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
}
}
}

162
modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java Normal file
View File

@ -0,0 +1,162 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.url;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;
import org.elasticsearch.common.SuppressForbidden;
import org.elasticsearch.mocksocket.MockHttpServer;
import org.elasticsearch.rest.RestStatus;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Map;
import java.util.Objects;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singleton;
import static java.util.Collections.singletonMap;
/**
* This {@link URLFixture} exposes a filesystem directory over HTTP. It is used in repository-url
* integration tests to expose a directory created by a regular FS repository.
*/
public class URLFixture {
public static void main(String[] args) throws Exception {
if (args == null || args.length != 2) {
throw new IllegalArgumentException("URLFixture <working directory> <repository directory>");
}
final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 0);
final HttpServer httpServer = MockHttpServer.createHttp(socketAddress, 0);
try {
final Path workingDirectory = dir(args[0]);
/// Writes the PID of the current Java process in a `pid` file located in the working directory
writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
final String addressAndPort = addressToString(httpServer.getAddress());
// Writes the address and port of the http server in a `ports` file located in the working directory
writeFile(workingDirectory, "ports", addressAndPort);
// Exposes the repository over HTTP
final String url = "http://" + addressAndPort;
httpServer.createContext("/", new ResponseHandler(dir(args[1])));
httpServer.start();
// Wait to be killed
Thread.sleep(Long.MAX_VALUE);
} finally {
httpServer.stop(0);
}
}
@SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")
private static Path dir(final String dir) {
return Paths.get(dir);
}
private static void writeFile(final Path dir, final String fileName, final String content) throws IOException {
final Path tempPidFile = Files.createTempFile(dir, null, null);
Files.write(tempPidFile, singleton(content));
Files.move(tempPidFile, dir.resolve(fileName), StandardCopyOption.ATOMIC_MOVE);
}
private static String addressToString(final SocketAddress address) {
final InetSocketAddress inetSocketAddress = (InetSocketAddress) address;
if (inetSocketAddress.getAddress() instanceof Inet6Address) {
return "[" + inetSocketAddress.getHostString() + "]:" + inetSocketAddress.getPort();
} else {
return inetSocketAddress.getHostString() + ":" + inetSocketAddress.getPort();
}
}
static class ResponseHandler implements HttpHandler {
private final Path repositoryDir;
ResponseHandler(final Path repositoryDir) {
this.repositoryDir = repositoryDir;
}
@Override
public void handle(HttpExchange exchange) throws IOException {
Response response;
if ("GET".equalsIgnoreCase(exchange.getRequestMethod())) {
String path = exchange.getRequestURI().toString();
if (path.length() > 0 && path.charAt(0) == '/') {
path = path.substring(1);
}
Path normalizedRepositoryDir = repositoryDir.normalize();
Path normalizedPath = normalizedRepositoryDir.resolve(path).normalize();
if (normalizedPath.startsWith(normalizedRepositoryDir)) {
if (Files.exists(normalizedPath) && Files.isReadable(normalizedPath) && Files.isRegularFile(normalizedPath)) {
byte[] content = Files.readAllBytes(normalizedPath);
Map<String, String> headers = singletonMap("Content-Length", String.valueOf(content.length));
response = new Response(RestStatus.OK, headers, "application/octet-stream", content);
} else {
response = new Response(RestStatus.NOT_FOUND, emptyMap(), "text/plain", new byte[0]);
}
} else {
response = new Response(RestStatus.FORBIDDEN, emptyMap(), "text/plain", new byte[0]);
}
} else {
response = new Response(RestStatus.INTERNAL_SERVER_ERROR, emptyMap(), "text/plain",
"Unsupported HTTP method".getBytes(StandardCharsets.UTF_8));
}
exchange.sendResponseHeaders(response.status.getStatus(), response.body.length);
if (response.body.length > 0) {
exchange.getResponseBody().write(response.body);
}
exchange.close();
}
}
/**
* Represents a HTTP Response.
*/
static class Response {
final RestStatus status;
final Map<String, String> headers;
final String contentType;
final byte[] body;
Response(final RestStatus status, final Map<String, String> headers, final String contentType, final byte[] body) {
this.status = Objects.requireNonNull(status);
this.headers = Objects.requireNonNull(headers);
this.contentType = Objects.requireNonNull(contentType);
this.body = Objects.requireNonNull(body);
}
}
}

240
modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml
View File

@ -1,6 +1,108 @@
# Integration tests for URL Repository component
# Integration tests for repository-url
#
"URL Repository plugin loaded":
# This test is based on 3 repositories, all registered before this
# test is executed. The repository-fs is used to create snapshots
# in a shared directory on the filesystem. Then the test uses a URL
# repository with a "http://" prefix to test the restore of the
# snapshots. In order to do that it uses a URLFixture that exposes
# the content of the shared directory over HTTP. A second URL
# repository is used to test the snapshot restore but this time
# with a "file://" prefix.
setup:
# Ensure that the FS repository is registered, so we can create
# snapshots that we later restore using the URL repository
- do:
snapshot.get_repository:
repository: repository-fs
# Index documents
- do:
bulk:
refresh: true
body:
- index:
_index: docs
_type: doc
_id: 1
- snapshot: one
- index:
_index: docs
_type: doc
_id: 2
- snapshot: one
- index:
_index: docs
_type: doc
_id: 3
- snapshot: one
# Create a first snapshot using the FS repository
- do:
snapshot.create:
repository: repository-fs
snapshot: snapshot-one
wait_for_completion: true
# Index more documents
- do:
bulk:
refresh: true
body:
- index:
_index: docs
_type: doc
_id: 4
- snapshot: two
- index:
_index: docs
_type: doc
_id: 5
- snapshot: two
- index:
_index: docs
_type: doc
_id: 6
- snapshot: two
- index:
_index: docs
_type: doc
_id: 7
- snapshot: two
# Create a second snapshot
- do:
snapshot.create:
repository: repository-fs
snapshot: snapshot-two
wait_for_completion: true
- do:
snapshot.get:
repository: repository-fs
snapshot: snapshot-one,snapshot-two
---
teardown:
- do:
indices.delete:
index: docs
ignore_unavailable: true
# Remove the snapshots
- do:
snapshot.delete:
repository: repository-fs
snapshot: snapshot-two
- do:
snapshot.delete:
repository: repository-fs
snapshot: snapshot-one
---
"Module repository-url is loaded":
- do:
cluster.state: {}
@ -10,23 +112,129 @@
- do:
nodes.info: {}
- match: { nodes.$master.modules.0.name: repository-url }
- match: { nodes.$master.modules.0.name: repository-url }
---
setup:
"Restore with repository-url using http://":
# Ensure that the URL repository is registered
- do:
snapshot.get_repository:
repository: repository-url
- match: { repository-url.type : "url" }
- match: { repository-url.settings.url: '/http://(.+):\d+/' }
- do:
snapshot.create_repository:
repository: test_repo1
body:
type: url
settings:
url: "http://snapshot.test1"
snapshot.get:
repository: repository-url
snapshot: snapshot-one,snapshot-two
- is_true: snapshots
- match: { snapshots.0.state : SUCCESS }
- match: { snapshots.1.state : SUCCESS }
# Delete the index
- do:
indices.delete:
index: docs
# Restore the second snapshot
- do:
snapshot.restore:
repository: repository-url
snapshot: snapshot-two
wait_for_completion: true
- do:
snapshot.create_repository:
repository: test_repo2
body:
type: url
settings:
url: "http://snapshot.test2"
count:
index: docs
- match: {count: 7}
# Delete the index again
- do:
indices.delete:
index: docs
# Restore the first snapshot
- do:
snapshot.restore:
repository: repository-url
snapshot: snapshot-one
wait_for_completion: true
- do:
count:
index: docs
- match: {count: 3}
- do:
catch: /cannot delete snapshot from a readonly repository/
snapshot.delete:
repository: repository-url
snapshot: snapshot-two
---
"Restore with repository-url using file://":
# Ensure that the URL repository is registered
- do:
snapshot.get_repository:
repository: repository-file
- match: { repository-file.type : "url" }
- match: { repository-file.settings.url: '/file://(.+)/' }
- do:
snapshot.get:
repository: repository-file
snapshot: snapshot-one,snapshot-two
- is_true: snapshots
- match: { snapshots.0.state : SUCCESS }
- match: { snapshots.1.state : SUCCESS }
# Delete the index
- do:
indices.delete:
index: docs
# Restore the second snapshot
- do:
snapshot.restore:
repository: repository-file
snapshot: snapshot-two
wait_for_completion: true
- do:
count:
index: docs
- match: {count: 7}
# Delete the index again
- do:
indices.delete:
index: docs
# Restore the first snapshot
- do:
snapshot.restore:
repository: repository-file
snapshot: snapshot-one
wait_for_completion: true
- do:
count:
index: docs
- match: {count: 3}
- do:
catch: /cannot delete snapshot from a readonly repository/
snapshot.delete:
repository: repository-file
snapshot: snapshot-one

15
modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yml
View File

@ -14,3 +14,18 @@
repository: test_repo1
- is_true : test_repo1
---
"Repository cannot be be registered":
- do:
catch: /doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls/
snapshot.create_repository:
repository: test_repo2
body:
type: url
settings:
url: "http://snapshot.unknown"