Use fixture to test repository-url module (#29355)

This commit adds a YAML integration test for the repository-url module
that uses a fixture to test URL based repositories on both http:// and
file:// prefixes.

modules/repository-url/build.gradle

@@ -16,12 +16,36 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+import org.elasticsearch.gradle.test.AntFixture
 
 esplugin {
     description 'Module for URL repository'
     classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
 }
 
-integTestCluster {
+forbiddenApisTest {
-    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+    // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
+    bundledSignatures -= 'jdk-non-portable'
+    bundledSignatures += 'jdk-internal'
+}
+
+// This directory is shared between two URL repositories and one FS repository in YAML integration tests
+File repositoryDir = new File(project.buildDir, "shared-repository")
+
+/** A task to start the URLFixture which exposes the repositoryDir over HTTP **/
+task urlFixture(type: AntFixture) {
+    doFirst {
+        repositoryDir.mkdirs()
+    }
+    env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
+    executable = new File(project.runtimeJavaHome, 'bin/java')
+    args 'org.elasticsearch.repositories.url.URLFixture', baseDir, "${repositoryDir.absolutePath}"
+}
+
+integTestCluster {
+    dependsOn urlFixture
+    // repositoryDir is used by a FS repository to create snapshots
+    setting 'path.repo', "${repositoryDir.absolutePath}"
+    // repositoryDir is used by two URL repositories to restore snapshots
+    setting 'repositories.url.allowed_urls', "http://snapshot.test*,http://${ -> urlFixture.addressAndPort }"
 }

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java

@@ -21,9 +21,31 @@ package org.elasticsearch.repositories.url;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NStringEntity;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 
 public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -35,5 +57,66 @@ public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCas
     public static Iterable<Object[]> parameters() throws Exception {
         return ESClientYamlSuiteTestCase.createParameters();
     }
+
+    /**
+     * This method registers 3 snapshot/restore repositories:
+     * - repository-fs: this FS repository is used to create snapshots.
+     * - repository-url: this URL repository is used to restore snapshots created using the previous repository. It uses
+     * the URLFixture to restore snapshots over HTTP.
+     * - repository-file: similar as the previous repository but using a file:// prefix instead of http://.
+     **/
+    @Before
+    public void registerRepositories() throws IOException {
+        Response clusterSettingsResponse = client().performRequest("GET", "/_cluster/settings?include_defaults=true" +
+            "&filter_path=defaults.path.repo,defaults.repositories.url.allowed_urls");
+        Map<String, Object> clusterSettings = entityAsMap(clusterSettingsResponse);
+
+        @SuppressWarnings("unchecked")
+        List<String> pathRepo = (List<String>) XContentMapValues.extractValue("defaults.path.repo", clusterSettings);
+        assertThat(pathRepo, hasSize(1));
+
+        // Create a FS repository using the path.repo location
+        Response createFsRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-fs", emptyMap(),
+            buildRepositorySettings(FsRepository.TYPE, Settings.builder().put("location", pathRepo.get(0)).build()));
+        assertThat(createFsRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+
+        // Create a URL repository using the file://{path.repo} URL
+        Response createFileRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-file", emptyMap(),
+            buildRepositorySettings(URLRepository.TYPE, Settings.builder().put("url", "file://" + pathRepo.get(0)).build()));
+        assertThat(createFileRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+
+        // Create a URL repository using the http://{fixture} URL
+        @SuppressWarnings("unchecked")
+        List<String> allowedUrls = (List<String>) XContentMapValues.extractValue("defaults.repositories.url.allowed_urls", clusterSettings);
+        for (String allowedUrl : allowedUrls) {
+            try {
+                InetAddress inetAddress = InetAddress.getByName(new URL(allowedUrl).getHost());
+                if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress()) {
+                    Response createUrlRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-url", emptyMap(),
+                        buildRepositorySettings(URLRepository.TYPE, Settings.builder().put("url", allowedUrl).build()));
+                    assertThat(createUrlRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
+                    break;
+                }
+            } catch (Exception e) {
+                logger.debug("Failed to resolve inet address for allowed URL [{}], skipping", allowedUrl);
+            }
+        }
+    }
+
+    private static HttpEntity buildRepositorySettings(final String type, final Settings settings) throws IOException {
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            {
+                builder.field("type", type);
+                builder.startObject("settings");
+                {
+                    settings.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+            return new NStringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
+        }
+    }
 }
 

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java

@@ -0,0 +1,162 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories.url;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.mocksocket.MockHttpServer;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Map;
+import java.util.Objects;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonMap;
+
+/**
+ * This {@link URLFixture} exposes a filesystem directory over HTTP. It is used in repository-url
+ * integration tests to expose a directory created by a regular FS repository.
+ */
+public class URLFixture {
+
+    public static void main(String[] args) throws Exception {
+        if (args == null || args.length != 2) {
+            throw new IllegalArgumentException("URLFixture <working directory> <repository directory>");
+        }
+
+        final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 0);
+        final HttpServer httpServer = MockHttpServer.createHttp(socketAddress, 0);
+
+        try {
+            final Path workingDirectory = dir(args[0]);
+            /// Writes the PID of the current Java process in a `pid` file located in the working directory
+            writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
+
+            final String addressAndPort = addressToString(httpServer.getAddress());
+            // Writes the address and port of the http server in a `ports` file located in the working directory
+            writeFile(workingDirectory, "ports", addressAndPort);
+
+            // Exposes the repository over HTTP
+            final String url = "http://" + addressAndPort;
+            httpServer.createContext("/", new ResponseHandler(dir(args[1])));
+            httpServer.start();
+
+            // Wait to be killed
+            Thread.sleep(Long.MAX_VALUE);
+
+        } finally {
+            httpServer.stop(0);
+        }
+    }
+
+    @SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")
+    private static Path dir(final String dir) {
+        return Paths.get(dir);
+    }
+
+    private static void writeFile(final Path dir, final String fileName, final String content) throws IOException {
+        final Path tempPidFile = Files.createTempFile(dir, null, null);
+        Files.write(tempPidFile, singleton(content));
+        Files.move(tempPidFile, dir.resolve(fileName), StandardCopyOption.ATOMIC_MOVE);
+    }
+
+    private static String addressToString(final SocketAddress address) {
+        final InetSocketAddress inetSocketAddress = (InetSocketAddress) address;
+        if (inetSocketAddress.getAddress() instanceof Inet6Address) {
+            return "[" + inetSocketAddress.getHostString() + "]:" + inetSocketAddress.getPort();
+        } else {
+            return inetSocketAddress.getHostString() + ":" + inetSocketAddress.getPort();
+        }
+    }
+
+    static class ResponseHandler implements HttpHandler {
+
+        private final Path repositoryDir;
+
+        ResponseHandler(final Path repositoryDir) {
+            this.repositoryDir = repositoryDir;
+        }
+
+        @Override
+        public void handle(HttpExchange exchange) throws IOException {
+            Response response;
+            if ("GET".equalsIgnoreCase(exchange.getRequestMethod())) {
+                String path = exchange.getRequestURI().toString();
+                if (path.length() > 0 && path.charAt(0) == '/') {
+                    path = path.substring(1);
+                }
+
+                Path normalizedRepositoryDir = repositoryDir.normalize();
+                Path normalizedPath = normalizedRepositoryDir.resolve(path).normalize();
+
+                if (normalizedPath.startsWith(normalizedRepositoryDir)) {
+                    if (Files.exists(normalizedPath) && Files.isReadable(normalizedPath) && Files.isRegularFile(normalizedPath)) {
+                        byte[] content = Files.readAllBytes(normalizedPath);
+                        Map<String, String> headers = singletonMap("Content-Length", String.valueOf(content.length));
+                        response = new Response(RestStatus.OK, headers, "application/octet-stream", content);
+                    } else {
+                        response = new Response(RestStatus.NOT_FOUND, emptyMap(), "text/plain", new byte[0]);
+                    }
+                } else {
+                    response = new Response(RestStatus.FORBIDDEN, emptyMap(), "text/plain", new byte[0]);
+                }
+            } else {
+                response = new Response(RestStatus.INTERNAL_SERVER_ERROR, emptyMap(), "text/plain",
+                    "Unsupported HTTP method".getBytes(StandardCharsets.UTF_8));
+            }
+            exchange.sendResponseHeaders(response.status.getStatus(), response.body.length);
+            if (response.body.length > 0) {
+                exchange.getResponseBody().write(response.body);
+            }
+            exchange.close();
+        }
+    }
+
+    /**
+     * Represents a HTTP Response.
+     */
+    static class Response {
+
+        final RestStatus status;
+        final Map<String, String> headers;
+        final String contentType;
+        final byte[] body;
+
+        Response(final RestStatus status, final Map<String, String> headers, final String contentType, final byte[] body) {
+            this.status = Objects.requireNonNull(status);
+            this.headers = Objects.requireNonNull(headers);
+            this.contentType = Objects.requireNonNull(contentType);
+            this.body = Objects.requireNonNull(body);
+        }
+    }
+}

modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml

@@ -1,6 +1,108 @@
-# Integration tests for URL Repository component
+# Integration tests for repository-url
 #
-"URL Repository plugin loaded":
+# This test is based on 3 repositories, all registered before this
+# test is executed. The repository-fs is used to create snapshots
+# in a shared directory on the filesystem. Then the test uses a URL
+# repository with a "http://" prefix to test the restore of the
+# snapshots. In order to do that it uses a URLFixture that exposes
+# the content of the shared directory over HTTP. A second URL
+# repository is used to test the snapshot restore but this time
+# with a "file://" prefix. 
+setup:
+
+  # Ensure that the FS repository is registered, so we can create
+  # snapshots that we later restore using the URL repository
+  - do:
+      snapshot.get_repository:
+        repository: repository-fs
+
+  # Index documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    1
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    2
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    3
+          - snapshot: one
+
+  # Create a first snapshot using the FS repository
+  - do:
+      snapshot.create:
+        repository: repository-fs
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  # Index more documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    4
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    5
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    6
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    7
+          - snapshot: two
+
+  # Create a second snapshot
+  - do:
+      snapshot.create:
+        repository: repository-fs
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - do:
+      snapshot.get:
+        repository: repository-fs
+        snapshot: snapshot-one,snapshot-two
+
+---
+teardown:
+
+  - do:
+      indices.delete:
+        index: docs
+        ignore_unavailable: true
+
+  # Remove the snapshots
+  - do:
+      snapshot.delete:
+        repository: repository-fs
+        snapshot: snapshot-two
+
+  - do:
+      snapshot.delete:
+        repository: repository-fs
+        snapshot: snapshot-one
+
+---
+"Module repository-url is loaded":
     - do:
         cluster.state: {}
 
@@ -10,23 +112,129 @@
     - do:
         nodes.info: {}
 
-    - match:  { nodes.$master.modules.0.name: repository-url }
+    - match: { nodes.$master.modules.0.name: repository-url }
 
 ---
-setup:
+"Restore with repository-url using http://":
+
+  # Ensure that the URL repository is registered
+  - do:
+      snapshot.get_repository:
+        repository: repository-url
+
+  - match: { repository-url.type : "url" }
+  - match: { repository-url.settings.url: '/http://(.+):\d+/' }
 
   - do:
-      snapshot.create_repository:
+      snapshot.get:
-        repository: test_repo1
+        repository: repository-url
-        body:
+        snapshot: snapshot-one,snapshot-two
-          type: url
+
-          settings:
+  - is_true: snapshots
-            url: "http://snapshot.test1"
+  - match: { snapshots.0.state : SUCCESS }
+  - match: { snapshots.1.state : SUCCESS }
+
+  # Delete the index
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the second snapshot
+  - do:
+      snapshot.restore:
+        repository: repository-url
+        snapshot: snapshot-two
+        wait_for_completion: true
 
   - do:
-      snapshot.create_repository:
+      count:
-        repository: test_repo2
+        index: docs
-        body:
+
-          type: url
+  - match: {count: 7}
-          settings:
+
-            url: "http://snapshot.test2"
+  # Delete the index again
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the first snapshot
+  - do:
+      snapshot.restore:
+        repository: repository-url
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  - do:
+      catch: /cannot delete snapshot from a readonly repository/
+      snapshot.delete:
+        repository: repository-url
+        snapshot: snapshot-two
+
+---
+"Restore with repository-url using file://":
+
+  # Ensure that the URL repository is registered
+  - do:
+      snapshot.get_repository:
+        repository: repository-file
+
+  - match: { repository-file.type : "url" }
+  - match: { repository-file.settings.url: '/file://(.+)/' }
+
+  - do:
+      snapshot.get:
+        repository: repository-file
+        snapshot: snapshot-one,snapshot-two
+
+  - is_true: snapshots
+  - match: { snapshots.0.state : SUCCESS }
+  - match: { snapshots.1.state : SUCCESS }
+
+  # Delete the index
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the second snapshot
+  - do:
+      snapshot.restore:
+        repository: repository-file
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Delete the index again
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the first snapshot
+  - do:
+      snapshot.restore:
+        repository: repository-file
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  - do:
+      catch: /cannot delete snapshot from a readonly repository/
+      snapshot.delete:
+        repository: repository-file
+        snapshot: snapshot-one
+

modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yml

@@ -14,3 +14,18 @@
         repository: test_repo1
 
   - is_true : test_repo1
+
+---
+"Repository cannot be be registered":
+
+  - do:
+      catch: /doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls/
+      snapshot.create_repository:
+        repository: test_repo2
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.unknown"
+
+
+