From 9a95be35cfc00a1e08a0bcface85b4c5d852db59 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Wed, 21 Feb 2018 13:20:35 +0100 Subject: [PATCH] [Tests] Extract the testing logic for Google Cloud Storage (#28576) This pull request extracts in a dedicated class the request/response logic that "emulates" a Google Cloud Storage service in our repository-gcs tests. The idea behind this is to make the logic more reusable. The class MockHttpTransport has been renamed to MockStorage which now only takes care of instantiating a Storage client and does the low-level request/response plumbing needed by this client. The "Google Cloud Storage" logic has been extracted from MockHttpTransport and put in a new GoogleCloudStorageTestServer that is now independent from the google client testing framework. --- ...leCloudStorageBlobStoreContainerTests.java | 6 +- ...eCloudStorageBlobStoreRepositoryTests.java | 2 +- .../gcs/GoogleCloudStorageBlobStoreTests.java | 5 +- .../gcs/GoogleCloudStorageTestServer.java | 495 ++++++++++++++++++ .../repositories/gcs/MockHttpTransport.java | 433 --------------- .../repositories/gcs/MockStorage.java | 93 ++++ 6 files changed, 594 insertions(+), 440 deletions(-) create mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestServer.java delete mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpTransport.java create mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java index f3fada5a463..7b985ebd176 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java @@ -23,13 +23,13 @@ import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.repositories.ESBlobStoreContainerTestCase; -import java.io.IOException; import java.util.Locale; public class GoogleCloudStorageBlobStoreContainerTests extends ESBlobStoreContainerTestCase { + @Override - protected BlobStore newBlobStore() throws IOException { + protected BlobStore newBlobStore() { String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); - return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockHttpTransport.newStorage(bucket, getTestName())); + return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockStorage.newStorageClient(bucket, getTestName())); } } diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index ed65caf0d59..dbad40ec083 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -65,7 +65,7 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepos @BeforeClass public static void setUpStorage() { - storage.set(MockHttpTransport.newStorage(BUCKET, GoogleCloudStorageBlobStoreRepositoryTests.class.getName())); + storage.set(MockStorage.newStorageClient(BUCKET, GoogleCloudStorageBlobStoreRepositoryTests.class.getName())); } public static class MockGoogleCloudStoragePlugin extends GoogleCloudStoragePlugin { diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java index fe94237f6c9..00c0538d198 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java @@ -23,14 +23,13 @@ import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.repositories.ESBlobStoreTestCase; -import java.io.IOException; import java.util.Locale; public class GoogleCloudStorageBlobStoreTests extends ESBlobStoreTestCase { @Override - protected BlobStore newBlobStore() throws IOException { + protected BlobStore newBlobStore() { String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); - return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockHttpTransport.newStorage(bucket, getTestName())); + return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockStorage.newStorageClient(bucket, getTestName())); } } diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestServer.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestServer.java new file mode 100644 index 00000000000..17255fa90ed --- /dev/null +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestServer.java @@ -0,0 +1,495 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.repositories.gcs; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.path.PathTrie; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.RestUtils; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * {@link GoogleCloudStorageTestServer} emulates a Google Cloud Storage service through a {@link #handle(String, String, byte[])} method + * that provides appropriate responses for specific requests like the real Google Cloud platform would do. It is largely based on official + * documentation available at https://cloud.google.com/storage/docs/json_api/v1/. + */ +public class GoogleCloudStorageTestServer { + + private static byte[] EMPTY_BYTE = new byte[0]; + + /** List of the buckets stored on this test server **/ + private final Map buckets = ConcurrentCollections.newConcurrentMap(); + + /** Request handlers for the requests made by the Google Cloud Storage client **/ + private final PathTrie handlers; + + /** + * Creates a {@link GoogleCloudStorageTestServer} with the default endpoint + */ + GoogleCloudStorageTestServer() { + this("https://www.googleapis.com", true); + } + + /** + * Creates a {@link GoogleCloudStorageTestServer} with a custom endpoint, + * potentially prefixing the URL patterns to match with the endpoint name. + */ + GoogleCloudStorageTestServer(final String endpoint, final boolean prefixWithEndpoint) { + this.handlers = defaultHandlers(endpoint, prefixWithEndpoint, buckets); + } + + /** Creates a bucket in the test server **/ + void createBucket(final String bucketName) { + buckets.put(bucketName, new Bucket(bucketName)); + } + + public Response handle(final String method, final String url, byte[] content) throws IOException { + final Map params = new HashMap<>(); + + // Splits the URL to extract query string parameters + final String rawPath; + int questionMark = url.indexOf('?'); + if (questionMark != -1) { + rawPath = url.substring(0, questionMark); + RestUtils.decodeQueryString(url, questionMark + 1, params); + } else { + rawPath = url; + } + + final RequestHandler handler = handlers.retrieve(method + " " + rawPath, params); + if (handler != null) { + return handler.execute(url, params, content); + } else { + return newError(RestStatus.INTERNAL_SERVER_ERROR, "No handler defined for request [method: " + method + ", url: " + url + "]"); + } + } + + @FunctionalInterface + interface RequestHandler { + + /** + * Simulates the execution of a Storage request and returns a corresponding response. + * + * @param url the request URL + * @param params the request URL parameters + * @param body the request body provided as a byte array + * @return the corresponding response + * + * @throws IOException if something goes wrong + */ + Response execute(String url, Map params, byte[] body) throws IOException; + } + + /** Builds the default request handlers **/ + private static PathTrie defaultHandlers(final String endpoint, + final boolean prefixWithEndpoint, + final Map buckets) { + + final PathTrie handlers = new PathTrie<>(RestUtils.REST_DECODER); + final String prefix = prefixWithEndpoint ? endpoint : ""; + + // GET Bucket + // + // https://cloud.google.com/storage/docs/json_api/v1/buckets/get + handlers.insert("GET " + prefix + "/storage/v1/b/{bucket}", (url, params, body) -> { + String name = params.get("bucket"); + if (Strings.hasText(name) == false) { + return newError(RestStatus.INTERNAL_SERVER_ERROR, "bucket name is missing"); + } + + if (buckets.containsKey(name)) { + return newResponse(RestStatus.OK, emptyMap(), buildBucketResource(name)); + } else { + return newError(RestStatus.NOT_FOUND, "bucket not found"); + } + }); + + // GET Object + // + // https://cloud.google.com/storage/docs/json_api/v1/objects/get + handlers.insert("GET " + prefix + "/storage/v1/b/{bucket}/o/{object}", (url, params, body) -> { + String objectName = params.get("object"); + if (Strings.hasText(objectName) == false) { + return newError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing"); + } + + final Bucket bucket = buckets.get(params.get("bucket")); + if (bucket == null) { + return newError(RestStatus.NOT_FOUND, "bucket not found"); + } + + for (Map.Entry object : bucket.objects.entrySet()) { + if (object.getKey().equals(objectName)) { + return newResponse(RestStatus.OK, emptyMap(), buildObjectResource(bucket.name, objectName, object.getValue())); + } + } + return newError(RestStatus.NOT_FOUND, "object not found"); + }); + + // Delete Object + // + // https://cloud.google.com/storage/docs/json_api/v1/objects/delete + handlers.insert("DELETE " + prefix + "/storage/v1/b/{bucket}/o/{object}", (url, params, body) -> { + String objectName = params.get("object"); + if (Strings.hasText(objectName) == false) { + return newError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing"); + } + + final Bucket bucket = buckets.get(params.get("bucket")); + if (bucket == null) { + return newError(RestStatus.NOT_FOUND, "bucket not found"); + } + + final byte[] bytes = bucket.objects.remove(objectName); + if (bytes != null) { + return new Response(RestStatus.NO_CONTENT, emptyMap(), XContentType.JSON.mediaType(), EMPTY_BYTE); + } + return newError(RestStatus.NOT_FOUND, "object not found"); + }); + + // Insert Object (initialization) + // + // https://cloud.google.com/storage/docs/json_api/v1/objects/insert + handlers.insert("POST " + prefix + "/upload/storage/v1/b/{bucket}/o", (url, params, body) -> { + if ("resumable".equals(params.get("uploadType")) == false) { + return newError(RestStatus.INTERNAL_SERVER_ERROR, "upload type must be resumable"); + } + + final String objectName = params.get("name"); + if (Strings.hasText(objectName) == false) { + return newError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing"); + } + + final Bucket bucket = buckets.get(params.get("bucket")); + if (bucket == null) { + return newError(RestStatus.NOT_FOUND, "bucket not found"); + } + + if (bucket.objects.put(objectName, EMPTY_BYTE) == null) { + String location = endpoint + "/upload/storage/v1/b/" + bucket.name + "/o?uploadType=resumable&upload_id=" + objectName; + return new Response(RestStatus.CREATED, singletonMap("Location", location), XContentType.JSON.mediaType(), EMPTY_BYTE); + } else { + return newError(RestStatus.CONFLICT, "object already exist"); + } + }); + + // Insert Object (upload) + // + // https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload + handlers.insert("PUT " + prefix + "/upload/storage/v1/b/{bucket}/o", (url, params, body) -> { + String objectId = params.get("upload_id"); + if (Strings.hasText(objectId) == false) { + return newError(RestStatus.INTERNAL_SERVER_ERROR, "upload id is missing"); + } + + final Bucket bucket = buckets.get(params.get("bucket")); + if (bucket == null) { + return newError(RestStatus.NOT_FOUND, "bucket not found"); + } + + if (bucket.objects.containsKey(objectId) == false) { + return newError(RestStatus.NOT_FOUND, "object name not found"); + } + + bucket.objects.put(objectId, body); + return newResponse(RestStatus.OK, emptyMap(), buildObjectResource(bucket.name, objectId, body)); + }); + + // Copy Object + // + // https://cloud.google.com/storage/docs/json_api/v1/objects/copy + handlers.insert("POST " + prefix + "/storage/v1/b/{srcBucket}/o/{src}/copyTo/b/{destBucket}/o/{dest}", (url, params, body) -> { + String source = params.get("src"); + if (Strings.hasText(source) == false) { + return newError(RestStatus.INTERNAL_SERVER_ERROR, "source object name is missing"); + } + + final Bucket srcBucket = buckets.get(params.get("srcBucket")); + if (srcBucket == null) { + return newError(RestStatus.NOT_FOUND, "source bucket not found"); + } + + String dest = params.get("dest"); + if (Strings.hasText(dest) == false) { + return newError(RestStatus.INTERNAL_SERVER_ERROR, "destination object name is missing"); + } + + final Bucket destBucket = buckets.get(params.get("destBucket")); + if (destBucket == null) { + return newError(RestStatus.NOT_FOUND, "destination bucket not found"); + } + + final byte[] sourceBytes = srcBucket.objects.get(source); + if (sourceBytes == null) { + return newError(RestStatus.NOT_FOUND, "source object not found"); + } + + destBucket.objects.put(dest, sourceBytes); + return newResponse(RestStatus.OK, emptyMap(), buildObjectResource(destBucket.name, dest, sourceBytes)); + }); + + // List Objects + // + // https://cloud.google.com/storage/docs/json_api/v1/objects/list + handlers.insert("GET " + prefix + "/storage/v1/b/{bucket}/o", (url, params, body) -> { + final Bucket bucket = buckets.get(params.get("bucket")); + if (bucket == null) { + return newError(RestStatus.NOT_FOUND, "bucket not found"); + } + + final XContentBuilder builder = jsonBuilder(); + builder.startObject(); + builder.field("kind", "storage#objects"); + { + builder.startArray("items"); + + final String prefixParam = params.get("prefix"); + for (Map.Entry object : bucket.objects.entrySet()) { + if (prefixParam != null && object.getKey().startsWith(prefixParam) == false) { + continue; + } + buildObjectResource(builder, bucket.name, object.getKey(), object.getValue()); + } + builder.endArray(); + } + builder.endObject(); + return newResponse(RestStatus.OK, emptyMap(), builder); + }); + + // Download Object + // + // https://cloud.google.com/storage/docs/request-body + handlers.insert("GET " + prefix + "/download/storage/v1/b/{bucket}/o/{object}", (url, params, body) -> { + String object = params.get("object"); + if (Strings.hasText(object) == false) { + return newError(RestStatus.INTERNAL_SERVER_ERROR, "object id is missing"); + } + + final Bucket bucket = buckets.get(params.get("bucket")); + if (bucket == null) { + return newError(RestStatus.NOT_FOUND, "bucket not found"); + } + + if (bucket.objects.containsKey(object) == false) { + return newError(RestStatus.NOT_FOUND, "object name not found"); + } + + return new Response(RestStatus.OK, emptyMap(), "application/octet-stream", bucket.objects.get(object)); + }); + + // Batch + // + // https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch + handlers.insert("POST " + prefix + "/batch", (url, params, req) -> { + final List batchedResponses = new ArrayList<>(); + + // A batch request body looks like this: + // + // --__END_OF_PART__ + // Content-Length: 71 + // Content-Type: application/http + // content-id: 1 + // content-transfer-encoding: binary + // + // DELETE https://www.googleapis.com/storage/v1/b/ohifkgu/o/foo%2Ftest HTTP/1.1 + // + // + // --__END_OF_PART__ + // Content-Length: 71 + // Content-Type: application/http + // content-id: 2 + // content-transfer-encoding: binary + // + // DELETE https://www.googleapis.com/storage/v1/b/ohifkgu/o/bar%2Ftest HTTP/1.1 + // + // + // --__END_OF_PART__-- + + // Here we simply process the request body line by line and delegate to other handlers + // if possible. + Streams.readAllLines(new BufferedInputStream(new ByteArrayInputStream(req)), line -> { + final int indexOfHttp = line.indexOf(" HTTP/1.1"); + if (indexOfHttp > 0) { + line = line.substring(0, indexOfHttp); + } + + RequestHandler handler = handlers.retrieve(line, params); + if (handler != null) { + try { + batchedResponses.add(handler.execute(line, params, req)); + } catch (IOException e) { + batchedResponses.add(newError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); + } + } + }); + + // Now we can build the response + String boundary = "__END_OF_PART__"; + String sep = "--"; + String line = "\r\n"; + + StringBuilder builder = new StringBuilder(); + for (Response response : batchedResponses) { + builder.append(sep).append(boundary).append(line); + builder.append(line); + builder.append("HTTP/1.1 ").append(response.status.getStatus()); + builder.append(' ').append(response.status.toString()); + builder.append(line); + builder.append("Content-Length: ").append(response.body.length).append(line); + builder.append(line); + } + builder.append(line); + builder.append(sep).append(boundary).append(sep); + + byte[] content = builder.toString().getBytes(StandardCharsets.UTF_8); + return new Response(RestStatus.OK, emptyMap(), "multipart/mixed; boundary=" + boundary, content); + }); + + return handlers; + } + + /** + * Represents a Storage bucket as if it was created on Google Cloud Storage. + */ + static class Bucket { + + /** Bucket name **/ + final String name; + + /** Blobs contained in the bucket **/ + final Map objects; + + Bucket(final String name) { + this.name = Objects.requireNonNull(name); + this.objects = ConcurrentCollections.newConcurrentMap(); + } + } + + /** + * Represents a Storage HTTP Response. + */ + static class Response { + + final RestStatus status; + final Map headers; + final String contentType; + final byte[] body; + + Response(final RestStatus status, final Map headers, final String contentType, final byte[] body) { + this.status = Objects.requireNonNull(status); + this.headers = Objects.requireNonNull(headers); + this.contentType = Objects.requireNonNull(contentType); + this.body = Objects.requireNonNull(body); + } + } + + /** + * Builds a JSON response + */ + private static Response newResponse(final RestStatus status, final Map headers, final XContentBuilder xContentBuilder) { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + xContentBuilder.bytes().writeTo(out); + return new Response(status, headers, XContentType.JSON.mediaType(), out.toByteArray()); + } catch (IOException e) { + return newError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + + /** + * Storage Error JSON representation + */ + private static Response newError(final RestStatus status, final String message) { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + try (XContentBuilder builder = jsonBuilder()) { + builder.startObject() + .startObject("error") + .field("code", status.getStatus()) + .field("message", message) + .startArray("errors") + .startObject() + .field("domain", "global") + .field("reason", status.toString()) + .field("message", message) + .endObject() + .endArray() + .endObject() + .endObject(); + builder.bytes().writeTo(out); + } + return new Response(status, emptyMap(), XContentType.JSON.mediaType(), out.toByteArray()); + } catch (IOException e) { + byte[] bytes = (message != null ? message : "something went wrong").getBytes(StandardCharsets.UTF_8); + return new Response(RestStatus.INTERNAL_SERVER_ERROR, emptyMap(), " text/plain", bytes); + } + } + + /** + * Storage Bucket JSON representation as defined in + * https://cloud.google.com/storage/docs/json_api/v1/bucket#resource + */ + private static XContentBuilder buildBucketResource(final String name) throws IOException { + return jsonBuilder().startObject() + .field("kind", "storage#bucket") + .field("id", name) + .endObject(); + } + + /** + * Storage Object JSON representation as defined in + * https://cloud.google.com/storage/docs/json_api/v1/objects#resource + */ + private static XContentBuilder buildObjectResource(final String bucket, final String name, final byte[] bytes) + throws IOException { + return buildObjectResource(jsonBuilder(), bucket, name, bytes); + } + + /** + * Storage Object JSON representation as defined in + * https://cloud.google.com/storage/docs/json_api/v1/objects#resource + */ + private static XContentBuilder buildObjectResource(final XContentBuilder builder, + final String bucket, + final String name, + final byte[] bytes) throws IOException { + return builder.startObject() + .field("kind", "storage#object") + .field("id", String.join("/", bucket, name)) + .field("name", name) + .field("size", String.valueOf(bytes.length)) + .endObject(); + } +} diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpTransport.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpTransport.java deleted file mode 100644 index f09854458cc..00000000000 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpTransport.java +++ /dev/null @@ -1,433 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.gcs; - -import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.LowLevelHttpRequest; -import com.google.api.client.http.LowLevelHttpResponse; -import com.google.api.client.json.Json; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.client.testing.http.MockLowLevelHttpRequest; -import com.google.api.client.testing.http.MockLowLevelHttpResponse; -import com.google.api.services.storage.Storage; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.path.PathTrie; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.rest.RestUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; - -/** - * Mock for {@link HttpTransport} to test Google Cloud Storage service. - *

- * This basically handles each type of request used by the {@link GoogleCloudStorageBlobStore} and provides appropriate responses like - * the Google Cloud Storage service would do. It is largely based on official documentation available at https://cloud.google - * .com/storage/docs/json_api/v1/. - */ -public class MockHttpTransport extends com.google.api.client.testing.http.MockHttpTransport { - - private final AtomicInteger objectsCount = new AtomicInteger(0); - private final Map objectsNames = ConcurrentCollections.newConcurrentMap(); - private final Map objectsContent = ConcurrentCollections.newConcurrentMap(); - - private final PathTrie handlers = new PathTrie<>(RestUtils.REST_DECODER); - - public MockHttpTransport(String bucket) { - - // GET Bucket - // - // https://cloud.google.com/storage/docs/json_api/v1/buckets/get - handlers.insert("GET https://www.googleapis.com/storage/v1/b/{bucket}", (url, params, req) -> { - String name = params.get("bucket"); - if (Strings.hasText(name) == false) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "bucket name is missing"); - } - - if (name.equals(bucket)) { - return newMockResponse().setContent(buildBucketResource(bucket)); - } else { - return newMockError(RestStatus.NOT_FOUND, "bucket not found"); - } - }); - - // GET Object - // - // https://cloud.google.com/storage/docs/json_api/v1/objects/get - handlers.insert("GET https://www.googleapis.com/storage/v1/b/{bucket}/o/{object}", (url, params, req) -> { - String name = params.get("object"); - if (Strings.hasText(name) == false) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing"); - } - - for (Map.Entry object : objectsNames.entrySet()) { - if (object.getValue().equals(name)) { - byte[] content = objectsContent.get(object.getKey()); - if (content != null) { - return newMockResponse().setContent(buildObjectResource(bucket, name, object.getKey(), content.length)); - } - } - } - return newMockError(RestStatus.NOT_FOUND, "object not found"); - }); - - // Download Object - // - // https://cloud.google.com/storage/docs/request-endpoints - handlers.insert("GET https://www.googleapis.com/download/storage/v1/b/{bucket}/o/{object}", (url, params, req) -> { - String name = params.get("object"); - if (Strings.hasText(name) == false) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing"); - } - - for (Map.Entry object : objectsNames.entrySet()) { - if (object.getValue().equals(name)) { - byte[] content = objectsContent.get(object.getKey()); - if (content == null) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object content is missing"); - } - return newMockResponse().setContent(new ByteArrayInputStream(content)); - } - } - return newMockError(RestStatus.NOT_FOUND, "object not found"); - }); - - // Insert Object (initialization) - // - // https://cloud.google.com/storage/docs/json_api/v1/objects/insert - handlers.insert("POST https://www.googleapis.com/upload/storage/v1/b/{bucket}/o", (url, params, req) -> { - if ("resumable".equals(params.get("uploadType")) == false) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "upload type must be resumable"); - } - - String name = params.get("name"); - if (Strings.hasText(name) == false) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing"); - } - - String objectId = String.valueOf(objectsCount.getAndIncrement()); - objectsNames.put(objectId, name); - - return newMockResponse() - .setStatusCode(RestStatus.CREATED.getStatus()) - .addHeader("Location", "https://www.googleapis.com/upload/storage/v1/b/" + bucket + - "/o?uploadType=resumable&upload_id=" + objectId); - }); - - // Insert Object (upload) - // - // https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload - handlers.insert("PUT https://www.googleapis.com/upload/storage/v1/b/{bucket}/o", (url, params, req) -> { - String objectId = params.get("upload_id"); - if (Strings.hasText(objectId) == false) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "upload id is missing"); - } - - String name = objectsNames.get(objectId); - if (Strings.hasText(name) == false) { - return newMockError(RestStatus.NOT_FOUND, "object name not found"); - } - - ByteArrayOutputStream os = new ByteArrayOutputStream((int) req.getContentLength()); - try { - req.getStreamingContent().writeTo(os); - os.close(); - } catch (IOException e) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage()); - } - - byte[] content = os.toByteArray(); - objectsContent.put(objectId, content); - return newMockResponse().setContent(buildObjectResource(bucket, name, objectId, content.length)); - }); - - // List Objects - // - // https://cloud.google.com/storage/docs/json_api/v1/objects/list - handlers.insert("GET https://www.googleapis.com/storage/v1/b/{bucket}/o", (url, params, req) -> { - String prefix = params.get("prefix"); - - try (XContentBuilder builder = jsonBuilder()) { - builder.startObject(); - builder.field("kind", "storage#objects"); - builder.startArray("items"); - for (Map.Entry o : objectsNames.entrySet()) { - if (prefix != null && o.getValue().startsWith(prefix) == false) { - continue; - } - buildObjectResource(builder, bucket, o.getValue(), o.getKey(), objectsContent.get(o.getKey()).length); - } - builder.endArray(); - builder.endObject(); - return newMockResponse().setContent(builder.string()); - } catch (IOException e) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage()); - } - }); - - // Delete Object - // - // https://cloud.google.com/storage/docs/json_api/v1/objects/delete - handlers.insert("DELETE https://www.googleapis.com/storage/v1/b/{bucket}/o/{object}", (url, params, req) -> { - String name = params.get("object"); - if (Strings.hasText(name) == false) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing"); - } - - String objectId = null; - for (Map.Entry object : objectsNames.entrySet()) { - if (object.getValue().equals(name)) { - objectId = object.getKey(); - break; - } - } - - if (objectId != null) { - objectsNames.remove(objectId); - objectsContent.remove(objectId); - return newMockResponse().setStatusCode(RestStatus.NO_CONTENT.getStatus()); - } - return newMockError(RestStatus.NOT_FOUND, "object not found"); - }); - - // Copy Object - // - // https://cloud.google.com/storage/docs/json_api/v1/objects/copy - handlers.insert("POST https://www.googleapis.com/storage/v1/b/{srcBucket}/o/{srcObject}/copyTo/b/{destBucket}/o/{destObject}", - (url, params, req) -> { - String source = params.get("srcObject"); - if (Strings.hasText(source) == false) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "source object name is missing"); - } - - String dest = params.get("destObject"); - if (Strings.hasText(dest) == false) { - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "destination object name is missing"); - } - - String srcObjectId = null; - for (Map.Entry object : objectsNames.entrySet()) { - if (object.getValue().equals(source)) { - srcObjectId = object.getKey(); - break; - } - } - - if (srcObjectId == null) { - return newMockError(RestStatus.NOT_FOUND, "source object not found"); - } - - byte[] content = objectsContent.get(srcObjectId); - if (content == null) { - return newMockError(RestStatus.NOT_FOUND, "source content can not be found"); - } - - String destObjectId = String.valueOf(objectsCount.getAndIncrement()); - objectsNames.put(destObjectId, dest); - objectsContent.put(destObjectId, content); - - return newMockResponse().setContent(buildObjectResource(bucket, dest, destObjectId, content.length)); - }); - - // Batch - // - // https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch - handlers.insert("POST https://www.googleapis.com/batch", (url, params, req) -> { - List responses = new ArrayList<>(); - - // A batch request body looks like this: - // - // --__END_OF_PART__ - // Content-Length: 71 - // Content-Type: application/http - // content-id: 1 - // content-transfer-encoding: binary - // - // DELETE https://www.googleapis.com/storage/v1/b/ohifkgu/o/foo%2Ftest HTTP/1.1 - // - // - // --__END_OF_PART__ - // Content-Length: 71 - // Content-Type: application/http - // content-id: 2 - // content-transfer-encoding: binary - // - // DELETE https://www.googleapis.com/storage/v1/b/ohifkgu/o/bar%2Ftest HTTP/1.1 - // - // - // --__END_OF_PART__-- - - // Here we simply process the request body line by line and delegate to other handlers - // if possible. - try (ByteArrayOutputStream os = new ByteArrayOutputStream((int) req.getContentLength())) { - req.getStreamingContent().writeTo(os); - - Streams.readAllLines(new ByteArrayInputStream(os.toByteArray()), line -> { - final int indexOfHttp = line.indexOf(" HTTP/1.1"); - if (indexOfHttp > 0) { - line = line.substring(0, indexOfHttp); - } - - Handler handler = handlers.retrieve(line, params); - if (handler != null) { - try { - responses.add(handler.execute(line, params, req)); - } catch (IOException e) { - responses.add(newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage())); - } - } - }); - } - - // Now we can build the response - String boundary = "__END_OF_PART__"; - String sep = "--"; - String line = "\r\n"; - - StringBuilder builder = new StringBuilder(); - for (MockLowLevelHttpResponse resp : responses) { - builder.append(sep).append(boundary).append(line); - builder.append(line); - builder.append("HTTP/1.1 ").append(resp.getStatusCode()).append(' ').append(resp.getReasonPhrase()).append(line); - builder.append("Content-Length: ").append(resp.getContentLength()).append(line); - builder.append(line); - } - builder.append(line); - builder.append(sep).append(boundary).append(sep); - - return newMockResponse().setContentType("multipart/mixed; boundary=" + boundary).setContent(builder.toString()); - }); - } - - @Override - public LowLevelHttpRequest buildRequest(String method, String url) throws IOException { - return new MockLowLevelHttpRequest() { - @Override - public LowLevelHttpResponse execute() throws IOException { - String rawPath = url; - Map params = new HashMap<>(); - - int pathEndPos = url.indexOf('?'); - if (pathEndPos != -1) { - rawPath = url.substring(0, pathEndPos); - RestUtils.decodeQueryString(url, pathEndPos + 1, params); - } - - Handler handler = handlers.retrieve(method + " " + rawPath, params); - if (handler != null) { - return handler.execute(rawPath, params, this); - } - return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "Unable to handle request [method=" + method + ", url=" + url + "]"); - } - }; - } - - private static MockLowLevelHttpResponse newMockResponse() { - return new MockLowLevelHttpResponse() - .setContentType(Json.MEDIA_TYPE) - .setStatusCode(RestStatus.OK.getStatus()) - .setReasonPhrase(RestStatus.OK.name()); - } - - private static MockLowLevelHttpResponse newMockError(RestStatus status, String message) { - MockLowLevelHttpResponse response = newMockResponse().setStatusCode(status.getStatus()).setReasonPhrase(status.name()); - try { - response.setContent(buildErrorResource(status, message)); - } catch (IOException e) { - response.setContent("Failed to build error resource [" + message + "] because of: " + e.getMessage()); - } - return response; - } - - /** - * Storage Error JSON representation - */ - private static String buildErrorResource(RestStatus status, String message) throws IOException { - return jsonBuilder() - .startObject() - .startObject("error") - .field("code", status.getStatus()) - .field("message", message) - .startArray("errors") - .startObject() - .field("domain", "global") - .field("reason", status.toString()) - .field("message", message) - .endObject() - .endArray() - .endObject() - .endObject() - .string(); - } - - /** - * Storage Bucket JSON representation as defined in - * https://cloud.google.com/storage/docs/json_api/v1/bucket#resource - */ - private static String buildBucketResource(String name) throws IOException { - return jsonBuilder().startObject() - .field("kind", "storage#bucket") - .field("id", name) - .endObject() - .string(); - } - - /** - * Storage Object JSON representation as defined in - * https://cloud.google.com/storage/docs/json_api/v1/objects#resource - */ - private static XContentBuilder buildObjectResource(XContentBuilder builder, String bucket, String name, String id, int size) - throws IOException { - return builder.startObject() - .field("kind", "storage#object") - .field("id", String.join("/", bucket, name, id)) - .field("name", name) - .field("size", String.valueOf(size)) - .endObject(); - } - - private static String buildObjectResource(String bucket, String name, String id, int size) throws IOException { - return buildObjectResource(jsonBuilder(), bucket, name, id, size).string(); - } - - interface Handler { - MockLowLevelHttpResponse execute(String url, Map params, MockLowLevelHttpRequest request) throws IOException; - } - - /** - * Instanciates a mocked Storage client for tests. - */ - public static Storage newStorage(String bucket, String applicationName) { - return new Storage.Builder(new MockHttpTransport(bucket), new JacksonFactory(), null) - .setApplicationName(applicationName) - .build(); - } -} diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java new file mode 100644 index 00000000000..8be7511ab58 --- /dev/null +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.gcs; + +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.api.services.storage.Storage; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; + +/** + * {@link MockStorage} is a utility class that provides {@link Storage} clients that works + * against an embedded {@link GoogleCloudStorageTestServer}. + */ +class MockStorage extends com.google.api.client.testing.http.MockHttpTransport { + + /** + * Embedded test server that emulates a Google Cloud Storage service + **/ + private final GoogleCloudStorageTestServer server = new GoogleCloudStorageTestServer(); + + private MockStorage() { + } + + @Override + public LowLevelHttpRequest buildRequest(String method, String url) throws IOException { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() throws IOException { + final GoogleCloudStorageTestServer.Response response = server.handle(method, url, getContentAsBytes()); + return convert(response); + } + + /** Returns the LowLevelHttpRequest body as an array of bytes **/ + byte[] getContentAsBytes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + if (getStreamingContent() != null) { + getStreamingContent().writeTo(out); + } + return out.toByteArray(); + } + }; + } + + private static MockLowLevelHttpResponse convert(final GoogleCloudStorageTestServer.Response response) { + final MockLowLevelHttpResponse lowLevelHttpResponse = new MockLowLevelHttpResponse(); + for (Map.Entry header : response.headers.entrySet()) { + lowLevelHttpResponse.addHeader(header.getKey(), header.getValue()); + } + lowLevelHttpResponse.setContentType(response.contentType); + lowLevelHttpResponse.setStatusCode(response.status.getStatus()); + lowLevelHttpResponse.setReasonPhrase(response.status.toString()); + if (response.body != null) { + lowLevelHttpResponse.setContent(response.body); + lowLevelHttpResponse.setContentLength(response.body.length); + } + return lowLevelHttpResponse; + } + + /** + * Instanciates a mocked Storage client for tests. + */ + public static Storage newStorageClient(final String bucket, final String applicationName) { + MockStorage mockStorage = new MockStorage(); + mockStorage.server.createBucket(bucket); + + return new Storage.Builder(mockStorage, JacksonFactory.getDefaultInstance(), null) + .setApplicationName(applicationName) + .build(); + } +}