honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-17 02:14:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Tests] Extract the testing logic for Google Cloud Storage (#28576)

Browse Source 
This pull request extracts in a dedicated class the request/response 
logic that "emulates" a Google Cloud Storage service in our 
repository-gcs tests.

The idea behind this is to make the logic more reusable. The class 
MockHttpTransport has been renamed to MockStorage which now 
only takes care of instantiating a Storage client and does the low-level 
request/response plumbing needed by this client.

The "Google Cloud Storage" logic has been extracted from 
MockHttpTransport and put in a new GoogleCloudStorageTestServer 
that is now independent from the google client testing framework.
This commit is contained in:
Tanguy Leroux 2018-02-21 13:20:35 +01:00 committed by GitHub
parent 793cbc651a
commit 9a95be35cf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 594 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java
View File

@ -23,13 +23,13 @@ import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
import java.io.IOException;
import java.util.Locale;
public class GoogleCloudStorageBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
@Override
protected BlobStore newBlobStore() throws IOException {
protected BlobStore newBlobStore() {
String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockHttpTransport.newStorage(bucket, getTestName()));
return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockStorage.newStorageClient(bucket, getTestName()));
}
}

2
plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
View File

@ -65,7 +65,7 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepos
@BeforeClass
public static void setUpStorage() {
storage.set(MockHttpTransport.newStorage(BUCKET, GoogleCloudStorageBlobStoreRepositoryTests.class.getName()));
storage.set(MockStorage.newStorageClient(BUCKET, GoogleCloudStorageBlobStoreRepositoryTests.class.getName()));
}
public static class MockGoogleCloudStoragePlugin extends GoogleCloudStoragePlugin {

5
plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java
View File

@ -23,14 +23,13 @@ import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.ESBlobStoreTestCase;
import java.io.IOException;
import java.util.Locale;
public class GoogleCloudStorageBlobStoreTests extends ESBlobStoreTestCase {
@Override
protected BlobStore newBlobStore() throws IOException {
protected BlobStore newBlobStore() {
String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockHttpTransport.newStorage(bucket, getTestName()));
return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockStorage.newStorageClient(bucket, getTestName()));
}
}

495
plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestServer.java Normal file
View File

@ -0,0 +1,495 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.gcs;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.path.PathTrie;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.rest.RestUtils;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
/**
* {@link GoogleCloudStorageTestServer} emulates a Google Cloud Storage service through a {@link #handle(String, String, byte[])} method
* that provides appropriate responses for specific requests like the real Google Cloud platform would do. It is largely based on official
* documentation available at https://cloud.google.com/storage/docs/json_api/v1/.
*/
public class GoogleCloudStorageTestServer {
private static byte[] EMPTY_BYTE = new byte[0];
/** List of the buckets stored on this test server **/
private final Map<String, Bucket> buckets = ConcurrentCollections.newConcurrentMap();
/** Request handlers for the requests made by the Google Cloud Storage client **/
private final PathTrie<RequestHandler> handlers;
/**
* Creates a {@link GoogleCloudStorageTestServer} with the default endpoint
*/
GoogleCloudStorageTestServer() {
this("https://www.googleapis.com", true);
}
/**
* Creates a {@link GoogleCloudStorageTestServer} with a custom endpoint,
* potentially prefixing the URL patterns to match with the endpoint name.
*/
GoogleCloudStorageTestServer(final String endpoint, final boolean prefixWithEndpoint) {
this.handlers = defaultHandlers(endpoint, prefixWithEndpoint, buckets);
}
/** Creates a bucket in the test server **/
void createBucket(final String bucketName) {
buckets.put(bucketName, new Bucket(bucketName));
}
public Response handle(final String method, final String url, byte[] content) throws IOException {
final Map<String, String> params = new HashMap<>();
// Splits the URL to extract query string parameters
final String rawPath;
int questionMark = url.indexOf('?');
if (questionMark != -1) {
rawPath = url.substring(0, questionMark);
RestUtils.decodeQueryString(url, questionMark + 1, params);
} else {
rawPath = url;
}
final RequestHandler handler = handlers.retrieve(method + " " + rawPath, params);
if (handler != null) {
return handler.execute(url, params, content);
} else {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "No handler defined for request [method: " + method + ", url: " + url + "]");
}
}
@FunctionalInterface
interface RequestHandler {
/**
* Simulates the execution of a Storage request and returns a corresponding response.
*
* @param url the request URL
* @param params the request URL parameters
* @param body the request body provided as a byte array
* @return the corresponding response
*
* @throws IOException if something goes wrong
*/
Response execute(String url, Map<String, String> params, byte[] body) throws IOException;
}
/** Builds the default request handlers **/
private static PathTrie<RequestHandler> defaultHandlers(final String endpoint,
final boolean prefixWithEndpoint,
final Map<String, Bucket> buckets) {
final PathTrie<RequestHandler> handlers = new PathTrie<>(RestUtils.REST_DECODER);
final String prefix = prefixWithEndpoint ? endpoint : "";
// GET Bucket
//
// https://cloud.google.com/storage/docs/json_api/v1/buckets/get
handlers.insert("GET " + prefix + "/storage/v1/b/{bucket}", (url, params, body) -> {
String name = params.get("bucket");
if (Strings.hasText(name) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "bucket name is missing");
}
if (buckets.containsKey(name)) {
return newResponse(RestStatus.OK, emptyMap(), buildBucketResource(name));
} else {
return newError(RestStatus.NOT_FOUND, "bucket not found");
}
});
// GET Object
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/get
handlers.insert("GET " + prefix + "/storage/v1/b/{bucket}/o/{object}", (url, params, body) -> {
String objectName = params.get("object");
if (Strings.hasText(objectName) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
}
final Bucket bucket = buckets.get(params.get("bucket"));
if (bucket == null) {
return newError(RestStatus.NOT_FOUND, "bucket not found");
}
for (Map.Entry<String, byte[]> object : bucket.objects.entrySet()) {
if (object.getKey().equals(objectName)) {
return newResponse(RestStatus.OK, emptyMap(), buildObjectResource(bucket.name, objectName, object.getValue()));
}
}
return newError(RestStatus.NOT_FOUND, "object not found");
});
// Delete Object
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/delete
handlers.insert("DELETE " + prefix + "/storage/v1/b/{bucket}/o/{object}", (url, params, body) -> {
String objectName = params.get("object");
if (Strings.hasText(objectName) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
}
final Bucket bucket = buckets.get(params.get("bucket"));
if (bucket == null) {
return newError(RestStatus.NOT_FOUND, "bucket not found");
}
final byte[] bytes = bucket.objects.remove(objectName);
if (bytes != null) {
return new Response(RestStatus.NO_CONTENT, emptyMap(), XContentType.JSON.mediaType(), EMPTY_BYTE);
}
return newError(RestStatus.NOT_FOUND, "object not found");
});
// Insert Object (initialization)
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/insert
handlers.insert("POST " + prefix + "/upload/storage/v1/b/{bucket}/o", (url, params, body) -> {
if ("resumable".equals(params.get("uploadType")) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "upload type must be resumable");
}
final String objectName = params.get("name");
if (Strings.hasText(objectName) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
}
final Bucket bucket = buckets.get(params.get("bucket"));
if (bucket == null) {
return newError(RestStatus.NOT_FOUND, "bucket not found");
}
if (bucket.objects.put(objectName, EMPTY_BYTE) == null) {
String location = endpoint + "/upload/storage/v1/b/" + bucket.name + "/o?uploadType=resumable&upload_id=" + objectName;
return new Response(RestStatus.CREATED, singletonMap("Location", location), XContentType.JSON.mediaType(), EMPTY_BYTE);
} else {
return newError(RestStatus.CONFLICT, "object already exist");
}
});
// Insert Object (upload)
//
// https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload
handlers.insert("PUT " + prefix + "/upload/storage/v1/b/{bucket}/o", (url, params, body) -> {
String objectId = params.get("upload_id");
if (Strings.hasText(objectId) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "upload id is missing");
}
final Bucket bucket = buckets.get(params.get("bucket"));
if (bucket == null) {
return newError(RestStatus.NOT_FOUND, "bucket not found");
}
if (bucket.objects.containsKey(objectId) == false) {
return newError(RestStatus.NOT_FOUND, "object name not found");
}
bucket.objects.put(objectId, body);
return newResponse(RestStatus.OK, emptyMap(), buildObjectResource(bucket.name, objectId, body));
});
// Copy Object
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/copy
handlers.insert("POST " + prefix + "/storage/v1/b/{srcBucket}/o/{src}/copyTo/b/{destBucket}/o/{dest}", (url, params, body) -> {
String source = params.get("src");
if (Strings.hasText(source) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "source object name is missing");
}
final Bucket srcBucket = buckets.get(params.get("srcBucket"));
if (srcBucket == null) {
return newError(RestStatus.NOT_FOUND, "source bucket not found");
}
String dest = params.get("dest");
if (Strings.hasText(dest) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "destination object name is missing");
}
final Bucket destBucket = buckets.get(params.get("destBucket"));
if (destBucket == null) {
return newError(RestStatus.NOT_FOUND, "destination bucket not found");
}
final byte[] sourceBytes = srcBucket.objects.get(source);
if (sourceBytes == null) {
return newError(RestStatus.NOT_FOUND, "source object not found");
}
destBucket.objects.put(dest, sourceBytes);
return newResponse(RestStatus.OK, emptyMap(), buildObjectResource(destBucket.name, dest, sourceBytes));
});
// List Objects
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/list
handlers.insert("GET " + prefix + "/storage/v1/b/{bucket}/o", (url, params, body) -> {
final Bucket bucket = buckets.get(params.get("bucket"));
if (bucket == null) {
return newError(RestStatus.NOT_FOUND, "bucket not found");
}
final XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.field("kind", "storage#objects");
{
builder.startArray("items");
final String prefixParam = params.get("prefix");
for (Map.Entry<String, byte[]> object : bucket.objects.entrySet()) {
if (prefixParam != null && object.getKey().startsWith(prefixParam) == false) {
continue;
}
buildObjectResource(builder, bucket.name, object.getKey(), object.getValue());
}
builder.endArray();
}
builder.endObject();
return newResponse(RestStatus.OK, emptyMap(), builder);
});
// Download Object
//
// https://cloud.google.com/storage/docs/request-body
handlers.insert("GET " + prefix + "/download/storage/v1/b/{bucket}/o/{object}", (url, params, body) -> {
String object = params.get("object");
if (Strings.hasText(object) == false) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, "object id is missing");
}
final Bucket bucket = buckets.get(params.get("bucket"));
if (bucket == null) {
return newError(RestStatus.NOT_FOUND, "bucket not found");
}
if (bucket.objects.containsKey(object) == false) {
return newError(RestStatus.NOT_FOUND, "object name not found");
}
return new Response(RestStatus.OK, emptyMap(), "application/octet-stream", bucket.objects.get(object));
});
// Batch
//
// https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
handlers.insert("POST " + prefix + "/batch", (url, params, req) -> {
final List<Response> batchedResponses = new ArrayList<>();
// A batch request body looks like this:
//
// --__END_OF_PART__
// Content-Length: 71
// Content-Type: application/http
// content-id: 1
// content-transfer-encoding: binary
//
// DELETE https://www.googleapis.com/storage/v1/b/ohifkgu/o/foo%2Ftest HTTP/1.1
//
//
// --__END_OF_PART__
// Content-Length: 71
// Content-Type: application/http
// content-id: 2
// content-transfer-encoding: binary
//
// DELETE https://www.googleapis.com/storage/v1/b/ohifkgu/o/bar%2Ftest HTTP/1.1
//
//
// --__END_OF_PART__--
// Here we simply process the request body line by line and delegate to other handlers
// if possible.
Streams.readAllLines(new BufferedInputStream(new ByteArrayInputStream(req)), line -> {
final int indexOfHttp = line.indexOf(" HTTP/1.1");
if (indexOfHttp > 0) {
line = line.substring(0, indexOfHttp);
}
RequestHandler handler = handlers.retrieve(line, params);
if (handler != null) {
try {
batchedResponses.add(handler.execute(line, params, req));
} catch (IOException e) {
batchedResponses.add(newError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage()));
}
}
});
// Now we can build the response
String boundary = "__END_OF_PART__";
String sep = "--";
String line = "\r\n";
StringBuilder builder = new StringBuilder();
for (Response response : batchedResponses) {
builder.append(sep).append(boundary).append(line);
builder.append(line);
builder.append("HTTP/1.1 ").append(response.status.getStatus());
builder.append(' ').append(response.status.toString());
builder.append(line);
builder.append("Content-Length: ").append(response.body.length).append(line);
builder.append(line);
}
builder.append(line);
builder.append(sep).append(boundary).append(sep);
byte[] content = builder.toString().getBytes(StandardCharsets.UTF_8);
return new Response(RestStatus.OK, emptyMap(), "multipart/mixed; boundary=" + boundary, content);
});
return handlers;
}
/**
* Represents a Storage bucket as if it was created on Google Cloud Storage.
*/
static class Bucket {
/** Bucket name **/
final String name;
/** Blobs contained in the bucket **/
final Map<String, byte[]> objects;
Bucket(final String name) {
this.name = Objects.requireNonNull(name);
this.objects = ConcurrentCollections.newConcurrentMap();
}
}
/**
* Represents a Storage HTTP Response.
*/
static class Response {
final RestStatus status;
final Map<String, String> headers;
final String contentType;
final byte[] body;
Response(final RestStatus status, final Map<String, String> headers, final String contentType, final byte[] body) {
this.status = Objects.requireNonNull(status);
this.headers = Objects.requireNonNull(headers);
this.contentType = Objects.requireNonNull(contentType);
this.body = Objects.requireNonNull(body);
}
}
/**
* Builds a JSON response
*/
private static Response newResponse(final RestStatus status, final Map<String, String> headers, final XContentBuilder xContentBuilder) {
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
xContentBuilder.bytes().writeTo(out);
return new Response(status, headers, XContentType.JSON.mediaType(), out.toByteArray());
} catch (IOException e) {
return newError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage());
}
}
/**
* Storage Error JSON representation
*/
private static Response newError(final RestStatus status, final String message) {
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
try (XContentBuilder builder = jsonBuilder()) {
builder.startObject()
.startObject("error")
.field("code", status.getStatus())
.field("message", message)
.startArray("errors")
.startObject()
.field("domain", "global")
.field("reason", status.toString())
.field("message", message)
.endObject()
.endArray()
.endObject()
.endObject();
builder.bytes().writeTo(out);
}
return new Response(status, emptyMap(), XContentType.JSON.mediaType(), out.toByteArray());
} catch (IOException e) {
byte[] bytes = (message != null ? message : "something went wrong").getBytes(StandardCharsets.UTF_8);
return new Response(RestStatus.INTERNAL_SERVER_ERROR, emptyMap(), " text/plain", bytes);
}
}
/**
* Storage Bucket JSON representation as defined in
* https://cloud.google.com/storage/docs/json_api/v1/bucket#resource
*/
private static XContentBuilder buildBucketResource(final String name) throws IOException {
return jsonBuilder().startObject()
.field("kind", "storage#bucket")
.field("id", name)
.endObject();
}
/**
* Storage Object JSON representation as defined in
* https://cloud.google.com/storage/docs/json_api/v1/objects#resource
*/
private static XContentBuilder buildObjectResource(final String bucket, final String name, final byte[] bytes)
throws IOException {
return buildObjectResource(jsonBuilder(), bucket, name, bytes);
}
/**
* Storage Object JSON representation as defined in
* https://cloud.google.com/storage/docs/json_api/v1/objects#resource
*/
private static XContentBuilder buildObjectResource(final XContentBuilder builder,
final String bucket,
final String name,
final byte[] bytes) throws IOException {
return builder.startObject()
.field("kind", "storage#object")
.field("id", String.join("/", bucket, name))
.field("name", name)
.field("size", String.valueOf(bytes.length))
.endObject();
}
}

433
plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockHttpTransport.java
View File

@ -1,433 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.gcs;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.http.LowLevelHttpRequest;
import com.google.api.client.http.LowLevelHttpResponse;
import com.google.api.client.json.Json;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.testing.http.MockLowLevelHttpRequest;
import com.google.api.client.testing.http.MockLowLevelHttpResponse;
import com.google.api.services.storage.Storage;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.path.PathTrie;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.rest.RestUtils;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
/**
* Mock for {@link HttpTransport} to test Google Cloud Storage service.
* <p>
* This basically handles each type of request used by the {@link GoogleCloudStorageBlobStore} and provides appropriate responses like
* the Google Cloud Storage service would do. It is largely based on official documentation available at https://cloud.google
* .com/storage/docs/json_api/v1/.
*/
public class MockHttpTransport extends com.google.api.client.testing.http.MockHttpTransport {
private final AtomicInteger objectsCount = new AtomicInteger(0);
private final Map<String, String> objectsNames = ConcurrentCollections.newConcurrentMap();
private final Map<String, byte[]> objectsContent = ConcurrentCollections.newConcurrentMap();
private final PathTrie<Handler> handlers = new PathTrie<>(RestUtils.REST_DECODER);
public MockHttpTransport(String bucket) {
// GET Bucket
//
// https://cloud.google.com/storage/docs/json_api/v1/buckets/get
handlers.insert("GET https://www.googleapis.com/storage/v1/b/{bucket}", (url, params, req) -> {
String name = params.get("bucket");
if (Strings.hasText(name) == false) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "bucket name is missing");
}
if (name.equals(bucket)) {
return newMockResponse().setContent(buildBucketResource(bucket));
} else {
return newMockError(RestStatus.NOT_FOUND, "bucket not found");
}
});
// GET Object
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/get
handlers.insert("GET https://www.googleapis.com/storage/v1/b/{bucket}/o/{object}", (url, params, req) -> {
String name = params.get("object");
if (Strings.hasText(name) == false) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
}
for (Map.Entry<String, String> object : objectsNames.entrySet()) {
if (object.getValue().equals(name)) {
byte[] content = objectsContent.get(object.getKey());
if (content != null) {
return newMockResponse().setContent(buildObjectResource(bucket, name, object.getKey(), content.length));
}
}
}
return newMockError(RestStatus.NOT_FOUND, "object not found");
});
// Download Object
//
// https://cloud.google.com/storage/docs/request-endpoints
handlers.insert("GET https://www.googleapis.com/download/storage/v1/b/{bucket}/o/{object}", (url, params, req) -> {
String name = params.get("object");
if (Strings.hasText(name) == false) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
}
for (Map.Entry<String, String> object : objectsNames.entrySet()) {
if (object.getValue().equals(name)) {
byte[] content = objectsContent.get(object.getKey());
if (content == null) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object content is missing");
}
return newMockResponse().setContent(new ByteArrayInputStream(content));
}
}
return newMockError(RestStatus.NOT_FOUND, "object not found");
});
// Insert Object (initialization)
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/insert
handlers.insert("POST https://www.googleapis.com/upload/storage/v1/b/{bucket}/o", (url, params, req) -> {
if ("resumable".equals(params.get("uploadType")) == false) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "upload type must be resumable");
}
String name = params.get("name");
if (Strings.hasText(name) == false) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
}
String objectId = String.valueOf(objectsCount.getAndIncrement());
objectsNames.put(objectId, name);
return newMockResponse()
.setStatusCode(RestStatus.CREATED.getStatus())
.addHeader("Location", "https://www.googleapis.com/upload/storage/v1/b/" + bucket +
"/o?uploadType=resumable&upload_id=" + objectId);
});
// Insert Object (upload)
//
// https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload
handlers.insert("PUT https://www.googleapis.com/upload/storage/v1/b/{bucket}/o", (url, params, req) -> {
String objectId = params.get("upload_id");
if (Strings.hasText(objectId) == false) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "upload id is missing");
}
String name = objectsNames.get(objectId);
if (Strings.hasText(name) == false) {
return newMockError(RestStatus.NOT_FOUND, "object name not found");
}
ByteArrayOutputStream os = new ByteArrayOutputStream((int) req.getContentLength());
try {
req.getStreamingContent().writeTo(os);
os.close();
} catch (IOException e) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage());
}
byte[] content = os.toByteArray();
objectsContent.put(objectId, content);
return newMockResponse().setContent(buildObjectResource(bucket, name, objectId, content.length));
});
// List Objects
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/list
handlers.insert("GET https://www.googleapis.com/storage/v1/b/{bucket}/o", (url, params, req) -> {
String prefix = params.get("prefix");
try (XContentBuilder builder = jsonBuilder()) {
builder.startObject();
builder.field("kind", "storage#objects");
builder.startArray("items");
for (Map.Entry<String, String> o : objectsNames.entrySet()) {
if (prefix != null && o.getValue().startsWith(prefix) == false) {
continue;
}
buildObjectResource(builder, bucket, o.getValue(), o.getKey(), objectsContent.get(o.getKey()).length);
}
builder.endArray();
builder.endObject();
return newMockResponse().setContent(builder.string());
} catch (IOException e) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage());
}
});
// Delete Object
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/delete
handlers.insert("DELETE https://www.googleapis.com/storage/v1/b/{bucket}/o/{object}", (url, params, req) -> {
String name = params.get("object");
if (Strings.hasText(name) == false) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
}
String objectId = null;
for (Map.Entry<String, String> object : objectsNames.entrySet()) {
if (object.getValue().equals(name)) {
objectId = object.getKey();
break;
}
}
if (objectId != null) {
objectsNames.remove(objectId);
objectsContent.remove(objectId);
return newMockResponse().setStatusCode(RestStatus.NO_CONTENT.getStatus());
}
return newMockError(RestStatus.NOT_FOUND, "object not found");
});
// Copy Object
//
// https://cloud.google.com/storage/docs/json_api/v1/objects/copy
handlers.insert("POST https://www.googleapis.com/storage/v1/b/{srcBucket}/o/{srcObject}/copyTo/b/{destBucket}/o/{destObject}",
(url, params, req) -> {
String source = params.get("srcObject");
if (Strings.hasText(source) == false) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "source object name is missing");
}
String dest = params.get("destObject");
if (Strings.hasText(dest) == false) {
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "destination object name is missing");
}
String srcObjectId = null;
for (Map.Entry<String, String> object : objectsNames.entrySet()) {
if (object.getValue().equals(source)) {
srcObjectId = object.getKey();
break;
}
}
if (srcObjectId == null) {
return newMockError(RestStatus.NOT_FOUND, "source object not found");
}
byte[] content = objectsContent.get(srcObjectId);
if (content == null) {
return newMockError(RestStatus.NOT_FOUND, "source content can not be found");
}
String destObjectId = String.valueOf(objectsCount.getAndIncrement());
objectsNames.put(destObjectId, dest);
objectsContent.put(destObjectId, content);
return newMockResponse().setContent(buildObjectResource(bucket, dest, destObjectId, content.length));
});
// Batch
//
// https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
handlers.insert("POST https://www.googleapis.com/batch", (url, params, req) -> {
List<MockLowLevelHttpResponse> responses = new ArrayList<>();
// A batch request body looks like this:
//
// --__END_OF_PART__
// Content-Length: 71
// Content-Type: application/http
// content-id: 1
// content-transfer-encoding: binary
//
// DELETE https://www.googleapis.com/storage/v1/b/ohifkgu/o/foo%2Ftest HTTP/1.1
//
//
// --__END_OF_PART__
// Content-Length: 71
// Content-Type: application/http
// content-id: 2
// content-transfer-encoding: binary
//
// DELETE https://www.googleapis.com/storage/v1/b/ohifkgu/o/bar%2Ftest HTTP/1.1
//
//
// --__END_OF_PART__--
// Here we simply process the request body line by line and delegate to other handlers
// if possible.
try (ByteArrayOutputStream os = new ByteArrayOutputStream((int) req.getContentLength())) {
req.getStreamingContent().writeTo(os);
Streams.readAllLines(new ByteArrayInputStream(os.toByteArray()), line -> {
final int indexOfHttp = line.indexOf(" HTTP/1.1");
if (indexOfHttp > 0) {
line = line.substring(0, indexOfHttp);
}
Handler handler = handlers.retrieve(line, params);
if (handler != null) {
try {
responses.add(handler.execute(line, params, req));
} catch (IOException e) {
responses.add(newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage()));
}
}
});
}
// Now we can build the response
String boundary = "__END_OF_PART__";
String sep = "--";
String line = "\r\n";
StringBuilder builder = new StringBuilder();
for (MockLowLevelHttpResponse resp : responses) {
builder.append(sep).append(boundary).append(line);
builder.append(line);
builder.append("HTTP/1.1 ").append(resp.getStatusCode()).append(' ').append(resp.getReasonPhrase()).append(line);
builder.append("Content-Length: ").append(resp.getContentLength()).append(line);
builder.append(line);
}
builder.append(line);
builder.append(sep).append(boundary).append(sep);
return newMockResponse().setContentType("multipart/mixed; boundary=" + boundary).setContent(builder.toString());
});
}
@Override
public LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
return new MockLowLevelHttpRequest() {
@Override
public LowLevelHttpResponse execute() throws IOException {
String rawPath = url;
Map<String, String> params = new HashMap<>();
int pathEndPos = url.indexOf('?');
if (pathEndPos != -1) {
rawPath = url.substring(0, pathEndPos);
RestUtils.decodeQueryString(url, pathEndPos + 1, params);
}
Handler handler = handlers.retrieve(method + " " + rawPath, params);
if (handler != null) {
return handler.execute(rawPath, params, this);
}
return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "Unable to handle request [method=" + method + ", url=" + url + "]");
}
};
}
private static MockLowLevelHttpResponse newMockResponse() {
return new MockLowLevelHttpResponse()
.setContentType(Json.MEDIA_TYPE)
.setStatusCode(RestStatus.OK.getStatus())
.setReasonPhrase(RestStatus.OK.name());
}
private static MockLowLevelHttpResponse newMockError(RestStatus status, String message) {
MockLowLevelHttpResponse response = newMockResponse().setStatusCode(status.getStatus()).setReasonPhrase(status.name());
try {
response.setContent(buildErrorResource(status, message));
} catch (IOException e) {
response.setContent("Failed to build error resource [" + message + "] because of: " + e.getMessage());
}
return response;
}
/**
* Storage Error JSON representation
*/
private static String buildErrorResource(RestStatus status, String message) throws IOException {
return jsonBuilder()
.startObject()
.startObject("error")
.field("code", status.getStatus())
.field("message", message)
.startArray("errors")
.startObject()
.field("domain", "global")
.field("reason", status.toString())
.field("message", message)
.endObject()
.endArray()
.endObject()
.endObject()
.string();
}
/**
* Storage Bucket JSON representation as defined in
* https://cloud.google.com/storage/docs/json_api/v1/bucket#resource
*/
private static String buildBucketResource(String name) throws IOException {
return jsonBuilder().startObject()
.field("kind", "storage#bucket")
.field("id", name)
.endObject()
.string();
}
/**
* Storage Object JSON representation as defined in
* https://cloud.google.com/storage/docs/json_api/v1/objects#resource
*/
private static XContentBuilder buildObjectResource(XContentBuilder builder, String bucket, String name, String id, int size)
throws IOException {
return builder.startObject()
.field("kind", "storage#object")
.field("id", String.join("/", bucket, name, id))
.field("name", name)
.field("size", String.valueOf(size))
.endObject();
}
private static String buildObjectResource(String bucket, String name, String id, int size) throws IOException {
return buildObjectResource(jsonBuilder(), bucket, name, id, size).string();
}
interface Handler {
MockLowLevelHttpResponse execute(String url, Map<String, String> params, MockLowLevelHttpRequest request) throws IOException;
}
/**
* Instanciates a mocked Storage client for tests.
*/
public static Storage newStorage(String bucket, String applicationName) {
return new Storage.Builder(new MockHttpTransport(bucket), new JacksonFactory(), null)
.setApplicationName(applicationName)
.build();
}
}

93
plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/MockStorage.java Normal file
View File

@ -0,0 +1,93 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.gcs;
import com.google.api.client.http.LowLevelHttpRequest;
import com.google.api.client.http.LowLevelHttpResponse;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.testing.http.MockLowLevelHttpRequest;
import com.google.api.client.testing.http.MockLowLevelHttpResponse;
import com.google.api.services.storage.Storage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Map;
/**
* {@link MockStorage} is a utility class that provides {@link Storage} clients that works
* against an embedded {@link GoogleCloudStorageTestServer}.
*/
class MockStorage extends com.google.api.client.testing.http.MockHttpTransport {
/**
* Embedded test server that emulates a Google Cloud Storage service
**/
private final GoogleCloudStorageTestServer server = new GoogleCloudStorageTestServer();
private MockStorage() {
}
@Override
public LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
return new MockLowLevelHttpRequest() {
@Override
public LowLevelHttpResponse execute() throws IOException {
final GoogleCloudStorageTestServer.Response response = server.handle(method, url, getContentAsBytes());
return convert(response);
}
/** Returns the LowLevelHttpRequest body as an array of bytes **/
byte[] getContentAsBytes() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
if (getStreamingContent() != null) {
getStreamingContent().writeTo(out);
}
return out.toByteArray();
}
};
}
private static MockLowLevelHttpResponse convert(final GoogleCloudStorageTestServer.Response response) {
final MockLowLevelHttpResponse lowLevelHttpResponse = new MockLowLevelHttpResponse();
for (Map.Entry<String, String> header : response.headers.entrySet()) {
lowLevelHttpResponse.addHeader(header.getKey(), header.getValue());
}
lowLevelHttpResponse.setContentType(response.contentType);
lowLevelHttpResponse.setStatusCode(response.status.getStatus());
lowLevelHttpResponse.setReasonPhrase(response.status.toString());
if (response.body != null) {
lowLevelHttpResponse.setContent(response.body);
lowLevelHttpResponse.setContentLength(response.body.length);
}
return lowLevelHttpResponse;
}
/**
* Instanciates a mocked Storage client for tests.
*/
public static Storage newStorageClient(final String bucket, final String applicationName) {
MockStorage mockStorage = new MockStorage();
mockStorage.server.createBucket(bucket);
return new Storage.Builder(mockStorage, JacksonFactory.getDefaultInstance(), null)
.setApplicationName(applicationName)
.build();
}
}