From 54de108a5199001aa267ec4af0d680bdf4c650c9 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Wed, 4 Feb 2015 17:11:56 +0100 Subject: [PATCH] Use Azure Storage 2.0.0 Microsoft team has released a new specific project to deal with storage with a much cleaner API than the previous version. See https://github.com/azure/azure-storage-java Documentation is here: http://azure.microsoft.com/en-us/documentation/articles/storage-java-how-to-use-blob-storage/ Note that the produced ZIP file has been reduced from 5mb to 1.3mb. Related to #38 (cherry picked from commit 4467254) (cherry picked from commit b2f1e4d) --- pom.xml | 12 +--- src/main/assemblies/plugin.xml | 2 +- .../cloud/azure/AzureStorageService.java | 9 ++- .../cloud/azure/AzureStorageServiceImpl.java | 67 ++++++------------- .../azure/blobstore/AzureBlobContainer.java | 9 +-- .../cloud/azure/blobstore/AzureBlobStore.java | 5 +- .../repositories/azure/AzureRepository.java | 2 +- .../AbstractAzureRepositoryServiceTest.java | 7 +- .../azure/AzureSnapshotRestoreITest.java | 13 ++-- .../azure/AzureStorageServiceMock.java | 7 +- 10 files changed, 51 insertions(+), 82 deletions(-) diff --git a/pom.xml b/pom.xml index b70f84d5119..387c788b755 100644 --- a/pom.xml +++ b/pom.xml @@ -68,15 +68,9 @@ governing permissions and limitations under the License. --> - com.microsoft.windowsazure - microsoft-windowsazure-api - 0.4.6 - - - javax.xml.stream - stax-api - - + com.microsoft.azure + azure-storage + 2.0.0 diff --git a/src/main/assemblies/plugin.xml b/src/main/assemblies/plugin.xml index b18ecaff3a0..3de1c5befa5 100644 --- a/src/main/assemblies/plugin.xml +++ b/src/main/assemblies/plugin.xml @@ -30,7 +30,7 @@ governing permissions and limitations under the License. --> true true - com.microsoft.windowsazure:microsoft-windowsazure-api + com.microsoft.azure:azure-storage diff --git a/src/main/java/org/elasticsearch/cloud/azure/AzureStorageService.java b/src/main/java/org/elasticsearch/cloud/azure/AzureStorageService.java index 80d948c259a..97b5c26a69e 100644 --- a/src/main/java/org/elasticsearch/cloud/azure/AzureStorageService.java +++ b/src/main/java/org/elasticsearch/cloud/azure/AzureStorageService.java @@ -19,8 +19,7 @@ package org.elasticsearch.cloud.azure; -import com.microsoft.windowsazure.services.core.ServiceException; -import com.microsoft.windowsazure.services.core.storage.StorageException; +import com.microsoft.azure.storage.StorageException; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.collect.ImmutableMap; @@ -48,17 +47,17 @@ public interface AzureStorageService { void createContainer(String container) throws URISyntaxException, StorageException; - void deleteFiles(String container, String path) throws URISyntaxException, StorageException, ServiceException; + void deleteFiles(String container, String path) throws URISyntaxException, StorageException; boolean blobExists(String container, String blob) throws URISyntaxException, StorageException; void deleteBlob(String container, String blob) throws URISyntaxException, StorageException; - InputStream getInputStream(String container, String blob) throws ServiceException; + InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException; OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException; - ImmutableMap listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException, ServiceException; + ImmutableMap listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException; void moveBlob(String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException; } diff --git a/src/main/java/org/elasticsearch/cloud/azure/AzureStorageServiceImpl.java b/src/main/java/org/elasticsearch/cloud/azure/AzureStorageServiceImpl.java index 1254f8bc785..894ad5d413c 100644 --- a/src/main/java/org/elasticsearch/cloud/azure/AzureStorageServiceImpl.java +++ b/src/main/java/org/elasticsearch/cloud/azure/AzureStorageServiceImpl.java @@ -19,21 +19,9 @@ package org.elasticsearch.cloud.azure; -import com.microsoft.windowsazure.services.blob.BlobConfiguration; -import com.microsoft.windowsazure.services.blob.BlobContract; -import com.microsoft.windowsazure.services.blob.BlobService; -import com.microsoft.windowsazure.services.blob.client.CloudBlobClient; -import com.microsoft.windowsazure.services.blob.client.CloudBlobContainer; -import com.microsoft.windowsazure.services.blob.client.CloudBlockBlob; -import com.microsoft.windowsazure.services.blob.client.ListBlobItem; -import com.microsoft.windowsazure.services.blob.models.BlobProperties; -import com.microsoft.windowsazure.services.blob.models.GetBlobResult; -import com.microsoft.windowsazure.services.blob.models.ListBlobsOptions; -import com.microsoft.windowsazure.services.blob.models.ListBlobsResult; -import com.microsoft.windowsazure.services.core.Configuration; -import com.microsoft.windowsazure.services.core.ServiceException; -import com.microsoft.windowsazure.services.core.storage.CloudStorageAccount; -import com.microsoft.windowsazure.services.core.storage.StorageException; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.*; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.support.PlainBlobMetaData; @@ -48,7 +36,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; -import java.util.List; /** * @@ -60,10 +47,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent blobs = service.listBlobs(container, options).getBlobs(); - for (ListBlobsResult.BlobEntry blob : blobs) { - logger.trace("removing in container [{}], path [{}], blob [{}]", container, path, blob.getName()); - service.deleteBlob(container, blob.getName()); + for (ListBlobItem blobItem : blob_container.listBlobs(path)) { + logger.trace("removing blob [{}]", blobItem.getUri()); + deleteBlob(container, blobItem.getUri().toString()); } } } @@ -180,10 +157,9 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException, ServiceException { + public ImmutableMap listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException { logger.debug("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix); ImmutableMap.Builder blobsBuilder = ImmutableMap.builder(); CloudBlobContainer blob_container = client.getContainerReference(container); if (blob_container.exists()) { - Iterable blobs = blob_container.listBlobs(keyPath + prefix); - for (ListBlobItem blob : blobs) { - URI uri = blob.getUri(); + for (ListBlobItem blobItem : blob_container.listBlobs(keyPath + prefix)) { + URI uri = blobItem.getUri(); logger.trace("blob url [{}]", uri); String blobpath = uri.getPath().substring(container.length() + 1); - BlobProperties properties = service.getBlobProperties(container, blobpath).getProperties(); + BlobProperties properties = blob_container.getBlockBlobReference(blobpath).getProperties(); String name = blobpath.substring(keyPath.length() + 1); - logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getContentLength()); - blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getContentLength())); + logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength()); + blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength())); } } @@ -221,7 +196,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent done", container, sourceBlob, targetBlob); } diff --git a/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java index dd032591eb8..fd8aeca30f3 100644 --- a/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java @@ -19,8 +19,7 @@ package org.elasticsearch.cloud.azure.blobstore; -import com.microsoft.windowsazure.services.core.ServiceException; -import com.microsoft.windowsazure.services.core.storage.StorageException; +import com.microsoft.azure.storage.StorageException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; @@ -73,11 +72,13 @@ public class AzureBlobContainer extends AbstractBlobContainer { public InputStream openInput(String blobName) throws IOException { try { return blobStore.client().getInputStream(blobStore.container(), buildKey(blobName)); - } catch (ServiceException e) { + } catch (StorageException e) { if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) { throw new FileNotFoundException(e.getMessage()); } throw new IOException(e); + } catch (URISyntaxException e) { + throw new IOException(e); } } @@ -112,7 +113,7 @@ public class AzureBlobContainer extends AbstractBlobContainer { try { return blobStore.client().listBlobsByPrefix(blobStore.container(), keyPath, prefix); - } catch (URISyntaxException | StorageException | ServiceException e) { + } catch (URISyntaxException | StorageException e) { logger.warn("can not access [{}] in container {{}}: {}", prefix, blobStore.container(), e.getMessage()); throw new IOException(e); } diff --git a/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java index 79f4bc62c54..12581d3b63e 100644 --- a/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java +++ b/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java @@ -19,8 +19,7 @@ package org.elasticsearch.cloud.azure.blobstore; -import com.microsoft.windowsazure.services.core.ServiceException; -import com.microsoft.windowsazure.services.core.storage.StorageException; +import com.microsoft.azure.storage.StorageException; import org.elasticsearch.cloud.azure.AzureStorageService; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; @@ -81,7 +80,7 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore { try { client.deleteFiles(container, keyPath); - } catch (URISyntaxException | StorageException | ServiceException e) { + } catch (URISyntaxException | StorageException e) { logger.warn("can not remove [{}] in container {{}}: {}", keyPath, container, e.getMessage()); } } diff --git a/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index 36882fa281c..5cca6d2543d 100644 --- a/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -19,7 +19,7 @@ package org.elasticsearch.repositories.azure; -import com.microsoft.windowsazure.services.core.storage.StorageException; +import com.microsoft.azure.storage.StorageException; import org.elasticsearch.cloud.azure.AzureStorageService; import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore; import org.elasticsearch.cluster.metadata.MetaData; diff --git a/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java b/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java index 4c92b910f0b..38b8a4fea83 100644 --- a/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java +++ b/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java @@ -19,8 +19,7 @@ package org.elasticsearch.repositories.azure; -import com.microsoft.windowsazure.services.core.ServiceException; -import com.microsoft.windowsazure.services.core.storage.StorageException; +import com.microsoft.azure.storage.StorageException; import org.elasticsearch.cloud.azure.AbstractAzureTest; import org.elasticsearch.cloud.azure.AzureStorageService; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -87,7 +86,7 @@ public abstract class AbstractAzureRepositoryServiceTest extends AbstractAzureTe } @Before @After - public final void wipe() throws StorageException, ServiceException, URISyntaxException { + public final void wipe() throws StorageException, URISyntaxException { wipeRepositories(); cleanRepositoryFiles(basePath); } @@ -95,7 +94,7 @@ public abstract class AbstractAzureRepositoryServiceTest extends AbstractAzureTe /** * Purge the test container */ - public void cleanRepositoryFiles(String path) throws StorageException, ServiceException, URISyntaxException { + public void cleanRepositoryFiles(String path) throws StorageException, URISyntaxException { String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container"); logger.info("--> remove blobs in container [{}]", container); AzureStorageService client = internalCluster().getInstance(AzureStorageService.class); diff --git a/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java b/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java index 5697295be71..490bb5f35bc 100644 --- a/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java +++ b/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java @@ -20,8 +20,7 @@ package org.elasticsearch.repositories.azure; -import com.microsoft.windowsazure.services.core.ServiceException; -import com.microsoft.windowsazure.services.core.storage.StorageException; +import com.microsoft.azure.storage.StorageException; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -83,7 +82,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest { } @Before - public final void wipeBefore() throws StorageException, ServiceException, URISyntaxException { + public final void wipeBefore() throws StorageException, URISyntaxException { wipeRepositories(); cleanRepositoryFiles( getContainerName(), @@ -92,7 +91,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest { } @After - public final void wipeAfter() throws StorageException, ServiceException, URISyntaxException { + public final void wipeAfter() throws StorageException, URISyntaxException { wipeRepositories(); cleanRepositoryFiles( getContainerName(), @@ -243,7 +242,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest { * For issue #26: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/26 */ @Test - public void testListBlobs_26() throws StorageException, ServiceException, URISyntaxException { + public void testListBlobs_26() throws StorageException, URISyntaxException { createIndex("test-idx-1", "test-idx-2", "test-idx-3"); ensureGreen(); @@ -302,7 +301,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest { * For issue #28: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/28 */ @Test - public void testGetDeleteNonExistingSnapshot_28() throws StorageException, ServiceException, URISyntaxException { + public void testGetDeleteNonExistingSnapshot_28() throws StorageException, URISyntaxException { ClusterAdminClient client = client().admin().cluster(); logger.info("--> creating azure repository without any path"); PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure") @@ -452,7 +451,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest { /** * Purge the test containers */ - public void cleanRepositoryFiles(String... containers) throws StorageException, ServiceException, URISyntaxException { + public void cleanRepositoryFiles(String... containers) throws StorageException, URISyntaxException { AzureStorageService client = internalCluster().getInstance(AzureStorageService.class); for (String container : containers) { logger.info("--> remove container [{}]", container); diff --git a/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java b/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java index 257a0c4728a..3ce49321b2b 100644 --- a/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java +++ b/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java @@ -19,7 +19,7 @@ package org.elasticsearch.repositories.azure; -import com.microsoft.windowsazure.services.core.storage.StorageException; +import com.microsoft.azure.storage.StorageException; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cloud.azure.AzureStorageService; import org.elasticsearch.common.blobstore.BlobMetaData; @@ -29,7 +29,10 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; import java.net.URISyntaxException; import java.util.Locale; import java.util.Map;