Use Azure Storage 2.0.0

Microsoft team has released a new specific project to deal with storage with a much cleaner API than the previous version.

See https://github.com/azure/azure-storage-java
Documentation is here: http://azure.microsoft.com/en-us/documentation/articles/storage-java-how-to-use-blob-storage/

Note that the produced ZIP file has been reduced from 5mb to 1.3mb.

Related to #38

(cherry picked from commit 4467254)
(cherry picked from commit b2f1e4d)

pom.xml

@@ -68,15 +68,9 @@ governing permissions and limitations under the License. -->
 
         <!-- Azure API -->
         <dependency>
-            <groupId>com.microsoft.windowsazure</groupId>
+            <groupId>com.microsoft.azure</groupId>
-            <artifactId>microsoft-windowsazure-api</artifactId>
+            <artifactId>azure-storage</artifactId>
-            <version>0.4.6</version>
+            <version>2.0.0</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>javax.xml.stream</groupId>
-                    <artifactId>stax-api</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
 
         <dependency>

src/main/assemblies/plugin.xml

@@ -30,7 +30,7 @@ governing permissions and limitations under the License. -->
             <useProjectArtifact>true</useProjectArtifact>
             <useTransitiveFiltering>true</useTransitiveFiltering>
             <includes>
-                <include>com.microsoft.windowsazure:microsoft-windowsazure-api</include>
+                <include>com.microsoft.azure:azure-storage</include>
             </includes>
         </dependencySet>
     </dependencySets>

src/main/java/org/elasticsearch/cloud/azure/AzureStorageService.java

@@ -19,8 +19,7 @@
 
 package org.elasticsearch.cloud.azure;
 
-import com.microsoft.windowsazure.services.core.ServiceException;
+import com.microsoft.azure.storage.StorageException;
-import com.microsoft.windowsazure.services.core.storage.StorageException;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.collect.ImmutableMap;
 
@@ -48,17 +47,17 @@ public interface AzureStorageService {
 
     void createContainer(String container) throws URISyntaxException, StorageException;
 
-    void deleteFiles(String container, String path) throws URISyntaxException, StorageException, ServiceException;
+    void deleteFiles(String container, String path) throws URISyntaxException, StorageException;
 
     boolean blobExists(String container, String blob) throws URISyntaxException, StorageException;
 
     void deleteBlob(String container, String blob) throws URISyntaxException, StorageException;
 
-    InputStream getInputStream(String container, String blob) throws ServiceException;
+    InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException;
 
     OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException;
 
-    ImmutableMap<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException, ServiceException;
+    ImmutableMap<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException;
 
     void moveBlob(String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException;
 }

src/main/java/org/elasticsearch/cloud/azure/AzureStorageServiceImpl.java

@@ -19,21 +19,9 @@
 
 package org.elasticsearch.cloud.azure;
 
-import com.microsoft.windowsazure.services.blob.BlobConfiguration;
+import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.windowsazure.services.blob.BlobContract;
+import com.microsoft.azure.storage.StorageException;
-import com.microsoft.windowsazure.services.blob.BlobService;
+import com.microsoft.azure.storage.blob.*;
-import com.microsoft.windowsazure.services.blob.client.CloudBlobClient;
-import com.microsoft.windowsazure.services.blob.client.CloudBlobContainer;
-import com.microsoft.windowsazure.services.blob.client.CloudBlockBlob;
-import com.microsoft.windowsazure.services.blob.client.ListBlobItem;
-import com.microsoft.windowsazure.services.blob.models.BlobProperties;
-import com.microsoft.windowsazure.services.blob.models.GetBlobResult;
-import com.microsoft.windowsazure.services.blob.models.ListBlobsOptions;
-import com.microsoft.windowsazure.services.blob.models.ListBlobsResult;
-import com.microsoft.windowsazure.services.core.Configuration;
-import com.microsoft.windowsazure.services.core.ServiceException;
-import com.microsoft.windowsazure.services.core.storage.CloudStorageAccount;
-import com.microsoft.windowsazure.services.core.storage.StorageException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
@@ -48,7 +36,6 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.List;
 
 /**
  *
@@ -60,10 +47,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
     private final String key;
     private final String blob;
 
-    private CloudStorageAccount storage_account;
     private CloudBlobClient client;
-    private BlobContract service;
-
 
     @Inject
     public AzureStorageServiceImpl(Settings settings, SettingsFilter settingsFilter) {
@@ -84,14 +68,11 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
                                 + "AccountName="+ account +";"
                                 + "AccountKey=" + key;
 
-                Configuration configuration = Configuration.getInstance();
+                // Retrieve storage account from connection-string.
-                configuration.setProperty(BlobConfiguration.ACCOUNT_NAME, account);
+                CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
-                configuration.setProperty(BlobConfiguration.ACCOUNT_KEY, key);
-                configuration.setProperty(BlobConfiguration.URI, blob);
-                service = BlobService.create(configuration);
 
-                storage_account = CloudStorageAccount.parse(storageConnectionString);
+                // Create the blob client.
-                client = storage_account.createCloudBlobClient();
+                client = storageAccount.createCloudBlobClient();
             }
         } catch (Exception e) {
             // Can not start Azure Storage Client
@@ -129,7 +110,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
         try {
             CloudBlobContainer blob_container = client.getContainerReference(container);
             logger.trace("creating container [{}]", container);
-            blob_container.createIfNotExist();
+            blob_container.createIfNotExists();
         } catch (IllegalArgumentException e) {
             logger.trace("fails creating container [{}]", container, e.getMessage());
             throw new RepositoryException(container, e.getMessage());
@@ -137,19 +118,15 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
     }
 
     @Override
-    public void deleteFiles(String container, String path) throws URISyntaxException, StorageException, ServiceException {
+    public void deleteFiles(String container, String path) throws URISyntaxException, StorageException {
         logger.trace("delete files container [{}], path [{}]", container, path);
 
         // Container name must be lower case.
         CloudBlobContainer blob_container = client.getContainerReference(container);
         if (blob_container.exists()) {
-            ListBlobsOptions options = new ListBlobsOptions();
+            for (ListBlobItem blobItem : blob_container.listBlobs(path)) {
-            options.setPrefix(path);
+                logger.trace("removing blob [{}]", blobItem.getUri());
-
+                deleteBlob(container, blobItem.getUri().toString());
-            List<ListBlobsResult.BlobEntry> blobs = service.listBlobs(container, options).getBlobs();
-            for (ListBlobsResult.BlobEntry blob : blobs) {
-                logger.trace("removing in container [{}], path [{}], blob [{}]", container, path, blob.getName());
-                service.deleteBlob(container, blob.getName());
             }
         }
     }
@@ -180,10 +157,9 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
     }
 
     @Override
-    public InputStream getInputStream(String container, String blob) throws ServiceException {
+    public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException {
         logger.trace("reading container [{}], blob [{}]", container, blob);
-        GetBlobResult blobResult = service.getBlob(container, blob);
+        return client.getContainerReference(container).getBlockBlobReference(blob).openInputStream();
-        return blobResult.getContentStream();
     }
 
     @Override
@@ -193,21 +169,20 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
     }
 
     @Override
-    public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException, ServiceException {
+    public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException {
         logger.debug("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix);
         ImmutableMap.Builder<String, BlobMetaData> blobsBuilder = ImmutableMap.builder();
 
         CloudBlobContainer blob_container = client.getContainerReference(container);
         if (blob_container.exists()) {
-            Iterable<ListBlobItem> blobs = blob_container.listBlobs(keyPath + prefix);
+            for (ListBlobItem blobItem : blob_container.listBlobs(keyPath + prefix)) {
-            for (ListBlobItem blob : blobs) {
+                URI uri = blobItem.getUri();
-                URI uri = blob.getUri();
                 logger.trace("blob url [{}]", uri);
                 String blobpath = uri.getPath().substring(container.length() + 1);
-                BlobProperties properties = service.getBlobProperties(container, blobpath).getProperties();
+                BlobProperties properties = blob_container.getBlockBlobReference(blobpath).getProperties();
                 String name = blobpath.substring(keyPath.length() + 1);
-                logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getContentLength());
+                logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength());
-                blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getContentLength()));
+                blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
             }
         }
 
@@ -221,7 +196,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
         CloudBlockBlob blobSource = blob_container.getBlockBlobReference(sourceBlob);
         if (blobSource.exists()) {
             CloudBlockBlob blobTarget = blob_container.getBlockBlobReference(targetBlob);
-            blobTarget.copyFromBlob(blobSource);
+            blobTarget.startCopyFromBlob(blobSource);
             blobSource.delete();
             logger.debug("moveBlob container [{}], sourceBlob [{}], targetBlob [{}] -> done", container, sourceBlob, targetBlob);
         }

src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java

@@ -19,8 +19,7 @@
 
 package org.elasticsearch.cloud.azure.blobstore;
 
-import com.microsoft.windowsazure.services.core.ServiceException;
+import com.microsoft.azure.storage.StorageException;
-import com.microsoft.windowsazure.services.core.storage.StorageException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -73,11 +72,13 @@ public class AzureBlobContainer extends AbstractBlobContainer {
     public InputStream openInput(String blobName) throws IOException {
         try {
             return blobStore.client().getInputStream(blobStore.container(), buildKey(blobName));
-        } catch (ServiceException e) {
+        } catch (StorageException e) {
             if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
                 throw new FileNotFoundException(e.getMessage());
             }
             throw new IOException(e);
+        } catch (URISyntaxException e) {
+            throw new IOException(e);
         }
     }
 
@@ -112,7 +113,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
         try {
             return blobStore.client().listBlobsByPrefix(blobStore.container(), keyPath, prefix);
-        } catch (URISyntaxException | StorageException | ServiceException e) {
+        } catch (URISyntaxException | StorageException e) {
             logger.warn("can not access [{}] in container {{}}: {}", prefix, blobStore.container(), e.getMessage());
             throw new IOException(e);
         }

src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java

@@ -19,8 +19,7 @@
 
 package org.elasticsearch.cloud.azure.blobstore;
 
-import com.microsoft.windowsazure.services.core.ServiceException;
+import com.microsoft.azure.storage.StorageException;
-import com.microsoft.windowsazure.services.core.storage.StorageException;
 import org.elasticsearch.cloud.azure.AzureStorageService;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -81,7 +80,7 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
 
         try {
             client.deleteFiles(container, keyPath);
-        } catch (URISyntaxException | StorageException | ServiceException e) {
+        } catch (URISyntaxException | StorageException e) {
             logger.warn("can not remove [{}] in container {{}}: {}", keyPath, container, e.getMessage());
         }
     }

src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

@@ -19,7 +19,7 @@
 
 package org.elasticsearch.repositories.azure;
 
-import com.microsoft.windowsazure.services.core.storage.StorageException;
+import com.microsoft.azure.storage.StorageException;
 import org.elasticsearch.cloud.azure.AzureStorageService;
 import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
 import org.elasticsearch.cluster.metadata.MetaData;

src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java

@@ -19,8 +19,7 @@
 
 package org.elasticsearch.repositories.azure;
 
-import com.microsoft.windowsazure.services.core.ServiceException;
+import com.microsoft.azure.storage.StorageException;
-import com.microsoft.windowsazure.services.core.storage.StorageException;
 import org.elasticsearch.cloud.azure.AbstractAzureTest;
 import org.elasticsearch.cloud.azure.AzureStorageService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -87,7 +86,7 @@ public abstract class AbstractAzureRepositoryServiceTest extends AbstractAzureTe
     }
 
     @Before @After
-    public final void wipe() throws StorageException, ServiceException, URISyntaxException {
+    public final void wipe() throws StorageException, URISyntaxException {
         wipeRepositories();
         cleanRepositoryFiles(basePath);
     }
@@ -95,7 +94,7 @@ public abstract class AbstractAzureRepositoryServiceTest extends AbstractAzureTe
     /**
      * Purge the test container
      */
-    public void cleanRepositoryFiles(String path) throws StorageException, ServiceException, URISyntaxException {
+    public void cleanRepositoryFiles(String path) throws StorageException, URISyntaxException {
         String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container");
         logger.info("--> remove blobs in container [{}]", container);
         AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);

src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java

@@ -20,8 +20,7 @@
 package org.elasticsearch.repositories.azure;
 
 
-import com.microsoft.windowsazure.services.core.ServiceException;
+import com.microsoft.azure.storage.StorageException;
-import com.microsoft.windowsazure.services.core.storage.StorageException;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
@@ -83,7 +82,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
     }
 
     @Before
-    public final void wipeBefore() throws StorageException, ServiceException, URISyntaxException {
+    public final void wipeBefore() throws StorageException, URISyntaxException {
         wipeRepositories();
         cleanRepositoryFiles(
                 getContainerName(),
@@ -92,7 +91,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
     }
 
     @After
-    public final void wipeAfter() throws StorageException, ServiceException, URISyntaxException {
+    public final void wipeAfter() throws StorageException, URISyntaxException {
         wipeRepositories();
         cleanRepositoryFiles(
                 getContainerName(),
@@ -243,7 +242,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
      * For issue #26: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/26
      */
     @Test
-    public void testListBlobs_26() throws StorageException, ServiceException, URISyntaxException {
+    public void testListBlobs_26() throws StorageException, URISyntaxException {
         createIndex("test-idx-1", "test-idx-2", "test-idx-3");
         ensureGreen();
 
@@ -302,7 +301,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
      * For issue #28: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/28
      */
     @Test
-    public void testGetDeleteNonExistingSnapshot_28() throws StorageException, ServiceException, URISyntaxException {
+    public void testGetDeleteNonExistingSnapshot_28() throws StorageException, URISyntaxException {
         ClusterAdminClient client = client().admin().cluster();
         logger.info("-->  creating azure repository without any path");
         PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure")
@@ -452,7 +451,7 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
     /**
      * Purge the test containers
      */
-    public void cleanRepositoryFiles(String... containers) throws StorageException, ServiceException, URISyntaxException {
+    public void cleanRepositoryFiles(String... containers) throws StorageException, URISyntaxException {
         AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);
         for (String container : containers) {
             logger.info("--> remove container [{}]", container);

src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java

@@ -19,7 +19,7 @@
 
 package org.elasticsearch.repositories.azure;
 
-import com.microsoft.windowsazure.services.core.storage.StorageException;
+import com.microsoft.azure.storage.StorageException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cloud.azure.AzureStorageService;
 import org.elasticsearch.common.blobstore.BlobMetaData;
@@ -29,7 +29,10 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 
-import java.io.*;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.net.URISyntaxException;
 import java.util.Locale;
 import java.util.Map;