From 608ebd52bafecf980e9726d397c786a9c2422eba Mon Sep 17 00:00:00 2001 From: cnauroth Date: Fri, 6 Mar 2015 14:59:09 -0800 Subject: [PATCH] HADOOP-11642. Upgrade azure sdk version from 0.6.0 to 2.0.0. Contributed by Shashank Khandelwal and Ivan Mitic. --- .../hadoop-common/CHANGES.txt | 3 + hadoop-project/pom.xml | 6 +- hadoop-tools/hadoop-azure/pom.xml | 7 +- .../fs/azure/AzureNativeFileSystemStore.java | 37 +++++----- .../fs/azure/NativeAzureFileSystem.java | 10 +-- .../fs/azure/PageBlobFormatHelpers.java | 2 +- .../hadoop/fs/azure/PageBlobInputStream.java | 8 +-- .../hadoop/fs/azure/PageBlobOutputStream.java | 8 +-- .../hadoop/fs/azure/SelfRenewingLease.java | 6 +- .../fs/azure/SelfThrottlingIntercept.java | 10 +-- .../hadoop/fs/azure/SendRequestIntercept.java | 16 +++-- .../hadoop/fs/azure/StorageInterface.java | 24 +++---- .../hadoop/fs/azure/StorageInterfaceImpl.java | 46 ++++++------ .../fs/azure/metrics/ErrorMetricUpdater.java | 8 +-- .../ResponseReceivedMetricUpdater.java | 10 +-- .../fs/azure/AzureBlobStorageTestAccount.java | 28 ++++---- .../hadoop/fs/azure/MockStorageInterface.java | 70 +++++++++++++------ .../azure/NativeAzureFileSystemBaseTest.java | 6 +- .../TestAzureFileSystemErrorConditions.java | 6 +- .../fs/azure/TestBlobDataValidation.java | 20 +++--- .../hadoop/fs/azure/TestContainerChecks.java | 6 +- .../TestOutOfBandAzureBlobOperationsLive.java | 4 +- .../fs/azure/TestWasbUriAndConfiguration.java | 4 +- 23 files changed, 190 insertions(+), 155 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 628faa3d3aa..14cd75a8ca6 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -653,6 +653,9 @@ Release 2.7.0 - UNRELEASED HADOOP-11648. Set DomainSocketWatcher thread name explicitly. (Liang Xie via ozawa) + HADOOP-11642. Upgrade azure sdk version from 0.6.0 to 2.0.0. + (Shashank Khandelwal and Ivan Mitic via cnauroth) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 2c0f03a5f12..a6127c76305 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -902,9 +902,9 @@ - com.microsoft.windowsazure.storage - microsoft-windowsazure-storage-sdk - 0.6.0 + com.microsoft.azure + azure-storage + 2.0.0 diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index d39dd769e1a..e9b3af775f2 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -140,12 +140,13 @@ httpclient compile - + - com.microsoft.windowsazure.storage - microsoft-windowsazure-storage-sdk + com.microsoft.azure + azure-storage compile + com.google.guava diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index c0c03b3fce8..b664fe760cb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -65,23 +65,23 @@ import org.apache.hadoop.fs.permission.PermissionStatus; import org.mortbay.util.ajax.JSON; import com.google.common.annotations.VisibleForTesting; -import com.microsoft.windowsazure.storage.CloudStorageAccount; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RetryExponentialRetry; -import com.microsoft.windowsazure.storage.RetryNoRetry; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageCredentialsAccountAndKey; -import com.microsoft.windowsazure.storage.StorageCredentialsSharedAccessSignature; -import com.microsoft.windowsazure.storage.StorageErrorCode; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobListingDetails; -import com.microsoft.windowsazure.storage.blob.BlobProperties; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.blob.CopyStatus; -import com.microsoft.windowsazure.storage.blob.DeleteSnapshotsOption; -import com.microsoft.windowsazure.storage.blob.ListBlobItem; -import com.microsoft.windowsazure.storage.core.Utility; +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RetryExponentialRetry; +import com.microsoft.azure.storage.RetryNoRetry; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; +import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; +import com.microsoft.azure.storage.StorageErrorCode; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobListingDetails; +import com.microsoft.azure.storage.blob.BlobProperties; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.blob.CopyStatus; +import com.microsoft.azure.storage.blob.DeleteSnapshotsOption; +import com.microsoft.azure.storage.blob.ListBlobItem; +import com.microsoft.azure.storage.core.Utility; /** * Core implementation of Windows Azure Filesystem for Hadoop. @@ -2543,7 +2543,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { try { checkContainer(ContainerAccessType.ReadThenWrite); CloudBlobWrapper blob = getBlobReference(key); - blob.getProperties().setLastModified(lastModified); + //setLastModified function is not available in 2.0.0 version. blob.uploadProperties automatically updates last modified + //timestamp to current time blob.uploadProperties(getInstrumentedContext(), folderLease); } catch (Exception e) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 0248b85a283..e39b37d5e49 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -74,11 +74,11 @@ import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; import com.google.common.annotations.VisibleForTesting; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.CloudBlob; -import com.microsoft.windowsazure.storage.core.*; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.core.*; /** *

diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java index ad11aacbcf5..9a316a51bd2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java @@ -20,7 +20,7 @@ package org.apache.hadoop.fs.azure; import java.nio.ByteBuffer; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.BlobRequestOptions; /** * Constants and helper methods for ASV's custom data format in page blobs. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java index 62b47ee4f49..468ac65d76b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java @@ -33,10 +33,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.PageRange; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.PageRange; /** * An input stream that reads file data from a page blob stored diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java index 4d1d5c8e6af..2b8846c7f73 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java @@ -44,10 +44,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import com.google.common.annotations.VisibleForTesting; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; -import com.microsoft.windowsazure.storage.blob.CloudPageBlob; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudPageBlob; /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java index bda6006d602..06f32ce3618 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java @@ -22,9 +22,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper; -import com.microsoft.windowsazure.storage.AccessCondition; -import com.microsoft.windowsazure.storage.StorageException; -import com.microsoft.windowsazure.storage.blob.CloudBlob; +import com.microsoft.azure.storage.AccessCondition; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudBlob; import java.util.concurrent.atomic.AtomicInteger; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java index d18a14406ed..a9e3df907f9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java @@ -25,11 +25,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.RequestResult; -import com.microsoft.windowsazure.storage.ResponseReceivedEvent; -import com.microsoft.windowsazure.storage.SendingRequestEvent; -import com.microsoft.windowsazure.storage.StorageEvent; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.RequestResult; +import com.microsoft.azure.storage.ResponseReceivedEvent; +import com.microsoft.azure.storage.SendingRequestEvent; +import com.microsoft.azure.storage.StorageEvent; /* * Self throttling is implemented by hooking into send & response callbacks diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java index 18f173ebbf7..4d564d5338e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java @@ -25,12 +25,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import com.microsoft.windowsazure.storage.Constants.HeaderConstants; -import com.microsoft.windowsazure.storage.OperationContext; -import com.microsoft.windowsazure.storage.SendingRequestEvent; -import com.microsoft.windowsazure.storage.StorageCredentials; -import com.microsoft.windowsazure.storage.StorageEvent; -import com.microsoft.windowsazure.storage.StorageException; +import com.microsoft.azure.storage.Constants.HeaderConstants; +import com.microsoft.azure.storage.core.StorageCredentialsHelper; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.SendingRequestEvent; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageEvent; +import com.microsoft.azure.storage.StorageException; /** * Manages the lifetime of binding on the operation contexts to intercept send @@ -146,7 +147,8 @@ public final class SendRequestIntercept extends StorageEvent