Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-11642. Upgrade azure sdk version from 0.6.0 to 2.0.0. Contributed by Shashank Khandelwal and Ivan Mitic.

This commit is contained in:
cnauroth 2015-03-06 14:59:09 -08:00
parent 0f8ecb1d0c
commit 608ebd52ba
23 changed files with 190 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -653,6 +653,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11648. Set DomainSocketWatcher thread name explicitly. HADOOP-11648. Set DomainSocketWatcher thread name explicitly.
(Liang Xie via ozawa) (Liang Xie via ozawa)
HADOOP-11642. Upgrade azure sdk version from 0.6.0 to 2.0.0.
(Shashank Khandelwal and Ivan Mitic via cnauroth)
OPTIMIZATIONS OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array. HADOOP-11323. WritableComparator#compare keeps reference to byte array.

6
hadoop-project/pom.xml
View File

@ -902,9 +902,9 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.microsoft.windowsazure.storage</groupId> <groupId>com.microsoft.azure</groupId>
<artifactId>microsoft-windowsazure-storage-sdk</artifactId> <artifactId>azure-storage</artifactId>
<version>0.6.0</version> <version>2.0.0</version>
</dependency> </dependency>
<dependency> <dependency>

5
hadoop-tools/hadoop-azure/pom.xml
View File

@ -142,11 +142,12 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.microsoft.windowsazure.storage</groupId> <groupId>com.microsoft.azure</groupId>
<artifactId>microsoft-windowsazure-storage-sdk</artifactId> <artifactId>azure-storage</artifactId>
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.guava</groupId> <groupId>com.google.guava</groupId>
<artifactId>guava</artifactId> <artifactId>guava</artifactId>

37
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
View File

@ -65,23 +65,23 @@ import org.apache.hadoop.fs.permission.PermissionStatus;
import org.mortbay.util.ajax.JSON; import org.mortbay.util.ajax.JSON;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.microsoft.windowsazure.storage.CloudStorageAccount; import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.RetryExponentialRetry; import com.microsoft.azure.storage.RetryExponentialRetry;
import com.microsoft.windowsazure.storage.RetryNoRetry; import com.microsoft.azure.storage.RetryNoRetry;
import com.microsoft.windowsazure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.windowsazure.storage.StorageCredentialsAccountAndKey; import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
import com.microsoft.windowsazure.storage.StorageCredentialsSharedAccessSignature; import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
import com.microsoft.windowsazure.storage.StorageErrorCode; import com.microsoft.azure.storage.StorageErrorCode;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.windowsazure.storage.blob.BlobListingDetails; import com.microsoft.azure.storage.blob.BlobListingDetails;
import com.microsoft.windowsazure.storage.blob.BlobProperties; import com.microsoft.azure.storage.blob.BlobProperties;
import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.BlobRequestOptions;
import com.microsoft.windowsazure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.windowsazure.storage.blob.CopyStatus; import com.microsoft.azure.storage.blob.CopyStatus;
import com.microsoft.windowsazure.storage.blob.DeleteSnapshotsOption; import com.microsoft.azure.storage.blob.DeleteSnapshotsOption;
import com.microsoft.windowsazure.storage.blob.ListBlobItem; import com.microsoft.azure.storage.blob.ListBlobItem;
import com.microsoft.windowsazure.storage.core.Utility; import com.microsoft.azure.storage.core.Utility;
/** /**
* Core implementation of Windows Azure Filesystem for Hadoop. * Core implementation of Windows Azure Filesystem for Hadoop.
@ -2543,7 +2543,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
try { try {
checkContainer(ContainerAccessType.ReadThenWrite); checkContainer(ContainerAccessType.ReadThenWrite);
CloudBlobWrapper blob = getBlobReference(key); CloudBlobWrapper blob = getBlobReference(key);
blob.getProperties().setLastModified(lastModified); //setLastModified function is not available in 2.0.0 version. blob.uploadProperties automatically updates last modified
//timestamp to current time
blob.uploadProperties(getInstrumentedContext(), folderLease); blob.uploadProperties(getInstrumentedContext(), folderLease);
} catch (Exception e) { } catch (Exception e) {

10
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
View File

@ -74,11 +74,11 @@ import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.ObjectMapper;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.microsoft.windowsazure.storage.AccessCondition; import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.windowsazure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.windowsazure.storage.core.*; import com.microsoft.azure.storage.core.*;
/** /**
* <p> * <p>

2
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobFormatHelpers.java
View File

@ -20,7 +20,7 @@ package org.apache.hadoop.fs.azure;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.BlobRequestOptions;
/** /**
* Constants and helper methods for ASV's custom data format in page blobs. * Constants and helper methods for ASV's custom data format in page blobs.

8
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java
View File

@ -33,10 +33,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper; import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.BlobRequestOptions;
import com.microsoft.windowsazure.storage.blob.PageRange; import com.microsoft.azure.storage.blob.PageRange;
/** /**
* An input stream that reads file data from a page blob stored * An input stream that reads file data from a page blob stored

8
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
View File

@ -44,10 +44,10 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.BlobRequestOptions;
import com.microsoft.windowsazure.storage.blob.CloudPageBlob; import com.microsoft.azure.storage.blob.CloudPageBlob;
/** /**

6
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
View File

@ -22,9 +22,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper; import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper;
import com.microsoft.windowsazure.storage.AccessCondition; import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.windowsazure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlob;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;

10
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfThrottlingIntercept.java
View File

@ -25,11 +25,11 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.RequestResult; import com.microsoft.azure.storage.RequestResult;
import com.microsoft.windowsazure.storage.ResponseReceivedEvent; import com.microsoft.azure.storage.ResponseReceivedEvent;
import com.microsoft.windowsazure.storage.SendingRequestEvent; import com.microsoft.azure.storage.SendingRequestEvent;
import com.microsoft.windowsazure.storage.StorageEvent; import com.microsoft.azure.storage.StorageEvent;
/* /*
* Self throttling is implemented by hooking into send & response callbacks * Self throttling is implemented by hooking into send & response callbacks

16
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SendRequestIntercept.java
View File

@ -25,12 +25,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import com.microsoft.windowsazure.storage.Constants.HeaderConstants; import com.microsoft.azure.storage.Constants.HeaderConstants;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.core.StorageCredentialsHelper;
import com.microsoft.windowsazure.storage.SendingRequestEvent; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.StorageCredentials; import com.microsoft.azure.storage.SendingRequestEvent;
import com.microsoft.windowsazure.storage.StorageEvent; import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageEvent;
import com.microsoft.azure.storage.StorageException;
/** /**
* Manages the lifetime of binding on the operation contexts to intercept send * Manages the lifetime of binding on the operation contexts to intercept send
@ -146,7 +147,8 @@ public final class SendRequestIntercept extends StorageEvent<SendingRequestEvent
try { try {
// Sign the request. GET's have no payload so the content length is // Sign the request. GET's have no payload so the content length is
// zero. // zero.
getCredentials().signBlobAndQueueRequest(urlConnection, -1L, getOperationContext()); StorageCredentialsHelper.signBlobAndQueueRequest(getCredentials(),
urlConnection, -1L, getOperationContext());
} catch (InvalidKeyException e) { } catch (InvalidKeyException e) {
// Log invalid key exception to track signing error before the send // Log invalid key exception to track signing error before the send
// fails. // fails.

24
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
View File

@ -29,18 +29,18 @@ import java.util.HashMap;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import com.microsoft.windowsazure.storage.CloudStorageAccount; import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.RetryPolicyFactory; import com.microsoft.azure.storage.RetryPolicyFactory;
import com.microsoft.windowsazure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.windowsazure.storage.blob.BlobListingDetails; import com.microsoft.azure.storage.blob.BlobListingDetails;
import com.microsoft.windowsazure.storage.blob.BlobProperties; import com.microsoft.azure.storage.blob.BlobProperties;
import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.BlobRequestOptions;
import com.microsoft.windowsazure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.windowsazure.storage.blob.CopyState; import com.microsoft.azure.storage.blob.CopyState;
import com.microsoft.windowsazure.storage.blob.ListBlobItem; import com.microsoft.azure.storage.blob.ListBlobItem;
import com.microsoft.windowsazure.storage.blob.PageRange; import com.microsoft.azure.storage.blob.PageRange;
/** /**
* This is a very thin layer over the methods exposed by the Windows Azure * This is a very thin layer over the methods exposed by the Windows Azure

46
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
View File

@ -30,26 +30,26 @@ import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import com.microsoft.windowsazure.storage.AccessCondition; import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.windowsazure.storage.CloudStorageAccount; import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.RetryPolicyFactory; import com.microsoft.azure.storage.RetryPolicyFactory;
import com.microsoft.windowsazure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.windowsazure.storage.StorageUri; import com.microsoft.azure.storage.StorageUri;
import com.microsoft.windowsazure.storage.blob.BlobListingDetails; import com.microsoft.azure.storage.blob.BlobListingDetails;
import com.microsoft.windowsazure.storage.blob.BlobProperties; import com.microsoft.azure.storage.blob.BlobProperties;
import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.BlobRequestOptions;
import com.microsoft.windowsazure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.windowsazure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.windowsazure.storage.blob.CloudBlobDirectory; import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.CloudBlockBlob;
import com.microsoft.windowsazure.storage.blob.CloudPageBlob; import com.microsoft.azure.storage.blob.CloudPageBlob;
import com.microsoft.windowsazure.storage.blob.CopyState; import com.microsoft.azure.storage.blob.CopyState;
import com.microsoft.windowsazure.storage.blob.DeleteSnapshotsOption; import com.microsoft.azure.storage.blob.DeleteSnapshotsOption;
import com.microsoft.windowsazure.storage.blob.ListBlobItem; import com.microsoft.azure.storage.blob.ListBlobItem;
import com.microsoft.windowsazure.storage.blob.PageRange; import com.microsoft.azure.storage.blob.PageRange;
/** /**
* A real implementation of the Azure interaction layer that just redirects * A real implementation of the Azure interaction layer that just redirects
@ -61,12 +61,14 @@ class StorageInterfaceImpl extends StorageInterface {
@Override @Override
public void setRetryPolicyFactory(final RetryPolicyFactory retryPolicyFactory) { public void setRetryPolicyFactory(final RetryPolicyFactory retryPolicyFactory) {
serviceClient.setRetryPolicyFactory(retryPolicyFactory); serviceClient.getDefaultRequestOptions().setRetryPolicyFactory(
retryPolicyFactory);
} }
@Override @Override
public void setTimeoutInMs(int timeoutInMs) { public void setTimeoutInMs(int timeoutInMs) {
serviceClient.setTimeoutInMs(timeoutInMs); serviceClient.getDefaultRequestOptions().setTimeoutIntervalInMs(
timeoutInMs);
} }
@Override @Override

8
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ErrorMetricUpdater.java
View File

@ -24,10 +24,10 @@ import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; //500
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.RequestResult; import com.microsoft.azure.storage.RequestResult;
import com.microsoft.windowsazure.storage.ResponseReceivedEvent; import com.microsoft.azure.storage.ResponseReceivedEvent;
import com.microsoft.windowsazure.storage.StorageEvent; import com.microsoft.azure.storage.StorageEvent;
/** /**

10
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/metrics/ResponseReceivedMetricUpdater.java
View File

@ -24,11 +24,11 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import com.microsoft.windowsazure.storage.Constants.HeaderConstants; import com.microsoft.azure.storage.Constants.HeaderConstants;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.RequestResult; import com.microsoft.azure.storage.RequestResult;
import com.microsoft.windowsazure.storage.ResponseReceivedEvent; import com.microsoft.azure.storage.ResponseReceivedEvent;
import com.microsoft.windowsazure.storage.StorageEvent; import com.microsoft.azure.storage.StorageEvent;
/** /**

28
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java
View File

@ -40,20 +40,20 @@ import org.apache.hadoop.metrics2.MetricsSink;
import org.apache.hadoop.metrics2.MetricsTag; import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import com.microsoft.windowsazure.storage.AccessCondition; import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.windowsazure.storage.CloudStorageAccount; import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.windowsazure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.windowsazure.storage.StorageCredentialsAccountAndKey; import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
import com.microsoft.windowsazure.storage.StorageCredentialsAnonymous; import com.microsoft.azure.storage.StorageCredentialsAnonymous;
import com.microsoft.windowsazure.storage.blob.BlobContainerPermissions; import com.microsoft.azure.storage.blob.BlobContainerPermissions;
import com.microsoft.windowsazure.storage.blob.BlobContainerPublicAccessType; import com.microsoft.azure.storage.blob.BlobContainerPublicAccessType;
import com.microsoft.windowsazure.storage.blob.BlobOutputStream; import com.microsoft.azure.storage.blob.BlobOutputStream;
import com.microsoft.windowsazure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.CloudBlockBlob;
import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPermissions; import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
import com.microsoft.windowsazure.storage.blob.SharedAccessBlobPolicy; import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy;
import com.microsoft.windowsazure.storage.core.Base64; import com.microsoft.azure.storage.core.Base64;
/** /**
* Helper class to create WASB file systems backed by either a mock in-memory * Helper class to create WASB file systems backed by either a mock in-memory

66
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
View File

@ -22,10 +22,12 @@ import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
import java.lang.reflect.Method;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Date;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
@ -35,21 +37,21 @@ import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.util.URIUtil; import org.apache.commons.httpclient.util.URIUtil;
import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang.NotImplementedException;
import com.microsoft.windowsazure.storage.CloudStorageAccount; import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.RetryPolicyFactory; import com.microsoft.azure.storage.RetryPolicyFactory;
import com.microsoft.windowsazure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.windowsazure.storage.StorageUri; import com.microsoft.azure.storage.StorageUri;
import com.microsoft.windowsazure.storage.blob.BlobListingDetails; import com.microsoft.azure.storage.blob.BlobListingDetails;
import com.microsoft.windowsazure.storage.blob.BlobProperties; import com.microsoft.azure.storage.blob.BlobProperties;
import com.microsoft.windowsazure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.BlobRequestOptions;
import com.microsoft.windowsazure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlob;
import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.windowsazure.storage.blob.CloudBlobDirectory; import com.microsoft.azure.storage.blob.CloudBlobDirectory;
import com.microsoft.windowsazure.storage.blob.CopyState; import com.microsoft.azure.storage.blob.CopyState;
import com.microsoft.windowsazure.storage.blob.ListBlobItem; import com.microsoft.azure.storage.blob.ListBlobItem;
import com.microsoft.windowsazure.storage.blob.PageRange; import com.microsoft.azure.storage.blob.PageRange;
import javax.ws.rs.core.UriBuilder; import javax.ws.rs.core.UriBuilder;
import javax.ws.rs.core.UriBuilderException; import javax.ws.rs.core.UriBuilderException;
@ -357,18 +359,42 @@ public class MockStorageInterface extends StorageInterface {
this.uri = uri; this.uri = uri;
this.metadata = metadata; this.metadata = metadata;
this.properties = new BlobProperties(); this.properties = new BlobProperties();
this.properties.setLength(length);
this.properties.setLastModified( this.properties=updateLastModifed(this.properties);
this.properties=updateLength(this.properties,length);
}
protected BlobProperties updateLastModifed(BlobProperties properties){
try{
Method setLastModified =properties.getClass().
getDeclaredMethod("setLastModified", Date.class);
setLastModified.setAccessible(true);
setLastModified.invoke(this.properties,
Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime()); Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime());
}catch(Exception e){
throw new RuntimeException(e);
}
return properties;
}
protected BlobProperties updateLength(BlobProperties properties,int length) {
try{
Method setLength =properties.getClass().
getDeclaredMethod("setLength", long.class);
setLength.setAccessible(true);
setLength.invoke(this.properties, length);
}catch (Exception e){
throw new RuntimeException(e);
}
return properties;
} }
protected void refreshProperties(boolean getMetadata) { protected void refreshProperties(boolean getMetadata) {
if (backingStore.exists(convertUriToDecodedString(uri))) { if (backingStore.exists(convertUriToDecodedString(uri))) {
byte[] content = backingStore.getContent(convertUriToDecodedString(uri)); byte[] content = backingStore.getContent(convertUriToDecodedString(uri));
properties = new BlobProperties(); properties = new BlobProperties();
properties.setLength(content.length); this.properties=updateLastModifed(this.properties);
properties.setLastModified( this.properties=updateLength(this.properties, content.length);
Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime());
if (getMetadata) { if (getMetadata) {
metadata = backingStore.getMetadata(convertUriToDecodedString(uri)); metadata = backingStore.getMetadata(convertUriToDecodedString(uri));
} }

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
View File

@ -58,9 +58,9 @@ import org.junit.Test;
import org.apache.hadoop.fs.azure.AzureException; import org.apache.hadoop.fs.azure.AzureException;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem.FolderRenamePending; import org.apache.hadoop.fs.azure.NativeAzureFileSystem.FolderRenamePending;
import com.microsoft.windowsazure.storage.AccessCondition; import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.windowsazure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlob;
/* /*
* Tests the Native Azure file system (WASB) against an actual blob store if * Tests the Native Azure file system (WASB) against an actual blob store if

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
View File

@ -37,9 +37,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext; import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext;
import org.junit.Test; import org.junit.Test;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.SendingRequestEvent; import com.microsoft.azure.storage.SendingRequestEvent;
import com.microsoft.windowsazure.storage.StorageEvent; import com.microsoft.azure.storage.StorageEvent;
public class TestAzureFileSystemErrorConditions { public class TestAzureFileSystemErrorConditions {
private static final int ALL_THREE_FILE_SIZE = 1024; private static final int ALL_THREE_FILE_SIZE = 1024;

20
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
View File

@ -39,16 +39,16 @@ import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationCo
import org.junit.After; import org.junit.After;
import org.junit.Test; import org.junit.Test;
import com.microsoft.windowsazure.storage.Constants; import com.microsoft.azure.storage.Constants;
import com.microsoft.windowsazure.storage.OperationContext; import com.microsoft.azure.storage.OperationContext;
import com.microsoft.windowsazure.storage.ResponseReceivedEvent; import com.microsoft.azure.storage.ResponseReceivedEvent;
import com.microsoft.windowsazure.storage.StorageErrorCodeStrings; import com.microsoft.azure.storage.StorageErrorCodeStrings;
import com.microsoft.windowsazure.storage.StorageEvent; import com.microsoft.azure.storage.StorageEvent;
import com.microsoft.windowsazure.storage.StorageException; import com.microsoft.azure.storage.StorageException;
import com.microsoft.windowsazure.storage.blob.BlockEntry; import com.microsoft.azure.storage.blob.BlockEntry;
import com.microsoft.windowsazure.storage.blob.BlockSearchMode; import com.microsoft.azure.storage.blob.BlockSearchMode;
import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.CloudBlockBlob;
import com.microsoft.windowsazure.storage.core.Base64; import com.microsoft.azure.storage.core.Base64;
/** /**
* Test that we do proper data integrity validation with MD5 checks as * Test that we do proper data integrity validation with MD5 checks as

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestContainerChecks.java
View File

@ -32,9 +32,9 @@ import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
import org.junit.After; import org.junit.After;
import org.junit.Test; import org.junit.Test;
import com.microsoft.windowsazure.storage.blob.BlobOutputStream; import com.microsoft.azure.storage.blob.BlobOutputStream;
import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.CloudBlockBlob;
/** /**
* Tests that WASB creates containers only if needed. * Tests that WASB creates containers only if needed.

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestOutOfBandAzureBlobOperationsLive.java
View File

@ -29,8 +29,8 @@ import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import com.microsoft.windowsazure.storage.blob.BlobOutputStream; import com.microsoft.azure.storage.blob.BlobOutputStream;
import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.CloudBlockBlob;
public class TestOutOfBandAzureBlobOperationsLive { public class TestOutOfBandAzureBlobOperationsLive {
private FileSystem fs; private FileSystem fs;

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
View File

@ -42,8 +42,8 @@ import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import com.microsoft.windowsazure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.windowsazure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.CloudBlockBlob;
public class TestWasbUriAndConfiguration { public class TestWasbUriAndConfiguration {