Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-12634. Change Lazy Rename Pending Operation Completion of WASB to address case of potential data loss due to partial copy. Contributed by Gaurav Kanade.

Browse Source 
(cherry picked from commit 978bbdfeb2d12efd6e750da6a14849e072fb814b)
(cherry picked from commit 4753676a34c4a3de29f9d20281de09533b73ce53)

Conflicts:
	hadoop-common-project/hadoop-common/CHANGES.txt
This commit is contained in:
cnauroth 2016-01-06 11:15:59 -08:00
parent ae535ec93e
commit 506a517f9f
3 changed files with 27 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -904,6 +904,10 @@ Release 2.8.0 - UNRELEASED
HADOOP-12608. Fix exception message in WASB when connecting with anonymous
credential. (Dushyanth via xyao)
HADOOP-12634. Change Lazy Rename Pending Operation Completion of WASB to
address case of potential data loss due to partial copy
(Gaurav Kanade via cnauroth)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

34
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
View File

@ -536,45 +536,13 @@ private void finishSingleFileRename(String fileName)
Path dstFile = fullPath(dstKey, fileName);
boolean srcExists = fs.exists(srcFile);
boolean dstExists = fs.exists(dstFile);
if (srcExists && !dstExists) {
if(srcExists) {
// Rename gets exclusive access (via a lease) for HBase write-ahead log
// (WAL) file processing correctness. See the rename code for details.
String srcName = fs.pathToKey(srcFile);
String dstName = fs.pathToKey(dstFile);
fs.getStoreInterface().rename(srcName, dstName, true, null);
} else if (srcExists && dstExists) {
// Get a lease on source to block write access.
String srcName = fs.pathToKey(srcFile);
SelfRenewingLease lease = null;
try {
lease = fs.acquireLease(srcFile);
// Delete the file. This will free the lease too.
fs.getStoreInterface().delete(srcName, lease);
} catch(AzureException e) {
String errorCode = "";
try {
StorageException e2 = (StorageException) e.getCause();
errorCode = e2.getErrorCode();
} catch(Exception e3) {
// do nothing if cast fails
}
// If the rename already finished do nothing
if(!errorCode.equals("BlobNotFound")){
throw e;
}
} finally {
try {
if(lease != null){
lease.free();
}
} catch(StorageException e) {
LOG.warn("Unable to free lease because: " + e.getMessage());
}
}
} else if (!srcExists && dstExists) {
// The rename already finished, so do nothing.
;
} else {

22
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java
View File

@ -24,6 +24,8 @@
import java.util.concurrent.CountDownLatch;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@ -43,6 +45,26 @@ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
return AzureBlobStorageTestAccount.create();
}
@Test
public void testLazyRenamePendingCanOverwriteExistingFile()
throws Exception {
final String SRC_FILE_KEY = "srcFile";
final String DST_FILE_KEY = "dstFile";
Path srcPath = new Path(SRC_FILE_KEY);
FSDataOutputStream srcStream = fs.create(srcPath);
assertTrue(fs.exists(srcPath));
Path dstPath = new Path(DST_FILE_KEY);
FSDataOutputStream dstStream = fs.create(dstPath);
assertTrue(fs.exists(dstPath));
NativeAzureFileSystem nfs = (NativeAzureFileSystem)fs;
final String fullSrcKey = nfs.pathToKey(nfs.makeAbsolute(srcPath));
final String fullDstKey = nfs.pathToKey(nfs.makeAbsolute(dstPath));
nfs.getStoreInterface().rename(fullSrcKey, fullDstKey, true, null);
assertTrue(fs.exists(dstPath));
assertFalse(fs.exists(srcPath));
IOUtils.cleanup(null, srcStream);
IOUtils.cleanup(null, dstStream);
}
/**
* Tests fs.delete() function to delete a blob when another blob is holding a
* lease on it. Delete if called without a lease should fail if another process