Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-11685. StorageException complaining " no lease ID" during HBase distributed log splitting. Contributed by Duo Xu.

This commit is contained in:
cnauroth 2015-10-27 22:56:22 -07:00
parent 73822de7c3
commit 1f7ecb0c84
4 changed files with 48 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -919,6 +919,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12520. Use XInclude in hadoop-azure test configuration to isolate
Azure Storage account keys for service integration tests. (cnauroth)
HADOOP-11685. StorageException complaining " no lease ID" during HBase
distributed log splitting (Duo Xu via cnauroth)
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp

15
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
View File

@ -1503,10 +1503,23 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
storePermissionStatus(blob, permissionStatus);
storeFolderAttribute(blob);
openOutputStream(blob).close();
} catch (Exception e) {
} catch (StorageException e) {
// Caught exception while attempting upload. Re-throw as an Azure
// storage exception.
throw new AzureException(e);
} catch (URISyntaxException e) {
throw new AzureException(e);
} catch (IOException e) {
Throwable t = e.getCause();
if (t != null && t instanceof StorageException) {
StorageException se = (StorageException) t;
// If we got this exception, the blob should have already been created
if (!se.getErrorCode().equals("LeaseIdMissing")) {
throw new AzureException(e);
}
} else {
throw new AzureException(e);
}
}
}

5
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
View File

@ -1870,8 +1870,9 @@ public class NativeAzureFileSystem extends FileSystem {
* @throws IOException
* If login fails in getCurrentUser
*/
private PermissionStatus createPermissionStatus(FsPermission permission)
throws IOException {
@VisibleForTesting
PermissionStatus createPermissionStatus(FsPermission permission)
throws IOException {
// Create the permission status for this file based on current user
return new PermissionStatus(
UserGroupInformation.getCurrentUser().getShortUserName(),

29
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java
View File

@ -21,7 +21,8 @@ package org.apache.hadoop.fs.azure;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.Test;
/*
@ -104,4 +105,30 @@ public class TestNativeAzureFileSystemLive extends
assertTrue(store.isAtomicRenameKey(uriPrefix + s));
}
}
/**
* Tests fs.mkdir() function to create a target blob while another thread
* is holding the lease on the blob. mkdir should not fail since the blob
* already exists.
* This is a scenario that would happen in HBase distributed log splitting.
* Multiple threads will try to create and update "recovered.edits" folder
* under the same path.
*/
@Test
public void testMkdirOnExistingFolderWithLease() throws Exception {
SelfRenewingLease lease;
final String FILE_KEY = "folderWithLease";
// Create the folder
fs.mkdirs(new Path(FILE_KEY));
NativeAzureFileSystem nfs = (NativeAzureFileSystem) fs;
String fullKey = nfs.pathToKey(nfs.makeAbsolute(new Path(FILE_KEY)));
AzureNativeFileSystemStore store = nfs.getStore();
// Acquire the lease on the folder
lease = store.acquireLease(fullKey);
assertTrue(lease.getLeaseID() != null);
// Try to create the same folder
store.storeEmptyFolder(fullKey,
nfs.createPermissionStatus(FsPermission.getDirDefault()));
lease.free();
}
}