Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-7440. Consolidate snapshot related operations in a single class. Contributed by Haohui Mai.

This commit is contained in:
Haohui Mai 2014-11-25 21:03:41 -08:00
parent a655973e78
commit 4a31611829
4 changed files with 279 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -395,6 +395,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7436. Consolidate implementation of concat(). (wheat9)
HDFS-7440. Consolidate snapshot related operations in a single class.
(wheat9)
OPTIMIZATIONS
BUG FIXES

222
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java Normal file
View File

@ -0,0 +1,222 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager;
import org.apache.hadoop.hdfs.util.ChunkedArrayList;
import java.io.IOException;
import java.util.List;
class FSDirSnapshotOp {
/** Allow snapshot on a directory. */
static void allowSnapshot(FSDirectory fsd, SnapshotManager snapshotManager,
String path) throws IOException {
fsd.writeLock();
try {
snapshotManager.setSnapshottable(path, true);
} finally {
fsd.writeUnlock();
}
fsd.getEditLog().logAllowSnapshot(path);
}
static void disallowSnapshot(
FSDirectory fsd, SnapshotManager snapshotManager,
String path) throws IOException {
fsd.writeLock();
try {
snapshotManager.resetSnapshottable(path);
} finally {
fsd.writeUnlock();
}
fsd.getEditLog().logDisallowSnapshot(path);
}
/**
* Create a snapshot
* @param snapshotRoot The directory path where the snapshot is taken
* @param snapshotName The name of the snapshot
*/
static String createSnapshot(
FSDirectory fsd, SnapshotManager snapshotManager, String snapshotRoot,
String snapshotName, boolean logRetryCache)
throws IOException {
final FSPermissionChecker pc = fsd.getPermissionChecker();
String snapshotPath = null;
if (fsd.isPermissionEnabled()) {
fsd.checkOwner(pc, snapshotRoot);
}
if (snapshotName == null || snapshotName.isEmpty()) {
snapshotName = Snapshot.generateDefaultSnapshotName();
}
if(snapshotName != null){
if (!DFSUtil.isValidNameForComponent(snapshotName)) {
throw new InvalidPathException("Invalid snapshot name: " +
snapshotName);
}
}
fsd.verifySnapshotName(snapshotName, snapshotRoot);
fsd.writeLock();
try {
snapshotPath = snapshotManager.createSnapshot(snapshotRoot, snapshotName);
} finally {
fsd.writeUnlock();
}
fsd.getEditLog().logCreateSnapshot(snapshotRoot, snapshotName,
logRetryCache);
return snapshotPath;
}
static void renameSnapshot(FSDirectory fsd, SnapshotManager snapshotManager,
String path, String snapshotOldName, String snapshotNewName,
boolean logRetryCache) throws IOException {
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
fsd.checkOwner(pc, path);
}
fsd.verifySnapshotName(snapshotNewName, path);
fsd.writeLock();
try {
snapshotManager.renameSnapshot(path, snapshotOldName, snapshotNewName);
} finally {
fsd.writeUnlock();
}
fsd.getEditLog().logRenameSnapshot(path, snapshotOldName,
snapshotNewName, logRetryCache);
}
static SnapshottableDirectoryStatus[] getSnapshottableDirListing(
FSDirectory fsd, SnapshotManager snapshotManager) throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
fsd.readLock();
try {
final String user = pc.isSuperUser()? null : pc.getUser();
return snapshotManager.getSnapshottableDirListing(user);
} finally {
fsd.readUnlock();
}
}
static SnapshotDiffReport getSnapshotDiffReport(
FSDirectory fsd, SnapshotManager snapshotManager, String path,
String fromSnapshot, String toSnapshot) throws IOException {
SnapshotDiffReport diffs;
final FSPermissionChecker pc = fsd.getPermissionChecker();
fsd.readLock();
try {
if (fsd.isPermissionEnabled()) {
checkSubtreeReadPermission(fsd, pc, path, fromSnapshot);
checkSubtreeReadPermission(fsd, pc, path, toSnapshot);
}
diffs = snapshotManager.diff(path, fromSnapshot, toSnapshot);
} finally {
fsd.readUnlock();
}
return diffs;
}
/**
* Delete a snapshot of a snapshottable directory
* @param snapshotRoot The snapshottable directory
* @param snapshotName The name of the to-be-deleted snapshot
* @throws IOException
*/
static INode.BlocksMapUpdateInfo deleteSnapshot(
FSDirectory fsd, SnapshotManager snapshotManager, String snapshotRoot,
String snapshotName, boolean logRetryCache)
throws IOException {
final FSPermissionChecker pc = fsd.getPermissionChecker();
INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo();
if (fsd.isPermissionEnabled()) {
fsd.checkOwner(pc, snapshotRoot);
}
ChunkedArrayList<INode> removedINodes = new ChunkedArrayList<INode>();
fsd.writeLock();
try {
snapshotManager.deleteSnapshot(snapshotRoot, snapshotName,
collectedBlocks, removedINodes);
fsd.removeFromInodeMap(removedINodes);
} finally {
fsd.writeUnlock();
}
removedINodes.clear();
fsd.getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName,
logRetryCache);
return collectedBlocks;
}
private static void checkSubtreeReadPermission(
FSDirectory fsd, final FSPermissionChecker pc, String snapshottablePath,
String snapshot) throws IOException {
final String fromPath = snapshot == null ?
snapshottablePath : Snapshot.getSnapshotPath(snapshottablePath,
snapshot);
fsd.checkPermission(pc, fromPath, false, null, null, FsAction.READ,
FsAction.READ);
}
/**
* Check if the given INode (or one of its descendants) is snapshottable and
* already has snapshots.
*
* @param target The given INode
* @param snapshottableDirs The list of directories that are snapshottable
* but do not have snapshots yet
*/
static void checkSnapshot(
INode target, List<INodeDirectory> snapshottableDirs)
throws SnapshotException {
if (target.isDirectory()) {
INodeDirectory targetDir = target.asDirectory();
DirectorySnapshottableFeature sf = targetDir
.getDirectorySnapshottableFeature();
if (sf != null) {
if (sf.getNumSnapshots() > 0) {
String fullPath = targetDir.getFullPathName();
throw new SnapshotException("The directory " + fullPath
+ " cannot be deleted since " + fullPath
+ " is snapshottable and already has snapshots");
} else {
if (snapshottableDirs != null) {
snapshottableDirs.add(targetDir);
}
}
}
for (INode child : targetDir.getChildrenList(Snapshot.CURRENT_STATE_ID)) {
checkSnapshot(child, snapshottableDirs);
}
}
}
}

42
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
View File

@ -686,7 +686,7 @@ public class FSDirectory implements Closeable {
List<INodeDirectory> snapshottableDirs = new ArrayList<INodeDirectory>();
if (dstInode != null) { // Destination exists
validateRenameOverwrite(src, dst, overwrite, srcInode, dstInode);
checkSnapshot(dstInode, snapshottableDirs);
FSDirSnapshotOp.checkSnapshot(dstInode, snapshottableDirs);
}
INode dstParent = dstIIP.getINode(-2);
@ -863,7 +863,7 @@ public class FSDirectory implements Closeable {
}
// srcInode and its subtree cannot contain snapshottable directories with
// snapshots
checkSnapshot(srcInode, null);
FSDirSnapshotOp.checkSnapshot(srcInode, null);
}
/**
@ -1208,7 +1208,7 @@ public class FSDirectory implements Closeable {
filesRemoved = -1;
} else {
List<INodeDirectory> snapshottableDirs = new ArrayList<INodeDirectory>();
checkSnapshot(inodesInPath.getLastINode(), snapshottableDirs);
FSDirSnapshotOp.checkSnapshot(inodesInPath.getLastINode(), snapshottableDirs);
filesRemoved = unprotectedDelete(inodesInPath, collectedBlocks,
removedINodes, mtime);
namesystem.removeSnapshottableDirs(snapshottableDirs);
@ -1278,7 +1278,7 @@ public class FSDirectory implements Closeable {
long filesRemoved = -1;
if (deleteAllowed(inodesInPath, src)) {
List<INodeDirectory> snapshottableDirs = new ArrayList<INodeDirectory>();
checkSnapshot(inodesInPath.getLastINode(), snapshottableDirs);
FSDirSnapshotOp.checkSnapshot(inodesInPath.getLastINode(), snapshottableDirs);
filesRemoved = unprotectedDelete(inodesInPath, collectedBlocks,
removedINodes, mtime);
namesystem.removeSnapshottableDirs(snapshottableDirs);
@ -1343,38 +1343,6 @@ public class FSDirectory implements Closeable {
return removed;
}
/**
* Check if the given INode (or one of its descendants) is snapshottable and
* already has snapshots.
*
* @param target The given INode
* @param snapshottableDirs The list of directories that are snapshottable
* but do not have snapshots yet
*/
private static void checkSnapshot(INode target,
List<INodeDirectory> snapshottableDirs) throws SnapshotException {
if (target.isDirectory()) {
INodeDirectory targetDir = target.asDirectory();
DirectorySnapshottableFeature sf = targetDir
.getDirectorySnapshottableFeature();
if (sf != null) {
if (sf.getNumSnapshots() > 0) {
String fullPath = targetDir.getFullPathName();
throw new SnapshotException("The directory " + fullPath
+ " cannot be deleted since " + fullPath
+ " is snapshottable and already has snapshots");
} else {
if (snapshottableDirs != null) {
snapshottableDirs.add(targetDir);
}
}
}
for (INode child : targetDir.getChildrenList(Snapshot.CURRENT_STATE_ID)) {
checkSnapshot(child, snapshottableDirs);
}
}
}
private byte getStoragePolicyID(byte inodePolicy, byte parentPolicy) {
return inodePolicy != BlockStoragePolicySuite.ID_UNSPECIFIED ? inodePolicy :
parentPolicy;
@ -3321,7 +3289,7 @@ public class FSDirectory implements Closeable {
* details of the parameters, see
* {@link FSPermissionChecker#checkPermission}.
*/
private void checkPermission(
void checkPermission(
FSPermissionChecker pc, String path, boolean doCheckOwner,
FsAction ancestorAccess, FsAction parentAccess, FsAction access,
FsAction subAccess)

168
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -7594,55 +7594,39 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/** Allow snapshot on a directory. */
void allowSnapshot(String path) throws SafeModeException, IOException {
void allowSnapshot(String path) throws IOException {
checkOperation(OperationCategory.WRITE);
boolean success = false;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot allow snapshot for " + path);
checkSuperuserPrivilege();
dir.writeLock();
try {
snapshotManager.setSnapshottable(path, true);
} finally {
dir.writeUnlock();
}
getEditLog().logAllowSnapshot(path);
FSDirSnapshotOp.allowSnapshot(dir, snapshotManager, path);
success = true;
} finally {
writeUnlock();
}
getEditLog().logSync();
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "allowSnapshot", path, null, null);
}
logAuditEvent(success, "allowSnapshot", path, null, null);
}
/** Disallow snapshot on a directory. */
void disallowSnapshot(String path) throws SafeModeException, IOException {
void disallowSnapshot(String path) throws IOException {
checkOperation(OperationCategory.WRITE);
boolean success = false;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot disallow snapshot for " + path);
checkSuperuserPrivilege();
dir.writeLock();
try {
snapshotManager.resetSnapshottable(path);
} finally {
dir.writeUnlock();
}
getEditLog().logDisallowSnapshot(path);
FSDirSnapshotOp.disallowSnapshot(dir, snapshotManager, path);
success = true;
} finally {
writeUnlock();
}
getEditLog().logSync();
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "disallowSnapshot", path, null, null);
}
logAuditEvent(success, "disallowSnapshot", path, null, null);
}
/**
@ -7651,45 +7635,20 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* @param snapshotName The name of the snapshot
*/
String createSnapshot(String snapshotRoot, String snapshotName,
boolean logRetryCache)
throws SafeModeException, IOException {
checkOperation(OperationCategory.WRITE);
final FSPermissionChecker pc = getPermissionChecker();
boolean logRetryCache) throws IOException {
String snapshotPath = null;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot create snapshot for " + snapshotRoot);
if (isPermissionEnabled) {
checkOwner(pc, snapshotRoot);
}
if (snapshotName == null || snapshotName.isEmpty()) {
snapshotName = Snapshot.generateDefaultSnapshotName();
}
if(snapshotName != null){
if (!DFSUtil.isValidNameForComponent(snapshotName)) {
throw new InvalidPathException("Invalid snapshot name: "
+ snapshotName);
}
}
dir.verifySnapshotName(snapshotName, snapshotRoot);
dir.writeLock();
try {
snapshotPath = snapshotManager.createSnapshot(snapshotRoot, snapshotName);
} finally {
dir.writeUnlock();
}
getEditLog().logCreateSnapshot(snapshotRoot, snapshotName, logRetryCache);
snapshotPath = FSDirSnapshotOp.createSnapshot(dir,
snapshotManager, snapshotRoot, snapshotName, logRetryCache);
} finally {
writeUnlock();
}
getEditLog().logSync();
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "createSnapshot", snapshotRoot, snapshotPath, null);
}
logAuditEvent(snapshotPath != null, "createSnapshot", snapshotRoot,
snapshotPath, null);
return snapshotPath;
}
@ -7705,32 +7664,22 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
String path, String snapshotOldName, String snapshotNewName,
boolean logRetryCache) throws IOException {
checkOperation(OperationCategory.WRITE);
final FSPermissionChecker pc = getPermissionChecker();
boolean success = false;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot rename snapshot for " + path);
if (isPermissionEnabled) {
checkOwner(pc, path);
}
dir.verifySnapshotName(snapshotNewName, path);
snapshotManager.renameSnapshot(path, snapshotOldName, snapshotNewName);
getEditLog().logRenameSnapshot(path, snapshotOldName, snapshotNewName,
logRetryCache);
FSDirSnapshotOp.renameSnapshot(dir, snapshotManager, path,
snapshotOldName, snapshotNewName, logRetryCache);
success = true;
} finally {
writeUnlock();
}
getEditLog().logSync();
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
String oldSnapshotRoot = Snapshot.getSnapshotPath(path, snapshotOldName);
String newSnapshotRoot = Snapshot.getSnapshotPath(path, snapshotNewName);
logAuditEvent(true, "renameSnapshot", oldSnapshotRoot, newSnapshotRoot, null);
}
logAuditEvent(success, "renameSnapshot", oldSnapshotRoot,
newSnapshotRoot, null);
}
/**
@ -7744,18 +7693,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throws IOException {
SnapshottableDirectoryStatus[] status = null;
checkOperation(OperationCategory.READ);
final FSPermissionChecker checker = getPermissionChecker();
boolean success = false;
readLock();
try {
checkOperation(OperationCategory.READ);
final String user = checker.isSuperUser()? null : checker.getUser();
status = snapshotManager.getSnapshottableDirListing(user);
status = FSDirSnapshotOp.getSnapshottableDirListing(dir, snapshotManager);
success = true;
} finally {
readUnlock();
}
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "listSnapshottableDirectory", null, null, null);
}
logAuditEvent(success, "listSnapshottableDirectory", null, null, null);
return status;
}
@ -7776,35 +7723,21 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
SnapshotDiffReport getSnapshotDiffReport(String path,
String fromSnapshot, String toSnapshot) throws IOException {
SnapshotDiffReport diffs;
SnapshotDiffReport diffs = null;
checkOperation(OperationCategory.READ);
final FSPermissionChecker pc = getPermissionChecker();
readLock();
try {
checkOperation(OperationCategory.READ);
if (isPermissionEnabled) {
checkSubtreeReadPermission(pc, path, fromSnapshot);
checkSubtreeReadPermission(pc, path, toSnapshot);
}
diffs = snapshotManager.diff(path, fromSnapshot, toSnapshot);
diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, snapshotManager,
path, fromSnapshot, toSnapshot);
} finally {
readUnlock();
}
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
logAuditEvent(true, "computeSnapshotDiff", null, null, null);
}
logAuditEvent(diffs != null, "computeSnapshotDiff", null, null, null);
return diffs;
}
private void checkSubtreeReadPermission(final FSPermissionChecker pc,
final String snapshottablePath, final String snapshot)
throws AccessControlException, UnresolvedLinkException {
final String fromPath = snapshot == null?
snapshottablePath: Snapshot.getSnapshotPath(snapshottablePath, snapshot);
checkPermission(pc, fromPath, false, null, null, FsAction.READ, FsAction.READ);
}
/**
* Delete a snapshot of a snapshottable directory
* @param snapshotRoot The snapshottable directory
@ -7812,45 +7745,32 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* @throws SafeModeException
* @throws IOException
*/
void deleteSnapshot(String snapshotRoot, String snapshotName,
boolean logRetryCache)
throws SafeModeException, IOException {
void deleteSnapshot(
String snapshotRoot, String snapshotName, boolean logRetryCache)
throws IOException {
checkOperation(OperationCategory.WRITE);
final FSPermissionChecker pc = getPermissionChecker();
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
boolean success = false;
writeLock();
BlocksMapUpdateInfo blocksToBeDeleted = null;
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot delete snapshot for " + snapshotRoot);
if (isPermissionEnabled) {
checkOwner(pc, snapshotRoot);
}
List<INode> removedINodes = new ChunkedArrayList<INode>();
dir.writeLock();
try {
snapshotManager.deleteSnapshot(snapshotRoot, snapshotName,
collectedBlocks, removedINodes);
dir.removeFromInodeMap(removedINodes);
} finally {
dir.writeUnlock();
}
removedINodes.clear();
getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName, logRetryCache);
blocksToBeDeleted = FSDirSnapshotOp.deleteSnapshot(dir, snapshotManager,
snapshotRoot, snapshotName, logRetryCache);
success = true;
} finally {
writeUnlock();
}
getEditLog().logSync();
removeBlocks(collectedBlocks);
collectedBlocks.clear();
if (auditLog.isInfoEnabled() && isExternalInvocation()) {
String rootPath = Snapshot.getSnapshotPath(snapshotRoot, snapshotName);
logAuditEvent(true, "deleteSnapshot", rootPath, null, null);
// Breaking the pattern as removing blocks have to happen outside of the
// global lock
if (blocksToBeDeleted != null) {
removeBlocks(blocksToBeDeleted);
}
String rootPath = Snapshot.getSnapshotPath(snapshotRoot, snapshotName);
logAuditEvent(success, "deleteSnapshot", rootPath, null, null);
}
/**