HDFS-15500. In-order deletion of snapshots: Diff lists must be update only in the last snapshot. (#2233)

This commit is contained in:
Tsz-Wo Nicholas Sze 2020-08-27 02:24:52 -07:00 committed by GitHub
parent d8aaa8c338
commit 41182a9b6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 27 additions and 1 deletions

View File

@ -1569,6 +1569,12 @@ public void checkOperation(OperationCategory op) throws StandbyException {
// null in some unit tests // null in some unit tests
haContext.checkOperation(op); haContext.checkOperation(op);
} }
boolean assertsEnabled = false;
assert assertsEnabled = true; // Intentional side effect!!!
if (assertsEnabled && op == OperationCategory.WRITE) {
getSnapshotManager().initThreadLocals();
}
} }
/** /**

View File

@ -76,7 +76,11 @@ public final void deleteSnapshotDiff(INode.ReclaimContext reclaimContext,
if (diffs == null) { if (diffs == null) {
return; return;
} }
int snapshotIndex = diffs.binarySearch(snapshot); final int snapshotIndex = diffs.binarySearch(snapshot);
// DeletionOrdered: only can remove the element at index 0 and no prior
// check snapshotIndex <= 0 since the diff may not exist
assert !SnapshotManager.isDeletionOrdered()
|| (snapshotIndex <= 0 && prior == Snapshot.NO_SNAPSHOT_ID);
D removed; D removed;
if (snapshotIndex == 0) { if (snapshotIndex == 0) {

View File

@ -57,6 +57,8 @@ public int size() {
@Override @Override
public T remove(int i) { public T remove(int i) {
// DeletionOrdered: only can remove the element at index 0
assert !SnapshotManager.isDeletionOrdered() || i == 0;
return list.remove(i); return list.remove(i);
} }

View File

@ -175,6 +175,8 @@ void combinePosteriorAndCollectBlocks(
final INode.ReclaimContext reclaimContext, final INode.ReclaimContext reclaimContext,
final INodeDirectory currentDir, final INodeDirectory currentDir,
final DirectoryDiff posterior) { final DirectoryDiff posterior) {
// DeletionOrdered: must not combine posterior
assert !SnapshotManager.isDeletionOrdered();
diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() { diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() {
/** Collect blocks for deleted files. */ /** Collect blocks for deleted files. */
@Override @Override

View File

@ -95,6 +95,18 @@ public class SnapshotManager implements SnapshotStatsMXBean {
static final long DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT static final long DFS_NAMENODE_SNAPSHOT_DELETION_ORDERED_GC_PERIOD_MS_DEFAULT
= 5 * 60_000L; //5 minutes = 5 * 60_000L; //5 minutes
private static final ThreadLocal<Boolean> DELETION_ORDERED
= new ThreadLocal<>();
static boolean isDeletionOrdered() {
final Boolean b = DELETION_ORDERED.get();
return b != null? b: false;
}
public void initThreadLocals() {
DELETION_ORDERED.set(isSnapshotDeletionOrdered());
}
private final FSDirectory fsdir; private final FSDirectory fsdir;
private boolean captureOpenFiles; private boolean captureOpenFiles;
/** /**