HDFS-12042. Lazy initialize AbstractINodeDiffList#diffs for snapshots to reduce memory consumption. Contributed by Misha Dmitriev.

This commit is contained in:
Wei-Chiu Chuang 2017-06-30 10:28:01 -07:00
parent 6a9dc5f44b
commit bcba844d11
3 changed files with 46 additions and 15 deletions

View File

@ -65,8 +65,11 @@ public class INodeDirectory extends INodeWithAdditionalFields
return inode.asDirectory();
}
protected static final int DEFAULT_FILES_PER_DIRECTORY = 5;
final static byte[] ROOT_NAME = DFSUtil.string2Bytes("");
// Profiling shows that most of the file lists are between 1 and 4 elements.
// Thus allocate the corresponding ArrayLists with a small initial capacity.
public static final int DEFAULT_FILES_PER_DIRECTORY = 2;
static final byte[] ROOT_NAME = DFSUtil.string2Bytes("");
private List<INode> children = null;

View File

@ -24,6 +24,7 @@ import java.util.List;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
/**
* A list of snapshot diffs for storing snapshot data.
@ -35,17 +36,19 @@ abstract class AbstractINodeDiffList<N extends INode,
A extends INodeAttributes,
D extends AbstractINodeDiff<N, A, D>>
implements Iterable<D> {
/** Diff list sorted by snapshot IDs, i.e. in chronological order. */
private final List<D> diffs = new ArrayList<D>();
/** Diff list sorted by snapshot IDs, i.e. in chronological order.
* Created lazily to avoid wasting memory by empty lists. */
private List<D> diffs;
/** @return this list as a unmodifiable {@link List}. */
public final List<D> asList() {
return Collections.unmodifiableList(diffs);
return diffs != null ?
Collections.unmodifiableList(diffs) : Collections.emptyList();
}
/** Get the size of the list and then clear it. */
/** Clear the list. */
public void clear() {
diffs.clear();
diffs = null;
}
/** @return an {@link AbstractINodeDiff}. */
@ -66,6 +69,9 @@ abstract class AbstractINodeDiffList<N extends INode,
*/
public final void deleteSnapshotDiff(INode.ReclaimContext reclaimContext,
final int snapshot, final int prior, final N currentINode) {
if (diffs == null) {
return;
}
int snapshotIndex = Collections.binarySearch(diffs, snapshot);
D removed;
@ -75,6 +81,9 @@ abstract class AbstractINodeDiffList<N extends INode,
diffs.get(snapshotIndex).setSnapshotId(prior);
} else { // there is no snapshot before
removed = diffs.remove(0);
if (diffs.isEmpty()) {
diffs = null;
}
removed.destroyDiffAndCollectBlocks(reclaimContext, currentINode);
}
} else if (snapshotIndex > 0) {
@ -103,6 +112,7 @@ abstract class AbstractINodeDiffList<N extends INode,
/** Append the diff at the end of the list. */
private D addLast(D diff) {
createDiffsIfNeeded();
final D last = getLast();
diffs.add(diff);
if (last != null) {
@ -113,15 +123,25 @@ abstract class AbstractINodeDiffList<N extends INode,
/** Add the diff to the beginning of the list. */
final void addFirst(D diff) {
final D first = diffs.isEmpty()? null: diffs.get(0);
createDiffsIfNeeded();
final D first = diffs.isEmpty()? null : diffs.get(0);
diffs.add(0, diff);
diff.setPosterior(first);
}
/** @return the last diff. */
public final D getLast() {
final int n = diffs.size();
return n == 0? null: diffs.get(n - 1);
if (diffs == null) {
return null;
}
int n = diffs.size();
return n == 0 ? null : diffs.get(n - 1);
}
private void createDiffsIfNeeded() {
if (diffs == null) {
diffs = new ArrayList<>(INodeDirectory.DEFAULT_FILES_PER_DIRECTORY);
}
}
/** @return the id of the last snapshot. */
@ -139,10 +159,14 @@ abstract class AbstractINodeDiffList<N extends INode,
* @return The id of the latest snapshot before the given snapshot.
*/
public final int getPrior(int anchorId, boolean exclusive) {
if (diffs == null) {
return Snapshot.NO_SNAPSHOT_ID;
}
if (anchorId == Snapshot.CURRENT_STATE_ID) {
int last = getLastSnapshotId();
if(exclusive && last == anchorId)
if (exclusive && last == anchorId) {
return Snapshot.NO_SNAPSHOT_ID;
}
return last;
}
final int i = Collections.binarySearch(diffs, anchorId);
@ -181,7 +205,7 @@ abstract class AbstractINodeDiffList<N extends INode,
}
public final D getDiffById(final int snapshotId) {
if (snapshotId == Snapshot.CURRENT_STATE_ID) {
if (snapshotId == Snapshot.CURRENT_STATE_ID || diffs == null) {
return null;
}
final int i = Collections.binarySearch(diffs, snapshotId);
@ -193,7 +217,7 @@ abstract class AbstractINodeDiffList<N extends INode,
// given snapshot and the next state so that the diff for the given
// snapshot was not recorded. Thus, return the next state.
final int j = -i - 1;
return j < diffs.size()? diffs.get(j): null;
return j < diffs.size() ? diffs.get(j) : null;
}
}
@ -207,6 +231,9 @@ abstract class AbstractINodeDiffList<N extends INode,
}
final int[] changedBetweenSnapshots(Snapshot from, Snapshot to) {
if (diffs == null) {
return null;
}
Snapshot earlier = from;
Snapshot later = to;
if (Snapshot.ID_COMPARATOR.compare(from, to) > 0) {
@ -275,11 +302,11 @@ abstract class AbstractINodeDiffList<N extends INode,
@Override
public Iterator<D> iterator() {
return diffs.iterator();
return diffs != null ? diffs.iterator() : Collections.emptyIterator();
}
@Override
public String toString() {
return getClass().getSimpleName() + ": " + diffs;
return getClass().getSimpleName() + ": " + (diffs != null ? diffs : "[]");
}
}

View File

@ -156,6 +156,7 @@ public class TestTruncateQuotaUpdate {
FileDiff diff = mock(FileDiff.class);
when(diff.getBlocks()).thenReturn(blocks);
FileDiffList diffList = new FileDiffList();
Whitebox.setInternalState(diffList, "diffs", new ArrayList<FileDiff>());
@SuppressWarnings("unchecked")
ArrayList<FileDiff> diffs = ((ArrayList<FileDiff>)Whitebox.getInternalState
(diffList, "diffs"));