HDFS-14547. Improve memory efficiency of quotas when storage type quotas are not set. Contributed by Jinglun.

This commit is contained in:
Erik Krogen 2019-07-08 14:46:25 -07:00
parent de6b7bc67a
commit 4632708148
4 changed files with 381 additions and 34 deletions

View File

@ -18,53 +18,119 @@
package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.util.ConstEnumCounters;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.hdfs.util.ConstEnumCounters.ConstEnumException;
import java.util.function.Consumer;
/**
* Counters for namespace, storage space and storage type space quota and usage.
*/
public class QuotaCounts {
/**
* We pre-define 4 most common used EnumCounters objects. When the nsSsCounts
* and tsCounts are set to the 4 most common used value, we just point them to
* the pre-defined const EnumCounters objects instead of constructing many
* objects with the same value. See HDFS-14547.
*/
final static EnumCounters<Quota> QUOTA_RESET =
new ConstEnumCounters<>(Quota.class, HdfsConstants.QUOTA_RESET);
final static EnumCounters<Quota> QUOTA_DEFAULT =
new ConstEnumCounters<>(Quota.class, 0);
final static EnumCounters<StorageType> STORAGE_TYPE_RESET =
new ConstEnumCounters<>(StorageType.class, HdfsConstants.QUOTA_RESET);
final static EnumCounters<StorageType> STORAGE_TYPE_DEFAULT =
new ConstEnumCounters<>(StorageType.class, 0);
/**
* Modify counter with action. If the counter is ConstEnumCounters, copy all
* the values of it to a new EnumCounters object, and modify the new obj.
*
* @param counter the EnumCounters to be modified.
* @param action the modifying action on counter.
* @return the modified counter.
*/
static <T extends Enum<T>> EnumCounters<T> modify(EnumCounters<T> counter,
Consumer<EnumCounters<T>> action) {
try {
action.accept(counter);
} catch (ConstEnumException cee) {
// We don't call clone here because ConstEnumCounters.clone() will return
// an object of class ConstEnumCounters. We want EnumCounters.
counter = counter.deepCopyEnumCounter();
action.accept(counter);
}
return counter;
}
// Name space and storage space counts (HDFS-7775 refactors the original disk
// space count to storage space counts)
private EnumCounters<Quota> nsSsCounts;
@VisibleForTesting
EnumCounters<Quota> nsSsCounts;
// Storage type space counts
private EnumCounters<StorageType> tsCounts;
@VisibleForTesting
EnumCounters<StorageType> tsCounts;
public static class Builder {
private EnumCounters<Quota> nsSsCounts;
private EnumCounters<StorageType> tsCounts;
public Builder() {
this.nsSsCounts = new EnumCounters<Quota>(Quota.class);
this.tsCounts = new EnumCounters<StorageType>(StorageType.class);
this.nsSsCounts = QUOTA_DEFAULT;
this.tsCounts = STORAGE_TYPE_DEFAULT;
}
public Builder nameSpace(long val) {
this.nsSsCounts.set(Quota.NAMESPACE, val);
nsSsCounts =
setQuotaCounter(nsSsCounts, Quota.NAMESPACE, Quota.STORAGESPACE, val);
return this;
}
public Builder storageSpace(long val) {
this.nsSsCounts.set(Quota.STORAGESPACE, val);
nsSsCounts =
setQuotaCounter(nsSsCounts, Quota.STORAGESPACE, Quota.NAMESPACE, val);
return this;
}
public Builder typeSpaces(EnumCounters<StorageType> val) {
if (val != null) {
this.tsCounts.set(val);
if (val == STORAGE_TYPE_DEFAULT || val == STORAGE_TYPE_RESET) {
tsCounts = val;
} else {
tsCounts = modify(tsCounts, ec -> ec.set(val));
}
}
return this;
}
public Builder typeSpaces(long val) {
this.tsCounts.reset(val);
if (val == HdfsConstants.QUOTA_RESET) {
tsCounts = STORAGE_TYPE_RESET;
} else if (val == 0) {
tsCounts = STORAGE_TYPE_DEFAULT;
} else {
tsCounts = modify(tsCounts, ec -> ec.reset(val));
}
return this;
}
public Builder quotaCount(QuotaCounts that) {
this.nsSsCounts.set(that.nsSsCounts);
this.tsCounts.set(that.tsCounts);
if (that.nsSsCounts == QUOTA_DEFAULT || that.nsSsCounts == QUOTA_RESET) {
nsSsCounts = that.nsSsCounts;
} else {
nsSsCounts = modify(nsSsCounts, ec -> ec.set(that.nsSsCounts));
}
if (that.tsCounts == STORAGE_TYPE_DEFAULT
|| that.tsCounts == STORAGE_TYPE_RESET) {
tsCounts = that.tsCounts;
} else {
tsCounts = modify(tsCounts, ec -> ec.set(that.tsCounts));
}
return this;
}
@ -79,14 +145,14 @@ public class QuotaCounts {
}
public QuotaCounts add(QuotaCounts that) {
this.nsSsCounts.add(that.nsSsCounts);
this.tsCounts.add(that.tsCounts);
nsSsCounts = modify(nsSsCounts, ec -> ec.add(that.nsSsCounts));
tsCounts = modify(tsCounts, ec -> ec.add(that.tsCounts));
return this;
}
public QuotaCounts subtract(QuotaCounts that) {
this.nsSsCounts.subtract(that.nsSsCounts);
this.tsCounts.subtract(that.tsCounts);
nsSsCounts = modify(nsSsCounts, ec -> ec.subtract(that.nsSsCounts));
tsCounts = modify(tsCounts, ec -> ec.subtract(that.tsCounts));
return this;
}
@ -97,8 +163,8 @@ public class QuotaCounts {
*/
public QuotaCounts negation() {
QuotaCounts ret = new QuotaCounts.Builder().quotaCount(this).build();
ret.nsSsCounts.negation();
ret.tsCounts.negation();
ret.nsSsCounts = modify(ret.nsSsCounts, ec -> ec.negation());
ret.tsCounts = modify(ret.tsCounts, ec -> ec.negation());
return ret;
}
@ -107,11 +173,13 @@ public class QuotaCounts {
}
public void setNameSpace(long nameSpaceCount) {
this.nsSsCounts.set(Quota.NAMESPACE, nameSpaceCount);
nsSsCounts =
setQuotaCounter(nsSsCounts, Quota.NAMESPACE, Quota.STORAGESPACE,
nameSpaceCount);
}
public void addNameSpace(long nsDelta) {
this.nsSsCounts.add(Quota.NAMESPACE, nsDelta);
nsSsCounts = modify(nsSsCounts, ec -> ec.add(Quota.NAMESPACE, nsDelta));
}
public long getStorageSpace(){
@ -119,11 +187,13 @@ public class QuotaCounts {
}
public void setStorageSpace(long spaceCount) {
this.nsSsCounts.set(Quota.STORAGESPACE, spaceCount);
nsSsCounts =
setQuotaCounter(nsSsCounts, Quota.STORAGESPACE, Quota.NAMESPACE,
spaceCount);
}
public void addStorageSpace(long dsDelta) {
this.nsSsCounts.add(Quota.STORAGESPACE, dsDelta);
nsSsCounts = modify(nsSsCounts, ec -> ec.add(Quota.STORAGESPACE, dsDelta));
}
public EnumCounters<StorageType> getTypeSpaces() {
@ -134,8 +204,10 @@ public class QuotaCounts {
}
void setTypeSpaces(EnumCounters<StorageType> that) {
if (that != null) {
this.tsCounts.set(that);
if (that == STORAGE_TYPE_DEFAULT || that == STORAGE_TYPE_RESET) {
tsCounts = that;
} else if (that != null) {
tsCounts = modify(tsCounts, ec -> ec.set(that));
}
}
@ -144,21 +216,54 @@ public class QuotaCounts {
}
void setTypeSpace(StorageType type, long spaceCount) {
this.tsCounts.set(type, spaceCount);
tsCounts = modify(tsCounts, ec -> ec.set(type, spaceCount));
}
public void addTypeSpace(StorageType type, long delta) {
this.tsCounts.add(type, delta);
tsCounts = modify(tsCounts, ec -> ec.add(type, delta));
}
public boolean anyNsSsCountGreaterOrEqual(long val) {
if (nsSsCounts == QUOTA_DEFAULT) {
return val <= 0;
} else if (nsSsCounts == QUOTA_RESET) {
return val <= HdfsConstants.QUOTA_RESET;
}
return nsSsCounts.anyGreaterOrEqual(val);
}
public boolean anyTypeSpaceCountGreaterOrEqual(long val) {
if (tsCounts == STORAGE_TYPE_DEFAULT) {
return val <= 0;
} else if (tsCounts == STORAGE_TYPE_RESET) {
return val <= HdfsConstants.QUOTA_RESET;
}
return tsCounts.anyGreaterOrEqual(val);
}
/**
* Set inputCounts' value of Quota type quotaToSet to val.
* inputCounts should be the left side value of this method.
*
* @param inputCounts the EnumCounters instance.
* @param quotaToSet the quota type to be set.
* @param otherQuota the other quota type besides quotaToSet.
* @param val the value to be set.
* @return the modified inputCounts.
*/
private static EnumCounters<Quota> setQuotaCounter(
EnumCounters<Quota> inputCounts, Quota quotaToSet, Quota otherQuota,
long val) {
if (val == HdfsConstants.QUOTA_RESET
&& inputCounts.get(otherQuota) == HdfsConstants.QUOTA_RESET) {
return QUOTA_RESET;
} else if (val == 0 && inputCounts.get(otherQuota) == 0) {
return QUOTA_DEFAULT;
} else {
return modify(inputCounts, ec -> ec.set(quotaToSet, val));
}
}
@Override
public String toString() {
return "name space=" + getNameSpace() +

View File

@ -0,0 +1,98 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.util;
/**
* Const Counters for an enum type.
*
* It's the const version of EnumCounters. Any modification ends with a
* ConstEnumException.
*
* @see org.apache.hadoop.hdfs.util.EnumCounters
*/
public class ConstEnumCounters<E extends Enum<E>> extends EnumCounters<E> {
/**
* An exception class for modification on ConstEnumCounters.
*/
public static final class ConstEnumException extends RuntimeException {
private ConstEnumException(String msg) {
super(msg);
}
}
/**
* Throwing this exception if any modification occurs.
*/
private static final ConstEnumException CONST_ENUM_EXCEPTION =
new ConstEnumException("modification on const.");
public ConstEnumCounters(Class<E> enumClass, long defaultVal) {
super(enumClass);
forceReset(defaultVal);
}
@Override
public final void negation() {
throw CONST_ENUM_EXCEPTION;
}
@Override
public final void set(final E e, final long value) {
throw CONST_ENUM_EXCEPTION;
}
@Override
public final void set(final EnumCounters<E> that) {
throw CONST_ENUM_EXCEPTION;
}
@Override
public final void reset() {
throw CONST_ENUM_EXCEPTION;
}
@Override
public final void add(final E e, final long value) {
throw CONST_ENUM_EXCEPTION;
}
@Override
public final void add(final EnumCounters<E> that) {
throw CONST_ENUM_EXCEPTION;
}
@Override
public final void subtract(final E e, final long value) {
throw CONST_ENUM_EXCEPTION;
}
@Override
public final void subtract(final EnumCounters<E> that) {
throw CONST_ENUM_EXCEPTION;
}
@Override
public final void reset(long val) {
throw CONST_ENUM_EXCEPTION;
}
private void forceReset(long val) {
super.reset(val);
}
}

View File

@ -70,55 +70,55 @@ public class EnumCounters<E extends Enum<E>> {
}
/** Negate all counters. */
public final void negation() {
public void negation() {
for(int i = 0; i < counters.length; i++) {
counters[i] = -counters[i];
}
}
/** Set counter e to the given value. */
public final void set(final E e, final long value) {
public void set(final E e, final long value) {
counters[e.ordinal()] = value;
}
/** Set this counters to that counters. */
public final void set(final EnumCounters<E> that) {
public void set(final EnumCounters<E> that) {
for(int i = 0; i < counters.length; i++) {
this.counters[i] = that.counters[i];
}
}
/** Reset all counters to zero. */
public final void reset() {
public void reset() {
reset(0L);
}
/** Add the given value to counter e. */
public final void add(final E e, final long value) {
public void add(final E e, final long value) {
counters[e.ordinal()] += value;
}
/** Add that counters to this counters. */
public final void add(final EnumCounters<E> that) {
public void add(final EnumCounters<E> that) {
for(int i = 0; i < counters.length; i++) {
this.counters[i] += that.counters[i];
}
}
/** Subtract the given value from counter e. */
public final void subtract(final E e, final long value) {
public void subtract(final E e, final long value) {
counters[e.ordinal()] -= value;
}
/** Subtract this counters from that counters. */
public final void subtract(final EnumCounters<E> that) {
public void subtract(final EnumCounters<E> that) {
for(int i = 0; i < counters.length; i++) {
this.counters[i] -= that.counters[i];
}
}
/** @return the sum of all counters. */
public final long sum() {
public long sum() {
long sum = 0;
for(int i = 0; i < counters.length; i++) {
sum += counters[i];
@ -138,6 +138,15 @@ public class EnumCounters<E extends Enum<E>> {
&& Arrays.equals(this.counters, that.counters);
}
/**
* Return a deep copy of EnumCounter.
*/
public EnumCounters<E> deepCopyEnumCounter() {
EnumCounters<E> newCounter = new EnumCounters<>(enumClass);
newCounter.set(this);
return newCounter;
}
@Override
public int hashCode() {
return Arrays.hashCode(counters);
@ -154,7 +163,7 @@ public class EnumCounters<E extends Enum<E>> {
return b.substring(0, b.length() - 2);
}
public final void reset(long val) {
public void reset(long val) {
for(int i = 0; i < counters.length; i++) {
this.counters[i] = val;
}

View File

@ -0,0 +1,135 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
/**
* Test QuotaCounts.
*/
public class TestQuotaCounts {
@Test
public void testBuildConstEnumCounters() throws Exception {
QuotaCounts qc =
new QuotaCounts.Builder().nameSpace(HdfsConstants.QUOTA_RESET)
.storageSpace(HdfsConstants.QUOTA_RESET).build();
// compare the references
assertSame(QuotaCounts.QUOTA_RESET, qc.nsSsCounts);
assertSame(QuotaCounts.STORAGE_TYPE_DEFAULT, qc.tsCounts);
// compare the values
assertEquals(HdfsConstants.QUOTA_RESET, qc.getNameSpace());
assertEquals(HdfsConstants.QUOTA_RESET, qc.getStorageSpace());
for (StorageType st : StorageType.values()) {
assertEquals(0, qc.getTypeSpace(st));
}
}
@Test
public void testAddSpace() throws Exception {
QuotaCounts qc = new QuotaCounts.Builder().build();
qc.addNameSpace(1);
qc.addStorageSpace(1024);
assertEquals(1, qc.getNameSpace());
assertEquals(1024, qc.getStorageSpace());
}
@Test
public void testAdd() throws Exception {
QuotaCounts qc1 = new QuotaCounts.Builder().build();
QuotaCounts qc2 = new QuotaCounts.Builder().nameSpace(1).storageSpace(512)
.typeSpaces(5).build();
qc1.add(qc2);
assertEquals(1, qc1.getNameSpace());
assertEquals(512, qc1.getStorageSpace());
for (StorageType type : StorageType.values()) {
assertEquals(5, qc1.getTypeSpace(type));
}
}
@Test
public void testAddTypeSpaces() throws Exception {
QuotaCounts qc = new QuotaCounts.Builder().build();
for (StorageType t : StorageType.values()) {
qc.addTypeSpace(t, 10);
}
for (StorageType type : StorageType.values()) {
assertEquals(10, qc.getTypeSpace(type));
}
}
@Test
public void testSubtract() throws Exception {
QuotaCounts qc1 = new QuotaCounts.Builder().build();
QuotaCounts qc2 = new QuotaCounts.Builder().nameSpace(1).storageSpace(512)
.typeSpaces(5).build();
qc1.subtract(qc2);
assertEquals(-1, qc1.getNameSpace());
assertEquals(-512, qc1.getStorageSpace());
for (StorageType type : StorageType.values()) {
assertEquals(-5, qc1.getTypeSpace(type));
}
}
@Test
public void testSetTypeSpaces() throws Exception {
QuotaCounts qc1 = new QuotaCounts.Builder().build();
QuotaCounts qc2 = new QuotaCounts.Builder().nameSpace(1).storageSpace(512)
.typeSpaces(5).build();
qc1.setTypeSpaces(qc2.getTypeSpaces());
for (StorageType t : StorageType.values()) {
assertEquals(qc2.getTypeSpace(t), qc1.getTypeSpace(t));
}
// test ConstEnumCounters
qc1.setTypeSpaces(QuotaCounts.STORAGE_TYPE_RESET);
assertSame(QuotaCounts.STORAGE_TYPE_RESET, qc1.tsCounts);
}
@Test
public void testSetSpaces() {
QuotaCounts qc = new QuotaCounts.Builder().build();
qc.setNameSpace(10);
qc.setStorageSpace(1024);
assertEquals(10, qc.getNameSpace());
assertEquals(1024, qc.getStorageSpace());
// test ConstEnumCounters
qc.setNameSpace(HdfsConstants.QUOTA_RESET);
qc.setStorageSpace(HdfsConstants.QUOTA_RESET);
assertSame(QuotaCounts.QUOTA_RESET, qc.nsSsCounts);
}
@Test
public void testNegation() throws Exception {
QuotaCounts qc = new QuotaCounts.Builder()
.nameSpace(HdfsConstants.QUOTA_RESET)
.storageSpace(HdfsConstants.QUOTA_RESET)
.typeSpaces(HdfsConstants.QUOTA_RESET).build();
qc = qc.negation();
assertEquals(1, qc.getNameSpace());
assertEquals(1, qc.getStorageSpace());
for (StorageType t : StorageType.values()) {
assertEquals(1, qc.getTypeSpace(t));
}
}
}