From a59553b5e6bc6fa59ed31306708d899342013191 Mon Sep 17 00:00:00 2001 From: Erik Krogen Date: Mon, 8 Jul 2019 14:46:25 -0700 Subject: [PATCH] HDFS-14547. Improve memory efficiency of quotas when storage type quotas are not set. Contributed by Jinglun. (cherry-picked from 4632708148ed7dec75810feddb22ad98e4670483) (cherry-picked from 491dc7cc9d6836f34d2647b12dcefa86cc97a927) --- .../hdfs/server/namenode/QuotaCounts.java | 153 +++++++++++++++--- .../hadoop/hdfs/util/ConstEnumCounters.java | 98 +++++++++++ .../apache/hadoop/hdfs/util/EnumCounters.java | 29 ++-- .../hdfs/server/namenode/TestQuotaCounts.java | 135 ++++++++++++++++ 4 files changed, 381 insertions(+), 34 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ConstEnumCounters.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaCounts.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java index c3d4ba9c5cd..bcb21929c8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java @@ -18,53 +18,119 @@ package org.apache.hadoop.hdfs.server.namenode; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.util.ConstEnumCounters; import org.apache.hadoop.hdfs.util.EnumCounters; +import org.apache.hadoop.hdfs.util.ConstEnumCounters.ConstEnumException; + +import java.util.function.Consumer; /** * Counters for namespace, storage space and storage type space quota and usage. */ public class QuotaCounts { + + /** + * We pre-define 4 most common used EnumCounters objects. When the nsSsCounts + * and tsCounts are set to the 4 most common used value, we just point them to + * the pre-defined const EnumCounters objects instead of constructing many + * objects with the same value. See HDFS-14547. + */ + final static EnumCounters QUOTA_RESET = + new ConstEnumCounters<>(Quota.class, HdfsConstants.QUOTA_RESET); + final static EnumCounters QUOTA_DEFAULT = + new ConstEnumCounters<>(Quota.class, 0); + final static EnumCounters STORAGE_TYPE_RESET = + new ConstEnumCounters<>(StorageType.class, HdfsConstants.QUOTA_RESET); + final static EnumCounters STORAGE_TYPE_DEFAULT = + new ConstEnumCounters<>(StorageType.class, 0); + + /** + * Modify counter with action. If the counter is ConstEnumCounters, copy all + * the values of it to a new EnumCounters object, and modify the new obj. + * + * @param counter the EnumCounters to be modified. + * @param action the modifying action on counter. + * @return the modified counter. + */ + static > EnumCounters modify(EnumCounters counter, + Consumer> action) { + try { + action.accept(counter); + } catch (ConstEnumException cee) { + // We don't call clone here because ConstEnumCounters.clone() will return + // an object of class ConstEnumCounters. We want EnumCounters. + counter = counter.deepCopyEnumCounter(); + action.accept(counter); + } + return counter; + } + // Name space and storage space counts (HDFS-7775 refactors the original disk // space count to storage space counts) - private EnumCounters nsSsCounts; + @VisibleForTesting + EnumCounters nsSsCounts; // Storage type space counts - private EnumCounters tsCounts; + @VisibleForTesting + EnumCounters tsCounts; public static class Builder { private EnumCounters nsSsCounts; private EnumCounters tsCounts; public Builder() { - this.nsSsCounts = new EnumCounters(Quota.class); - this.tsCounts = new EnumCounters(StorageType.class); + this.nsSsCounts = QUOTA_DEFAULT; + this.tsCounts = STORAGE_TYPE_DEFAULT; } public Builder nameSpace(long val) { - this.nsSsCounts.set(Quota.NAMESPACE, val); + nsSsCounts = + setQuotaCounter(nsSsCounts, Quota.NAMESPACE, Quota.STORAGESPACE, val); return this; } public Builder storageSpace(long val) { - this.nsSsCounts.set(Quota.STORAGESPACE, val); + nsSsCounts = + setQuotaCounter(nsSsCounts, Quota.STORAGESPACE, Quota.NAMESPACE, val); return this; } public Builder typeSpaces(EnumCounters val) { if (val != null) { - this.tsCounts.set(val); + if (val == STORAGE_TYPE_DEFAULT || val == STORAGE_TYPE_RESET) { + tsCounts = val; + } else { + tsCounts = modify(tsCounts, ec -> ec.set(val)); + } } return this; } public Builder typeSpaces(long val) { - this.tsCounts.reset(val); + if (val == HdfsConstants.QUOTA_RESET) { + tsCounts = STORAGE_TYPE_RESET; + } else if (val == 0) { + tsCounts = STORAGE_TYPE_DEFAULT; + } else { + tsCounts = modify(tsCounts, ec -> ec.reset(val)); + } return this; } public Builder quotaCount(QuotaCounts that) { - this.nsSsCounts.set(that.nsSsCounts); - this.tsCounts.set(that.tsCounts); + if (that.nsSsCounts == QUOTA_DEFAULT || that.nsSsCounts == QUOTA_RESET) { + nsSsCounts = that.nsSsCounts; + } else { + nsSsCounts = modify(nsSsCounts, ec -> ec.set(that.nsSsCounts)); + } + if (that.tsCounts == STORAGE_TYPE_DEFAULT + || that.tsCounts == STORAGE_TYPE_RESET) { + tsCounts = that.tsCounts; + } else { + tsCounts = modify(tsCounts, ec -> ec.set(that.tsCounts)); + } return this; } @@ -79,14 +145,14 @@ public class QuotaCounts { } public QuotaCounts add(QuotaCounts that) { - this.nsSsCounts.add(that.nsSsCounts); - this.tsCounts.add(that.tsCounts); + nsSsCounts = modify(nsSsCounts, ec -> ec.add(that.nsSsCounts)); + tsCounts = modify(tsCounts, ec -> ec.add(that.tsCounts)); return this; } public QuotaCounts subtract(QuotaCounts that) { - this.nsSsCounts.subtract(that.nsSsCounts); - this.tsCounts.subtract(that.tsCounts); + nsSsCounts = modify(nsSsCounts, ec -> ec.subtract(that.nsSsCounts)); + tsCounts = modify(tsCounts, ec -> ec.subtract(that.tsCounts)); return this; } @@ -97,8 +163,8 @@ public class QuotaCounts { */ public QuotaCounts negation() { QuotaCounts ret = new QuotaCounts.Builder().quotaCount(this).build(); - ret.nsSsCounts.negation(); - ret.tsCounts.negation(); + ret.nsSsCounts = modify(ret.nsSsCounts, ec -> ec.negation()); + ret.tsCounts = modify(ret.tsCounts, ec -> ec.negation()); return ret; } @@ -107,11 +173,13 @@ public class QuotaCounts { } public void setNameSpace(long nameSpaceCount) { - this.nsSsCounts.set(Quota.NAMESPACE, nameSpaceCount); + nsSsCounts = + setQuotaCounter(nsSsCounts, Quota.NAMESPACE, Quota.STORAGESPACE, + nameSpaceCount); } public void addNameSpace(long nsDelta) { - this.nsSsCounts.add(Quota.NAMESPACE, nsDelta); + nsSsCounts = modify(nsSsCounts, ec -> ec.add(Quota.NAMESPACE, nsDelta)); } public long getStorageSpace(){ @@ -119,11 +187,13 @@ public class QuotaCounts { } public void setStorageSpace(long spaceCount) { - this.nsSsCounts.set(Quota.STORAGESPACE, spaceCount); + nsSsCounts = + setQuotaCounter(nsSsCounts, Quota.STORAGESPACE, Quota.NAMESPACE, + spaceCount); } public void addStorageSpace(long dsDelta) { - this.nsSsCounts.add(Quota.STORAGESPACE, dsDelta); + nsSsCounts = modify(nsSsCounts, ec -> ec.add(Quota.STORAGESPACE, dsDelta)); } public EnumCounters getTypeSpaces() { @@ -134,8 +204,10 @@ public class QuotaCounts { } void setTypeSpaces(EnumCounters that) { - if (that != null) { - this.tsCounts.set(that); + if (that == STORAGE_TYPE_DEFAULT || that == STORAGE_TYPE_RESET) { + tsCounts = that; + } else if (that != null) { + tsCounts = modify(tsCounts, ec -> ec.set(that)); } } @@ -144,21 +216,54 @@ public class QuotaCounts { } void setTypeSpace(StorageType type, long spaceCount) { - this.tsCounts.set(type, spaceCount); + tsCounts = modify(tsCounts, ec -> ec.set(type, spaceCount)); } public void addTypeSpace(StorageType type, long delta) { - this.tsCounts.add(type, delta); + tsCounts = modify(tsCounts, ec -> ec.add(type, delta)); } public boolean anyNsSsCountGreaterOrEqual(long val) { + if (nsSsCounts == QUOTA_DEFAULT) { + return val <= 0; + } else if (nsSsCounts == QUOTA_RESET) { + return val <= HdfsConstants.QUOTA_RESET; + } return nsSsCounts.anyGreaterOrEqual(val); } public boolean anyTypeSpaceCountGreaterOrEqual(long val) { + if (tsCounts == STORAGE_TYPE_DEFAULT) { + return val <= 0; + } else if (tsCounts == STORAGE_TYPE_RESET) { + return val <= HdfsConstants.QUOTA_RESET; + } return tsCounts.anyGreaterOrEqual(val); } + /** + * Set inputCounts' value of Quota type quotaToSet to val. + * inputCounts should be the left side value of this method. + * + * @param inputCounts the EnumCounters instance. + * @param quotaToSet the quota type to be set. + * @param otherQuota the other quota type besides quotaToSet. + * @param val the value to be set. + * @return the modified inputCounts. + */ + private static EnumCounters setQuotaCounter( + EnumCounters inputCounts, Quota quotaToSet, Quota otherQuota, + long val) { + if (val == HdfsConstants.QUOTA_RESET + && inputCounts.get(otherQuota) == HdfsConstants.QUOTA_RESET) { + return QUOTA_RESET; + } else if (val == 0 && inputCounts.get(otherQuota) == 0) { + return QUOTA_DEFAULT; + } else { + return modify(inputCounts, ec -> ec.set(quotaToSet, val)); + } + } + @Override public String toString() { return "name space=" + getNameSpace() + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ConstEnumCounters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ConstEnumCounters.java new file mode 100644 index 00000000000..4201d113096 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ConstEnumCounters.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.util; + +/** + * Const Counters for an enum type. + * + * It's the const version of EnumCounters. Any modification ends with a + * ConstEnumException. + * + * @see org.apache.hadoop.hdfs.util.EnumCounters + */ +public class ConstEnumCounters> extends EnumCounters { + + /** + * An exception class for modification on ConstEnumCounters. + */ + public static final class ConstEnumException extends RuntimeException { + private ConstEnumException(String msg) { + super(msg); + } + } + + /** + * Throwing this exception if any modification occurs. + */ + private static final ConstEnumException CONST_ENUM_EXCEPTION = + new ConstEnumException("modification on const."); + + public ConstEnumCounters(Class enumClass, long defaultVal) { + super(enumClass); + forceReset(defaultVal); + } + + @Override + public final void negation() { + throw CONST_ENUM_EXCEPTION; + } + + @Override + public final void set(final E e, final long value) { + throw CONST_ENUM_EXCEPTION; + } + + @Override + public final void set(final EnumCounters that) { + throw CONST_ENUM_EXCEPTION; + } + + @Override + public final void reset() { + throw CONST_ENUM_EXCEPTION; + } + + @Override + public final void add(final E e, final long value) { + throw CONST_ENUM_EXCEPTION; + } + + @Override + public final void add(final EnumCounters that) { + throw CONST_ENUM_EXCEPTION; + } + + @Override + public final void subtract(final E e, final long value) { + throw CONST_ENUM_EXCEPTION; + } + + @Override + public final void subtract(final EnumCounters that) { + throw CONST_ENUM_EXCEPTION; + } + + @Override + public final void reset(long val) { + throw CONST_ENUM_EXCEPTION; + } + + private void forceReset(long val) { + super.reset(val); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java index bec44a99e94..74453337650 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java @@ -70,55 +70,55 @@ public class EnumCounters> { } /** Negate all counters. */ - public final void negation() { + public void negation() { for(int i = 0; i < counters.length; i++) { counters[i] = -counters[i]; } } /** Set counter e to the given value. */ - public final void set(final E e, final long value) { + public void set(final E e, final long value) { counters[e.ordinal()] = value; } /** Set this counters to that counters. */ - public final void set(final EnumCounters that) { + public void set(final EnumCounters that) { for(int i = 0; i < counters.length; i++) { this.counters[i] = that.counters[i]; } } /** Reset all counters to zero. */ - public final void reset() { + public void reset() { reset(0L); } /** Add the given value to counter e. */ - public final void add(final E e, final long value) { + public void add(final E e, final long value) { counters[e.ordinal()] += value; } /** Add that counters to this counters. */ - public final void add(final EnumCounters that) { + public void add(final EnumCounters that) { for(int i = 0; i < counters.length; i++) { this.counters[i] += that.counters[i]; } } /** Subtract the given value from counter e. */ - public final void subtract(final E e, final long value) { + public void subtract(final E e, final long value) { counters[e.ordinal()] -= value; } /** Subtract this counters from that counters. */ - public final void subtract(final EnumCounters that) { + public void subtract(final EnumCounters that) { for(int i = 0; i < counters.length; i++) { this.counters[i] -= that.counters[i]; } } /** @return the sum of all counters. */ - public final long sum() { + public long sum() { long sum = 0; for(int i = 0; i < counters.length; i++) { sum += counters[i]; @@ -138,6 +138,15 @@ public class EnumCounters> { && Arrays.equals(this.counters, that.counters); } + /** + * Return a deep copy of EnumCounter. + */ + public EnumCounters deepCopyEnumCounter() { + EnumCounters newCounter = new EnumCounters<>(enumClass); + newCounter.set(this); + return newCounter; + } + @Override public int hashCode() { return Arrays.hashCode(counters); @@ -154,7 +163,7 @@ public class EnumCounters> { return b.substring(0, b.length() - 2); } - public final void reset(long val) { + public void reset(long val) { for(int i = 0; i < counters.length; i++) { this.counters[i] = val; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaCounts.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaCounts.java new file mode 100644 index 00000000000..e731f685699 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaCounts.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * Test QuotaCounts. + */ +public class TestQuotaCounts { + @Test + public void testBuildConstEnumCounters() throws Exception { + QuotaCounts qc = + new QuotaCounts.Builder().nameSpace(HdfsConstants.QUOTA_RESET) + .storageSpace(HdfsConstants.QUOTA_RESET).build(); + // compare the references + assertSame(QuotaCounts.QUOTA_RESET, qc.nsSsCounts); + assertSame(QuotaCounts.STORAGE_TYPE_DEFAULT, qc.tsCounts); + // compare the values + assertEquals(HdfsConstants.QUOTA_RESET, qc.getNameSpace()); + assertEquals(HdfsConstants.QUOTA_RESET, qc.getStorageSpace()); + for (StorageType st : StorageType.values()) { + assertEquals(0, qc.getTypeSpace(st)); + } + } + + @Test + public void testAddSpace() throws Exception { + QuotaCounts qc = new QuotaCounts.Builder().build(); + qc.addNameSpace(1); + qc.addStorageSpace(1024); + assertEquals(1, qc.getNameSpace()); + assertEquals(1024, qc.getStorageSpace()); + } + + @Test + public void testAdd() throws Exception { + QuotaCounts qc1 = new QuotaCounts.Builder().build(); + QuotaCounts qc2 = new QuotaCounts.Builder().nameSpace(1).storageSpace(512) + .typeSpaces(5).build(); + qc1.add(qc2); + assertEquals(1, qc1.getNameSpace()); + assertEquals(512, qc1.getStorageSpace()); + for (StorageType type : StorageType.values()) { + assertEquals(5, qc1.getTypeSpace(type)); + } + } + + @Test + public void testAddTypeSpaces() throws Exception { + QuotaCounts qc = new QuotaCounts.Builder().build(); + for (StorageType t : StorageType.values()) { + qc.addTypeSpace(t, 10); + } + for (StorageType type : StorageType.values()) { + assertEquals(10, qc.getTypeSpace(type)); + } + } + + @Test + public void testSubtract() throws Exception { + QuotaCounts qc1 = new QuotaCounts.Builder().build(); + QuotaCounts qc2 = new QuotaCounts.Builder().nameSpace(1).storageSpace(512) + .typeSpaces(5).build(); + qc1.subtract(qc2); + assertEquals(-1, qc1.getNameSpace()); + assertEquals(-512, qc1.getStorageSpace()); + for (StorageType type : StorageType.values()) { + assertEquals(-5, qc1.getTypeSpace(type)); + } + } + + @Test + public void testSetTypeSpaces() throws Exception { + QuotaCounts qc1 = new QuotaCounts.Builder().build(); + QuotaCounts qc2 = new QuotaCounts.Builder().nameSpace(1).storageSpace(512) + .typeSpaces(5).build(); + qc1.setTypeSpaces(qc2.getTypeSpaces()); + for (StorageType t : StorageType.values()) { + assertEquals(qc2.getTypeSpace(t), qc1.getTypeSpace(t)); + } + + // test ConstEnumCounters + qc1.setTypeSpaces(QuotaCounts.STORAGE_TYPE_RESET); + assertSame(QuotaCounts.STORAGE_TYPE_RESET, qc1.tsCounts); + } + + @Test + public void testSetSpaces() { + QuotaCounts qc = new QuotaCounts.Builder().build(); + qc.setNameSpace(10); + qc.setStorageSpace(1024); + assertEquals(10, qc.getNameSpace()); + assertEquals(1024, qc.getStorageSpace()); + + // test ConstEnumCounters + qc.setNameSpace(HdfsConstants.QUOTA_RESET); + qc.setStorageSpace(HdfsConstants.QUOTA_RESET); + assertSame(QuotaCounts.QUOTA_RESET, qc.nsSsCounts); + } + + @Test + public void testNegation() throws Exception { + QuotaCounts qc = new QuotaCounts.Builder() + .nameSpace(HdfsConstants.QUOTA_RESET) + .storageSpace(HdfsConstants.QUOTA_RESET) + .typeSpaces(HdfsConstants.QUOTA_RESET).build(); + qc = qc.negation(); + assertEquals(1, qc.getNameSpace()); + assertEquals(1, qc.getStorageSpace()); + for (StorageType t : StorageType.values()) { + assertEquals(1, qc.getTypeSpace(t)); + } + } +}