Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-8879. Quota by storage type usage incorrectly initialized upon namenode restart. Contributed by Xiaoyu Yao. 

(cherry picked from commit 3e715a4f4c)
This commit is contained in:
Xiaoyu Yao 2015-08-11 21:42:53 -07:00
parent 43f386e748
commit 458b684941
3 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -786,6 +786,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8866. Typo in docs: Rumtime -> Runtime. (Gabor Liptak via jghoman) HDFS-8866. Typo in docs: Rumtime -> Runtime. (Gabor Liptak via jghoman)
HDFS-8879. Quota by storage type usage incorrectly initialized upon namenode
restart. (xyao)
Release 2.7.2 - UNRELEASED Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
View File

@ -911,11 +911,9 @@ public class FSImage implements Closeable {
+ " quota = " + ssQuota + " < consumed = " + ssConsumed); + " quota = " + ssQuota + " < consumed = " + ssConsumed);
} }
final EnumCounters<StorageType> typeSpaces = final EnumCounters<StorageType> typeSpaces = counts.getTypeSpaces();
new EnumCounters<StorageType>(StorageType.class);
for (StorageType t : StorageType.getTypesSupportingQuota()) { for (StorageType t : StorageType.getTypesSupportingQuota()) {
final long typeSpace = counts.getTypeSpaces().get(t) - final long typeSpace = typeSpaces.get(t) - parentTypeSpaces.get(t);
parentTypeSpaces.get(t);
final long typeQuota = q.getTypeSpaces().get(t); final long typeQuota = q.getTypeSpaces().get(t);
if (Quota.isViolated(typeQuota, typeSpace)) { if (Quota.isViolated(typeQuota, typeSpace)) {
LOG.warn("Storage type quota violation in image for " LOG.warn("Storage type quota violation in image for "

14
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java
View File

@ -67,10 +67,7 @@ public class TestQuotaByStorageType {
.storageTypes(new StorageType[]{StorageType.SSD, StorageType.DEFAULT}) .storageTypes(new StorageType[]{StorageType.SSD, StorageType.DEFAULT})
.build(); .build();
cluster.waitActive(); cluster.waitActive();
refreshClusterState();
fsdir = cluster.getNamesystem().getFSDirectory();
dfs = cluster.getFileSystem();
fsn = cluster.getNamesystem();
} }
@After @After
@ -80,6 +77,13 @@ public class TestQuotaByStorageType {
} }
} }
// Cluster state must be refreshed after each start/restart in the test
private void refreshClusterState() throws IOException{
fsdir = cluster.getNamesystem().getFSDirectory();
dfs = cluster.getFileSystem();
fsn = cluster.getNamesystem();
}
@Test(timeout = 60000) @Test(timeout = 60000)
public void testQuotaByStorageTypeWithFileCreateOneSSD() throws Exception { public void testQuotaByStorageTypeWithFileCreateOneSSD() throws Exception {
testQuotaByStorageTypeWithFileCreateCase( testQuotaByStorageTypeWithFileCreateCase(
@ -662,6 +666,7 @@ public class TestQuotaByStorageType {
// Restart namenode to make sure the editlog is correct // Restart namenode to make sure the editlog is correct
cluster.restartNameNode(true); cluster.restartNameNode(true);
refreshClusterState();
INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString()); INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
// Verify quota is still set // Verify quota is still set
@ -714,6 +719,7 @@ public class TestQuotaByStorageType {
dfs.saveNamespace(); dfs.saveNamespace();
dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
cluster.restartNameNode(true); cluster.restartNameNode(true);
refreshClusterState();
INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString()); INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
assertTrue(testDirNode.isDirectory()); assertTrue(testDirNode.isDirectory());