Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-8879. Quota by storage type usage incorrectly initialized upon namenode restart. Contributed by Xiaoyu Yao. 

(cherry picked from commit 3e715a4f4c)
This commit is contained in:
Kihwal Lee 2015-08-28 13:27:17 -05:00
parent ba0f66984c
commit 42aa880df3
3 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -1,5 +1,8 @@
Hadoop HDFS Change Log Hadoop HDFS Change Log
HDFS-8879. Quota by storage type usage incorrectly initialized upon namenode
restart. (xyao)
Release 2.7.2 - UNRELEASED Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
View File

@ -909,11 +909,9 @@ public class FSImage implements Closeable {
+ " quota = " + ssQuota + " < consumed = " + ssConsumed); + " quota = " + ssQuota + " < consumed = " + ssConsumed);
} }
final EnumCounters<StorageType> typeSpaces = final EnumCounters<StorageType> typeSpaces = counts.getTypeSpaces();
new EnumCounters<StorageType>(StorageType.class);
for (StorageType t : StorageType.getTypesSupportingQuota()) { for (StorageType t : StorageType.getTypesSupportingQuota()) {
final long typeSpace = counts.getTypeSpaces().get(t) - final long typeSpace = typeSpaces.get(t) - parentTypeSpaces.get(t);
parentTypeSpaces.get(t);
final long typeQuota = q.getTypeSpaces().get(t); final long typeQuota = q.getTypeSpaces().get(t);
if (Quota.isViolated(typeQuota, typeSpace)) { if (Quota.isViolated(typeQuota, typeSpace)) {
LOG.warn("Storage type quota violation in image for " LOG.warn("Storage type quota violation in image for "

14
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java
View File

@ -68,10 +68,7 @@ public class TestQuotaByStorageType {
.storageTypes(new StorageType[]{StorageType.SSD, StorageType.DEFAULT}) .storageTypes(new StorageType[]{StorageType.SSD, StorageType.DEFAULT})
.build(); .build();
cluster.waitActive(); cluster.waitActive();
refreshClusterState();
fsdir = cluster.getNamesystem().getFSDirectory();
dfs = cluster.getFileSystem();
fsn = cluster.getNamesystem();
} }
@After @After
@ -81,6 +78,13 @@ public class TestQuotaByStorageType {
} }
} }
// Cluster state must be refreshed after each start/restart in the test
private void refreshClusterState() throws IOException{
fsdir = cluster.getNamesystem().getFSDirectory();
dfs = cluster.getFileSystem();
fsn = cluster.getNamesystem();
}
@Test(timeout = 60000) @Test(timeout = 60000)
public void testQuotaByStorageTypeWithFileCreateOneSSD() throws Exception { public void testQuotaByStorageTypeWithFileCreateOneSSD() throws Exception {
testQuotaByStorageTypeWithFileCreateCase( testQuotaByStorageTypeWithFileCreateCase(
@ -663,6 +667,7 @@ public class TestQuotaByStorageType {
// Restart namenode to make sure the editlog is correct // Restart namenode to make sure the editlog is correct
cluster.restartNameNode(true); cluster.restartNameNode(true);
refreshClusterState();
INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString()); INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
// Verify quota is still set // Verify quota is still set
@ -715,6 +720,7 @@ public class TestQuotaByStorageType {
dfs.saveNamespace(); dfs.saveNamespace();
dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
cluster.restartNameNode(true); cluster.restartNameNode(true);
refreshClusterState();
INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString()); INode testDirNodeAfterNNRestart = fsdir.getINode4Write(testDir.toString());
assertTrue(testDirNode.isDirectory()); assertTrue(testDirNode.isDirectory());