HDFS-7228. Add an SSD policy into the default BlockStoragePolicySuite. Contributed by Jing Zhao.
Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
This commit is contained in:
Jing Zhao
parent
262fef119e
commit
6c3ac6a485
|
|
@ -321,6 +321,9 @@ Release 2.6.0 - UNRELEASED
|
|||
HDFS-7195. Update user doc of secure mode about Datanodes don't require root
|
||||
or jsvc. (cnauroth)
|
||||
|
||||
HDFS-7228. Add an SSD policy into the default BlockStoragePolicySuite.
|
||||
(jing9)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HDFS-6690. Deduplicate xattr names in memory. (wang)
|
||||
|
|
|
|||
|
|
@ -165,4 +165,16 @@ public class HdfsConstants {
|
|||
|
||||
public static final String SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR
|
||||
= Path.SEPARATOR + DOT_SNAPSHOT_DIR + Path.SEPARATOR;
|
||||
|
||||
public static final String ALLSSD_STORAGE_POLICY_NAME = "ALL_SSD";
|
||||
public static final String ONESSD_STORAGE_POLICY_NAME = "ONE_SSD";
|
||||
public static final String HOT_STORAGE_POLICY_NAME = "HOT";
|
||||
public static final String WARM_STORAGE_POLICY_NAME = "WARM";
|
||||
public static final String COLD_STORAGE_POLICY_NAME = "COLD";
|
||||
|
||||
public static final byte ALLSSD_STORAGE_POLICY_ID = 12;
|
||||
public static final byte ONESSD_STORAGE_POLICY_ID = 10;
|
||||
public static final byte HOT_STORAGE_POLICY_ID = 7;
|
||||
public static final byte WARM_STORAGE_POLICY_ID = 5;
|
||||
public static final byte COLD_STORAGE_POLICY_ID = 2;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import org.apache.hadoop.fs.XAttr;
|
|||
import org.apache.hadoop.hdfs.StorageType;
|
||||
import org.apache.hadoop.hdfs.XAttrHelper;
|
||||
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
|
@ -44,17 +45,32 @@ public class BlockStoragePolicySuite {
|
|||
public static BlockStoragePolicySuite createDefaultSuite() {
|
||||
final BlockStoragePolicy[] policies =
|
||||
new BlockStoragePolicy[1 << ID_BIT_LENGTH];
|
||||
final byte hotId = 12;
|
||||
policies[hotId] = new BlockStoragePolicy(hotId, "HOT",
|
||||
final byte allssdId = HdfsConstants.ALLSSD_STORAGE_POLICY_ID;
|
||||
policies[allssdId] = new BlockStoragePolicy(allssdId,
|
||||
HdfsConstants.ALLSSD_STORAGE_POLICY_NAME,
|
||||
new StorageType[]{StorageType.SSD},
|
||||
new StorageType[]{StorageType.DISK},
|
||||
new StorageType[]{StorageType.DISK});
|
||||
final byte onessdId = HdfsConstants.ONESSD_STORAGE_POLICY_ID;
|
||||
policies[onessdId] = new BlockStoragePolicy(onessdId,
|
||||
HdfsConstants.ONESSD_STORAGE_POLICY_NAME,
|
||||
new StorageType[]{StorageType.SSD, StorageType.DISK},
|
||||
new StorageType[]{StorageType.SSD, StorageType.DISK},
|
||||
new StorageType[]{StorageType.SSD, StorageType.DISK});
|
||||
final byte hotId = HdfsConstants.HOT_STORAGE_POLICY_ID;
|
||||
policies[hotId] = new BlockStoragePolicy(hotId,
|
||||
HdfsConstants.HOT_STORAGE_POLICY_NAME,
|
||||
new StorageType[]{StorageType.DISK}, StorageType.EMPTY_ARRAY,
|
||||
new StorageType[]{StorageType.ARCHIVE});
|
||||
final byte warmId = 8;
|
||||
policies[warmId] = new BlockStoragePolicy(warmId, "WARM",
|
||||
final byte warmId = HdfsConstants.WARM_STORAGE_POLICY_ID;
|
||||
policies[warmId] = new BlockStoragePolicy(warmId,
|
||||
HdfsConstants.WARM_STORAGE_POLICY_NAME,
|
||||
new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
|
||||
new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
|
||||
new StorageType[]{StorageType.DISK, StorageType.ARCHIVE});
|
||||
final byte coldId = 4;
|
||||
policies[coldId] = new BlockStoragePolicy(coldId, "COLD",
|
||||
final byte coldId = HdfsConstants.COLD_STORAGE_POLICY_ID;
|
||||
policies[coldId] = new BlockStoragePolicy(coldId,
|
||||
HdfsConstants.COLD_STORAGE_POLICY_NAME,
|
||||
new StorageType[]{StorageType.ARCHIVE}, StorageType.EMPTY_ARRAY,
|
||||
StorageType.EMPTY_ARRAY);
|
||||
return new BlockStoragePolicySuite(hotId, policies);
|
||||
|
|
|
|||
|
|
@ -1136,7 +1136,8 @@ public class DFSTestUtil {
|
|||
// OP_CLOSE 9
|
||||
s.close();
|
||||
// OP_SET_STORAGE_POLICY 45
|
||||
filesystem.setStoragePolicy(pathFileCreate, "HOT");
|
||||
filesystem.setStoragePolicy(pathFileCreate,
|
||||
HdfsConstants.HOT_STORAGE_POLICY_NAME);
|
||||
// OP_RENAME_OLD 1
|
||||
final Path pathFileMoved = new Path("/file_moved");
|
||||
filesystem.rename(pathFileCreate, pathFileMoved);
|
||||
|
|
|
|||
|
|
@ -66,10 +66,11 @@ public class TestBlockStoragePolicy {
|
|||
static final long FILE_LEN = 1024;
|
||||
static final short REPLICATION = 3;
|
||||
|
||||
static final byte COLD = (byte) 4;
|
||||
static final byte WARM = (byte) 8;
|
||||
static final byte HOT = (byte) 12;
|
||||
|
||||
static final byte COLD = HdfsConstants.COLD_STORAGE_POLICY_ID;
|
||||
static final byte WARM = HdfsConstants.WARM_STORAGE_POLICY_ID;
|
||||
static final byte HOT = HdfsConstants.HOT_STORAGE_POLICY_ID;
|
||||
static final byte ONESSD = HdfsConstants.ONESSD_STORAGE_POLICY_ID;
|
||||
static final byte ALLSSD = HdfsConstants.ALLSSD_STORAGE_POLICY_ID;
|
||||
|
||||
@Test (timeout=300000)
|
||||
public void testConfigKeyEnabled() throws IOException {
|
||||
|
|
@ -79,7 +80,8 @@ public class TestBlockStoragePolicy {
|
|||
.numDataNodes(1).build();
|
||||
try {
|
||||
cluster.waitActive();
|
||||
cluster.getFileSystem().setStoragePolicy(new Path("/"), "COLD");
|
||||
cluster.getFileSystem().setStoragePolicy(new Path("/"),
|
||||
HdfsConstants.COLD_STORAGE_POLICY_NAME);
|
||||
} finally {
|
||||
cluster.shutdown();
|
||||
}
|
||||
|
|
@ -98,7 +100,8 @@ public class TestBlockStoragePolicy {
|
|||
.numDataNodes(1).build();
|
||||
try {
|
||||
cluster.waitActive();
|
||||
cluster.getFileSystem().setStoragePolicy(new Path("/"), "COLD");
|
||||
cluster.getFileSystem().setStoragePolicy(new Path("/"),
|
||||
HdfsConstants.COLD_STORAGE_POLICY_NAME);
|
||||
} finally {
|
||||
cluster.shutdown();
|
||||
}
|
||||
|
|
@ -108,14 +111,21 @@ public class TestBlockStoragePolicy {
|
|||
public void testDefaultPolicies() {
|
||||
final Map<Byte, String> expectedPolicyStrings = new HashMap<Byte, String>();
|
||||
expectedPolicyStrings.put(COLD,
|
||||
"BlockStoragePolicy{COLD:4, storageTypes=[ARCHIVE], " +
|
||||
"BlockStoragePolicy{COLD:" + COLD + ", storageTypes=[ARCHIVE], " +
|
||||
"creationFallbacks=[], replicationFallbacks=[]}");
|
||||
expectedPolicyStrings.put(WARM,
|
||||
"BlockStoragePolicy{WARM:8, storageTypes=[DISK, ARCHIVE], " +
|
||||
"creationFallbacks=[DISK, ARCHIVE], replicationFallbacks=[DISK, ARCHIVE]}");
|
||||
"BlockStoragePolicy{WARM:" + WARM + ", storageTypes=[DISK, ARCHIVE], " +
|
||||
"creationFallbacks=[DISK, ARCHIVE], " +
|
||||
"replicationFallbacks=[DISK, ARCHIVE]}");
|
||||
expectedPolicyStrings.put(HOT,
|
||||
"BlockStoragePolicy{HOT:12, storageTypes=[DISK], " +
|
||||
"BlockStoragePolicy{HOT:" + HOT + ", storageTypes=[DISK], " +
|
||||
"creationFallbacks=[], replicationFallbacks=[ARCHIVE]}");
|
||||
expectedPolicyStrings.put(ONESSD, "BlockStoragePolicy{ONE_SSD:" + ONESSD +
|
||||
", storageTypes=[SSD, DISK], creationFallbacks=[SSD, DISK], " +
|
||||
"replicationFallbacks=[SSD, DISK]}");
|
||||
expectedPolicyStrings.put(ALLSSD, "BlockStoragePolicy{ALL_SSD:" + ALLSSD +
|
||||
", storageTypes=[SSD], creationFallbacks=[DISK], " +
|
||||
"replicationFallbacks=[DISK]}");
|
||||
|
||||
for(byte i = 1; i < 16; i++) {
|
||||
final BlockStoragePolicy policy = POLICY_SUITE.getPolicy(i);
|
||||
|
|
@ -842,15 +852,15 @@ public class TestBlockStoragePolicy {
|
|||
|
||||
final Path invalidPath = new Path("/invalidPath");
|
||||
try {
|
||||
fs.setStoragePolicy(invalidPath, "WARM");
|
||||
fs.setStoragePolicy(invalidPath, HdfsConstants.WARM_STORAGE_POLICY_NAME);
|
||||
Assert.fail("Should throw a FileNotFoundException");
|
||||
} catch (FileNotFoundException e) {
|
||||
GenericTestUtils.assertExceptionContains(invalidPath.toString(), e);
|
||||
}
|
||||
|
||||
fs.setStoragePolicy(fooFile, "COLD");
|
||||
fs.setStoragePolicy(barDir, "WARM");
|
||||
fs.setStoragePolicy(barFile2, "HOT");
|
||||
fs.setStoragePolicy(fooFile, HdfsConstants.COLD_STORAGE_POLICY_NAME);
|
||||
fs.setStoragePolicy(barDir, HdfsConstants.WARM_STORAGE_POLICY_NAME);
|
||||
fs.setStoragePolicy(barFile2, HdfsConstants.HOT_STORAGE_POLICY_NAME);
|
||||
|
||||
dirList = fs.getClient().listPaths(dir.toString(),
|
||||
HdfsFileStatus.EMPTY_NAME).getPartialListing();
|
||||
|
|
@ -898,7 +908,7 @@ public class TestBlockStoragePolicy {
|
|||
DFSTestUtil.createFile(fs, fooFile1, FILE_LEN, REPLICATION, 0L);
|
||||
DFSTestUtil.createFile(fs, fooFile2, FILE_LEN, REPLICATION, 0L);
|
||||
|
||||
fs.setStoragePolicy(fooDir, "WARM");
|
||||
fs.setStoragePolicy(fooDir, HdfsConstants.WARM_STORAGE_POLICY_NAME);
|
||||
|
||||
HdfsFileStatus[] dirList = fs.getClient().listPaths(dir.toString(),
|
||||
HdfsFileStatus.EMPTY_NAME, true).getPartialListing();
|
||||
|
|
@ -910,7 +920,7 @@ public class TestBlockStoragePolicy {
|
|||
// take snapshot
|
||||
SnapshotTestHelper.createSnapshot(fs, dir, "s1");
|
||||
// change the storage policy of fooFile1
|
||||
fs.setStoragePolicy(fooFile1, "COLD");
|
||||
fs.setStoragePolicy(fooFile1, HdfsConstants.COLD_STORAGE_POLICY_NAME);
|
||||
|
||||
fooList = fs.getClient().listPaths(fooDir.toString(),
|
||||
HdfsFileStatus.EMPTY_NAME).getPartialListing();
|
||||
|
|
@ -933,7 +943,7 @@ public class TestBlockStoragePolicy {
|
|||
HdfsFileStatus.EMPTY_NAME).getPartialListing(), COLD);
|
||||
|
||||
// change the storage policy of foo dir
|
||||
fs.setStoragePolicy(fooDir, "HOT");
|
||||
fs.setStoragePolicy(fooDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);
|
||||
// /dir/foo is now hot
|
||||
dirList = fs.getClient().listPaths(dir.toString(),
|
||||
HdfsFileStatus.EMPTY_NAME, true).getPartialListing();
|
||||
|
|
@ -1050,7 +1060,7 @@ public class TestBlockStoragePolicy {
|
|||
*/
|
||||
@Test
|
||||
public void testChangeHotFileRep() throws Exception {
|
||||
testChangeFileRep("HOT", HOT,
|
||||
testChangeFileRep(HdfsConstants.HOT_STORAGE_POLICY_NAME, HOT,
|
||||
new StorageType[]{StorageType.DISK, StorageType.DISK,
|
||||
StorageType.DISK},
|
||||
new StorageType[]{StorageType.DISK, StorageType.DISK, StorageType.DISK,
|
||||
|
|
@ -1064,7 +1074,7 @@ public class TestBlockStoragePolicy {
|
|||
*/
|
||||
@Test
|
||||
public void testChangeWarmRep() throws Exception {
|
||||
testChangeFileRep("WARM", WARM,
|
||||
testChangeFileRep(HdfsConstants.WARM_STORAGE_POLICY_NAME, WARM,
|
||||
new StorageType[]{StorageType.DISK, StorageType.ARCHIVE,
|
||||
StorageType.ARCHIVE},
|
||||
new StorageType[]{StorageType.DISK, StorageType.ARCHIVE,
|
||||
|
|
@ -1077,7 +1087,7 @@ public class TestBlockStoragePolicy {
|
|||
*/
|
||||
@Test
|
||||
public void testChangeColdRep() throws Exception {
|
||||
testChangeFileRep("COLD", COLD,
|
||||
testChangeFileRep(HdfsConstants.COLD_STORAGE_POLICY_NAME, COLD,
|
||||
new StorageType[]{StorageType.ARCHIVE, StorageType.ARCHIVE,
|
||||
StorageType.ARCHIVE},
|
||||
new StorageType[]{StorageType.ARCHIVE, StorageType.ARCHIVE,
|
||||
|
|
@ -1141,7 +1151,7 @@ public class TestBlockStoragePolicy {
|
|||
final DistributedFileSystem fs = cluster.getFileSystem();
|
||||
try {
|
||||
BlockStoragePolicy[] policies = fs.getStoragePolicies();
|
||||
Assert.assertEquals(3, policies.length);
|
||||
Assert.assertEquals(5, policies.length);
|
||||
Assert.assertEquals(POLICY_SUITE.getPolicy(COLD).toString(),
|
||||
policies[0].toString());
|
||||
Assert.assertEquals(POLICY_SUITE.getPolicy(WARM).toString(),
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
|
|||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.hdfs.StorageType;
|
||||
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
|
||||
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
||||
|
|
@ -97,9 +98,9 @@ public class TestStorageMover {
|
|||
DEFAULT_CONF.setLong(DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_KEY, 2000L);
|
||||
|
||||
DEFAULT_POLICIES = BlockStoragePolicySuite.createDefaultSuite();
|
||||
HOT = DEFAULT_POLICIES.getPolicy("HOT");
|
||||
WARM = DEFAULT_POLICIES.getPolicy("WARM");
|
||||
COLD = DEFAULT_POLICIES.getPolicy("COLD");
|
||||
HOT = DEFAULT_POLICIES.getPolicy(HdfsConstants.HOT_STORAGE_POLICY_NAME);
|
||||
WARM = DEFAULT_POLICIES.getPolicy(HdfsConstants.WARM_STORAGE_POLICY_NAME);
|
||||
COLD = DEFAULT_POLICIES.getPolicy(HdfsConstants.COLD_STORAGE_POLICY_NAME);
|
||||
TestBalancer.initTestSetup();
|
||||
Dispatcher.setDelayAfterErrors(1000L);
|
||||
}
|
||||
|
|
@ -198,14 +199,6 @@ public class TestStorageMover {
|
|||
this.policies = DEFAULT_POLICIES;
|
||||
}
|
||||
|
||||
MigrationTest(ClusterScheme cScheme, NamespaceScheme nsScheme,
|
||||
BlockStoragePolicySuite policies) {
|
||||
this.clusterScheme = cScheme;
|
||||
this.nsScheme = nsScheme;
|
||||
this.conf = clusterScheme.conf;
|
||||
this.policies = policies;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up the cluster and start NameNode and DataNodes according to the
|
||||
* corresponding scheme.
|
||||
|
|
@ -270,9 +263,6 @@ public class TestStorageMover {
|
|||
}
|
||||
if (verifyAll) {
|
||||
verifyNamespace();
|
||||
} else {
|
||||
// TODO verify according to the given path list
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -425,22 +415,6 @@ public class TestStorageMover {
|
|||
}
|
||||
return types;
|
||||
}
|
||||
|
||||
private static long[][] genCapacities(int nDatanodes, int numAllDisk,
|
||||
int numAllArchive, long diskCapacity, long archiveCapacity) {
|
||||
final long[][] capacities = new long[nDatanodes][];
|
||||
int i = 0;
|
||||
for (; i < numAllDisk; i++) {
|
||||
capacities[i] = new long[]{diskCapacity, diskCapacity};
|
||||
}
|
||||
for (; i < numAllDisk + numAllArchive; i++) {
|
||||
capacities[i] = new long[]{archiveCapacity, archiveCapacity};
|
||||
}
|
||||
for(; i < capacities.length; i++) {
|
||||
capacities[i] = new long[]{diskCapacity, archiveCapacity};
|
||||
}
|
||||
return capacities;
|
||||
}
|
||||
|
||||
private static class PathPolicyMap {
|
||||
final Map<Path, BlockStoragePolicy> map = Maps.newHashMap();
|
||||
|
|
@ -647,8 +621,8 @@ public class TestStorageMover {
|
|||
|
||||
private void setVolumeFull(DataNode dn, StorageType type) {
|
||||
List<? extends FsVolumeSpi> volumes = dn.getFSDataset().getVolumes();
|
||||
for (int j = 0; j < volumes.size(); ++j) {
|
||||
FsVolumeImpl volume = (FsVolumeImpl) volumes.get(j);
|
||||
for (FsVolumeSpi v : volumes) {
|
||||
FsVolumeImpl volume = (FsVolumeImpl) v;
|
||||
if (volume.getStorageType() == type) {
|
||||
LOG.info("setCapacity to 0 for [" + volume.getStorageType() + "]"
|
||||
+ volume.getStorageID());
|
||||
|
|
|
|||
Loading…
Reference in New Issue