Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9730. Storage ID update does not happen when there is a layout change. Contributed by Tsz Wo Nicholas Sze.

Browse Source 
(cherry picked from commit 496f33de0ce80dc455cfd51f19612da6f9b914f9)

Conflicts:
	hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
This commit is contained in:
Kihwal Lee 2016-02-04 13:32:11 -06:00
parent bef53683b6
commit c10bf788eb
2 changed files with 23 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -86,6 +86,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9740. Use a reasonable limit in DFSTestUtil.waitForMetric()
(Chang Li via vinayakumarb)
HDFS-9730. Storage ID update does not happen when there is a layout change
(Tsz Wo Nicholas Sze via kihwal)
Release 2.7.2 - 2016-01-25
INCOMPATIBLE CHANGES

47
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
View File

@ -533,11 +533,7 @@ void format(StorageDirectory sd, NamespaceInfo nsInfo,
this.cTime = 0;
setDatanodeUuid(datanodeUuid);
if (sd.getStorageUuid() == null) {
// Assign a new Storage UUID.
sd.setStorageUuid(DatanodeStorage.generateUuid());
}
createStorageID(sd, false);
writeProperties(sd);
}
@ -709,7 +705,13 @@ private boolean doTransition(StorageDirectory sd, NamespaceInfo nsInfo,
// do upgrade
if (this.layoutVersion > HdfsConstants.DATANODE_LAYOUT_VERSION) {
doUpgrade(sd, nsInfo, conf); // upgrade
if (federationSupported) {
// If the existing on-disk layout version supports federation,
// simply update the properties.
upgradeProperties(sd);
} else {
doUpgradePreFederation(sd, nsInfo, conf);
}
return true; // doUgrade already has written properties
}
@ -723,7 +725,8 @@ private boolean doTransition(StorageDirectory sd, NamespaceInfo nsInfo,
}
/**
* Upgrade -- Move current storage into a backup directory,
* Upgrade from a pre-federation layout.
* Move current storage into a backup directory,
* and hardlink all its blocks into the new current directory.
*
* Upgrade from pre-0.22 to 0.22 or later release e.g. 0.19/0.20/ => 0.22/0.23
@ -742,25 +745,9 @@ private boolean doTransition(StorageDirectory sd, NamespaceInfo nsInfo,
* There should be only ONE namenode in the cluster for first
* time upgrade to 0.22
* @param sd storage directory
* @throws IOException on error
*/
void doUpgrade(final StorageDirectory sd, final NamespaceInfo nsInfo,
final Configuration conf) throws IOException {
// If the existing on-disk layout version supportes federation, simply
// update its layout version.
if (DataNodeLayoutVersion.supports(
LayoutVersion.Feature.FEDERATION, layoutVersion)) {
// The VERSION file is already read in. Override the layoutVersion
// field and overwrite the file. The upgrade work is handled by
// {@link BlockPoolSliceStorage#doUpgrade}
LOG.info("Updating layout version from " + layoutVersion + " to "
+ HdfsConstants.DATANODE_LAYOUT_VERSION + " for storage "
+ sd.getRoot());
layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
writeProperties(sd);
return;
}
void doUpgradePreFederation(final StorageDirectory sd,
final NamespaceInfo nsInfo, final Configuration conf) throws IOException {
final int oldLV = getLayoutVersion();
LOG.info("Upgrading storage directory " + sd.getRoot()
+ ".\n old LV = " + oldLV
@ -804,15 +791,21 @@ private void doUgrade(final StorageDirectory sd,
linkAllBlocks(tmpDir, bbwDir, toDir, oldLV, conf);
// 4. Write version file under <SD>/current
layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
clusterID = nsInfo.getClusterID();
writeProperties(sd);
upgradeProperties(sd);
// 5. Rename <SD>/previous.tmp to <SD>/previous
rename(tmpDir, prevDir);
LOG.info("Upgrade of " + sd.getRoot()+ " is complete");
}
void upgradeProperties(StorageDirectory sd) throws IOException {
createStorageID(sd, layoutVersion);
LOG.info("Updating layout version from " + layoutVersion
+ " to " + HdfsConstants.DATANODE_LAYOUT_VERSION
+ " for storage " + sd.getRoot());
layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION;
writeProperties(sd);
}
/**