From 3848b44b378259e16a4471f6f786f967c46dbb74 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 27 Oct 2020 19:18:08 +0900 Subject: [PATCH] HDFS-15580. [JDK 12] DFSTestUtil#addDataNodeLayoutVersion fails (#2309) --- .../server/common/HdfsServerConstants.java | 7 ---- .../hdfs/server/common/StorageInfo.java | 3 +- .../datanode/BlockPoolSliceStorage.java | 23 +++++------ .../hadoop/hdfs/server/datanode/DataNode.java | 2 +- .../datanode/DataNodeLayoutVersion.java | 23 ++++++++++- .../hdfs/server/datanode/DataStorage.java | 36 +++++++++--------- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 38 ------------------- .../apache/hadoop/hdfs/TestDFSRollback.java | 7 ++-- .../hadoop/hdfs/TestDFSStartupVersions.java | 5 ++- .../apache/hadoop/hdfs/TestDFSUpgrade.java | 5 ++- .../hadoop/hdfs/TestDatanodeRegistration.java | 12 +++--- .../datanode/TestDataNodeRollingUpgrade.java | 25 +++++++++--- 12 files changed, 92 insertions(+), 94 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 9dc141b2717..254e9790da0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -27,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext; @@ -61,12 +60,6 @@ public interface HdfsServerConstants { */ int NAMENODE_LAYOUT_VERSION = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION; - /** - * Current layout version for DataNode. - * Please see {@link DataNodeLayoutVersion.Feature} on adding new layout version. - */ - int DATANODE_LAYOUT_VERSION - = DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION; /** * Path components that are reserved in HDFS. *

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java index 23911c0ac74..8643821c869 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java @@ -224,7 +224,8 @@ public class StorageInfo { } public int getServiceLayoutVersion() { - return storageType == NodeType.DATA_NODE ? HdfsServerConstants.DATANODE_LAYOUT_VERSION + return storageType == NodeType.DATA_NODE + ? DataNodeLayoutVersion.getCurrentLayoutVersion() : HdfsServerConstants.NAMENODE_LAYOUT_VERSION; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java index 5c8e6f48ce7..d2da2c56789 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java @@ -38,7 +38,6 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.HardLink; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.LayoutVersion; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; @@ -285,7 +284,7 @@ public class BlockPoolSliceStorage extends Storage { LOG.info("Formatting block pool {} directory {}", blockpoolID, bpSdir .getCurrentDir()); bpSdir.clearDirectory(); // create directory - this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + this.layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion(); this.cTime = nsInfo.getCTime(); this.namespaceID = nsInfo.getNamespaceID(); this.blockpoolID = nsInfo.getBlockPoolID(); @@ -388,7 +387,7 @@ public class BlockPoolSliceStorage extends Storage { } readProperties(sd); checkVersionUpgradable(this.layoutVersion); - assert this.layoutVersion >= HdfsServerConstants.DATANODE_LAYOUT_VERSION + assert this.layoutVersion >= DataNodeLayoutVersion.getCurrentLayoutVersion() : "Future version is not allowed"; if (getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException("Incompatible namespaceIDs in " @@ -402,17 +401,17 @@ public class BlockPoolSliceStorage extends Storage { + nsInfo.getBlockPoolID() + "; datanode blockpoolID = " + blockpoolID); } - if (this.layoutVersion == HdfsServerConstants.DATANODE_LAYOUT_VERSION + if (this.layoutVersion == DataNodeLayoutVersion.getCurrentLayoutVersion() && this.cTime == nsInfo.getCTime()) { return false; // regular startup } - if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION) { + if (this.layoutVersion > DataNodeLayoutVersion.getCurrentLayoutVersion()) { int restored = restoreBlockFilesFromTrash(getTrashRootDir(sd)); LOG.info("Restored {} block files from trash " + "before the layout upgrade. These blocks will be moved to " + "the previous directory during the upgrade", restored); } - if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION + if (this.layoutVersion > DataNodeLayoutVersion.getCurrentLayoutVersion() || this.cTime < nsInfo.getCTime()) { doUpgrade(sd, nsInfo, callables, conf); // upgrade return true; @@ -459,8 +458,8 @@ public class BlockPoolSliceStorage extends Storage { final int oldLV = getLayoutVersion(); LOG.info("Upgrading block pool storage directory {}.\n old LV = {}; old" + " CTime = {}.\n new LV = {}; new CTime = {}", - bpSd.getRoot(), oldLV, this.getCTime(), HdfsServerConstants - .DATANODE_LAYOUT_VERSION, nsInfo.getCTime()); + bpSd.getRoot(), oldLV, this.getCTime(), + DataNodeLayoutVersion.getCurrentLayoutVersion(), nsInfo.getCTime()); // get /previous directory String dnRoot = getDataNodeStorageRoot(bpSd.getRoot().getCanonicalPath()); StorageDirectory dnSdStorage = new StorageDirectory(new File(dnRoot)); @@ -507,7 +506,7 @@ public class BlockPoolSliceStorage extends Storage { throws IOException { // 3. Create new /current with block files hardlinks and VERSION linkAllBlocks(bpTmpDir, bpCurDir, oldLV, conf); - this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + this.layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion(); assert this.namespaceID == nsInfo.getNamespaceID() : "Data-node and name-node layout versions must be the same."; this.cTime = nsInfo.getCTime(); @@ -616,13 +615,15 @@ public class BlockPoolSliceStorage extends Storage { // the namespace state or can be further upgraded to it. // In another word, we can only roll back when ( storedLV >= software LV) // && ( DN.previousCTime <= NN.ctime) - if (!(prevInfo.getLayoutVersion() >= HdfsServerConstants.DATANODE_LAYOUT_VERSION && + if (!(prevInfo.getLayoutVersion() >= + DataNodeLayoutVersion.getCurrentLayoutVersion() && prevInfo.getCTime() <= nsInfo.getCTime())) { // cannot rollback throw new InconsistentFSStateException(bpSd.getRoot(), "Cannot rollback to a newer state.\nDatanode previous state: LV = " + prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime() + " is newer than the namespace state: LV = " - + HdfsServerConstants.DATANODE_LAYOUT_VERSION + " CTime = " + nsInfo.getCTime()); + + DataNodeLayoutVersion.getCurrentLayoutVersion() + " CTime = " + + nsInfo.getCTime()); } LOG.info("Rolling back storage directory {}.\n target LV = {}; target " diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index e2f98c37266..933a15faaaa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1579,7 +1579,7 @@ public class DataNode extends ReconfigurableBase if (storageInfo == null) { // it's null in the case of SimulatedDataSet storageInfo = new StorageInfo( - DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION, + DataNodeLayoutVersion.getCurrentLayoutVersion(), nsInfo.getNamespaceID(), nsInfo.clusterID, nsInfo.getCTime(), NodeType.DATA_NODE); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java index abe7b742f93..ceccbd98a0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeLayoutVersion.java @@ -26,15 +26,36 @@ import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LayoutVersion.FeatureInfo; import org.apache.hadoop.hdfs.protocol.LayoutVersion.LayoutFeature; +import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; + @InterfaceAudience.Private public class DataNodeLayoutVersion { /** Build layout version and corresponding feature matrix */ public final static Map> FEATURES = new HashMap>(); - public static final int CURRENT_LAYOUT_VERSION + private static int currentLayoutVersion = LayoutVersion.getCurrentLayoutVersion(Feature.values()); + /** + * Method for testing rolling upgrade. + * Do not call this method from production. + * + * @param lv new layout version to set + */ + @VisibleForTesting + static void setCurrentLayoutVersionForTesting(int lv) { + currentLayoutVersion = lv; + } + + /** + * Get current layout version of the DataNode. + * @return the current layout version of the DataNode + */ + public static int getCurrentLayoutVersion() { + return currentLayoutVersion; + } + static{ LayoutVersion.updateMap(FEATURES, LayoutVersion.Feature.values()); LayoutVersion.updateMap(FEATURES, DataNodeLayoutVersion.Feature.values()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index b6689fafb10..4e216db892f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -53,7 +53,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.LayoutVersion; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; @@ -571,7 +570,7 @@ public class DataStorage extends Storage { void format(StorageDirectory sd, NamespaceInfo nsInfo, String newDatanodeUuid, Configuration conf) throws IOException { sd.clearDirectory(); // create directory - this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + this.layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion(); this.clusterID = nsInfo.getClusterID(); this.namespaceID = nsInfo.getNamespaceID(); this.cTime = 0; @@ -728,8 +727,9 @@ public class DataStorage extends Storage { } readProperties(sd); checkVersionUpgradable(this.layoutVersion); - assert this.layoutVersion >= HdfsServerConstants.DATANODE_LAYOUT_VERSION : - "Future version is not allowed"; + assert this.layoutVersion >= + DataNodeLayoutVersion.getCurrentLayoutVersion() : + "Future version is not allowed"; boolean federationSupported = DataNodeLayoutVersion.supports( @@ -752,13 +752,13 @@ public class DataStorage extends Storage { } // regular start up. - if (this.layoutVersion == HdfsServerConstants.DATANODE_LAYOUT_VERSION) { + if (this.layoutVersion == DataNodeLayoutVersion.getCurrentLayoutVersion()) { createStorageID(sd, layoutVersion, conf); return false; // need to write properties } // do upgrade - if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION) { + if (this.layoutVersion > DataNodeLayoutVersion.getCurrentLayoutVersion()) { if (federationSupported) { // If the existing on-disk layout version supports federation, // simply update the properties. @@ -775,7 +775,7 @@ public class DataStorage extends Storage { // failed. throw new IOException("BUG: The stored LV = " + this.getLayoutVersion() + " is newer than the supported LV = " - + HdfsServerConstants.DATANODE_LAYOUT_VERSION); + + DataNodeLayoutVersion.getCurrentLayoutVersion()); } /** @@ -807,7 +807,7 @@ public class DataStorage extends Storage { final int oldLV = getLayoutVersion(); LOG.info("Upgrading storage directory {}.\n old LV = {}; old CTime = {}" + ".\n new LV = {}; new CTime = {}", sd.getRoot(), oldLV, - this.getCTime(), HdfsServerConstants.DATANODE_LAYOUT_VERSION, + this.getCTime(), DataNodeLayoutVersion.getCurrentLayoutVersion(), nsInfo.getCTime()); final File curDir = sd.getCurrentDir(); @@ -868,9 +868,9 @@ public class DataStorage extends Storage { throws IOException { createStorageID(sd, layoutVersion, conf); LOG.info("Updating layout version from {} to {} for storage {}", - layoutVersion, HdfsServerConstants.DATANODE_LAYOUT_VERSION, + layoutVersion, DataNodeLayoutVersion.getCurrentLayoutVersion(), sd.getRoot()); - layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion(); writeProperties(sd); } @@ -922,11 +922,11 @@ public class DataStorage extends Storage { // This is a regular startup or a post-federation rollback if (!prevDir.exists()) { if (DataNodeLayoutVersion.supports(LayoutVersion.Feature.FEDERATION, - HdfsServerConstants.DATANODE_LAYOUT_VERSION)) { - readProperties(sd, HdfsServerConstants.DATANODE_LAYOUT_VERSION); + DataNodeLayoutVersion.getCurrentLayoutVersion())) { + readProperties(sd, DataNodeLayoutVersion.getCurrentLayoutVersion()); writeProperties(sd); LOG.info("Layout version rolled back to {} for storage {}", - HdfsServerConstants.DATANODE_LAYOUT_VERSION, sd.getRoot()); + DataNodeLayoutVersion.getCurrentLayoutVersion(), sd.getRoot()); } return; } @@ -935,17 +935,19 @@ public class DataStorage extends Storage { // We allow rollback to a state, which is either consistent with // the namespace state or can be further upgraded to it. - if (!(prevInfo.getLayoutVersion() >= HdfsServerConstants.DATANODE_LAYOUT_VERSION - && prevInfo.getCTime() <= nsInfo.getCTime())) // cannot rollback + if (!(prevInfo.getLayoutVersion() >= + DataNodeLayoutVersion.getCurrentLayoutVersion() + && prevInfo.getCTime() <= nsInfo.getCTime())) { // cannot rollback throw new InconsistentFSStateException(sd.getRoot(), "Cannot rollback to a newer state.\nDatanode previous state: LV = " + prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime() + " is newer than the namespace state: LV = " - + HdfsServerConstants.DATANODE_LAYOUT_VERSION + " CTime = " + + DataNodeLayoutVersion.getCurrentLayoutVersion() + " CTime = " + nsInfo.getCTime()); + } LOG.info("Rolling back storage directory {}.\n target LV = {}; target " + "CTime = {}", sd.getRoot(), - HdfsServerConstants.DATANODE_LAYOUT_VERSION, nsInfo.getCTime()); + DataNodeLayoutVersion.getCurrentLayoutVersion(), nsInfo.getCTime()); File tmpDir = sd.getRemovedTmp(); assert !tmpDir.exists() : "removed.tmp directory must not exist."; // rename current to tmp diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 02dcb6d49a7..45a521fe961 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -45,8 +45,6 @@ import java.io.InputStream; import java.io.InterruptedIOException; import java.io.PrintStream; import java.io.RandomAccessFile; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.Socket; @@ -131,7 +129,6 @@ import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; @@ -147,12 +144,10 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; @@ -1953,39 +1948,6 @@ public class DFSTestUtil { FsShellRun(cmd, 0, null, conf); } - public static void addDataNodeLayoutVersion(final int lv, final String description) - throws NoSuchFieldException, IllegalAccessException { - Preconditions.checkState(lv < DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION); - - // Override {@link DataNodeLayoutVersion#CURRENT_LAYOUT_VERSION} via reflection. - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - Field field = DataNodeLayoutVersion.class.getField("CURRENT_LAYOUT_VERSION"); - field.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - field.setInt(null, lv); - - field = HdfsServerConstants.class.getField("DATANODE_LAYOUT_VERSION"); - field.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - field.setInt(null, lv); - - // Inject the feature into the FEATURES map. - final LayoutVersion.FeatureInfo featureInfo = - new LayoutVersion.FeatureInfo(lv, lv + 1, description, false); - final LayoutVersion.LayoutFeature feature = - new LayoutVersion.LayoutFeature() { - @Override - public LayoutVersion.FeatureInfo getInfo() { - return featureInfo; - } - }; - - // Update the FEATURES map with the new layout version. - LayoutVersion.updateMap(DataNodeLayoutVersion.FEATURES, - new LayoutVersion.LayoutFeature[] { feature }); - } - /** * Wait for datanode to reach alive or dead state for waitTime given in * milliseconds. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java index cc158877a38..99aed611824 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRollback.java @@ -30,10 +30,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.util.StringUtils; @@ -191,7 +191,7 @@ public class TestDFSRollback { UpgradeUtilities.getCurrentBlockPoolID(cluster)); // Put newer layout version in current. storageInfo = new StorageInfo( - HdfsServerConstants.DATANODE_LAYOUT_VERSION - 1, + DataNodeLayoutVersion.getCurrentLayoutVersion() - 1, UpgradeUtilities.getCurrentNamespaceID(cluster), UpgradeUtilities.getCurrentClusterID(cluster), UpgradeUtilities.getCurrentFsscTime(cluster), @@ -277,7 +277,8 @@ public class TestDFSRollback { UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "current"); baseDirs = UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "previous"); - storageInfo = new StorageInfo(HdfsServerConstants.DATANODE_LAYOUT_VERSION, + storageInfo = new StorageInfo( + DataNodeLayoutVersion.getCurrentLayoutVersion(), UpgradeUtilities.getCurrentNamespaceID(cluster), UpgradeUtilities.getCurrentClusterID(cluster), Long.MAX_VALUE, NodeType.DATA_NODE); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java index 7e7f4aa83e1..860794c083e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.junit.After; import org.junit.Test; @@ -89,7 +90,7 @@ public class TestDFSStartupVersions { */ private StorageData[] initializeVersions() throws Exception { int layoutVersionOld = Storage.LAST_UPGRADABLE_LAYOUT_VERSION; - int layoutVersionCur = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + int layoutVersionCur = DataNodeLayoutVersion.getCurrentLayoutVersion(); int layoutVersionNew = Integer.MIN_VALUE; int namespaceIdCur = UpgradeUtilities.getCurrentNamespaceID(null); int namespaceIdOld = Integer.MIN_VALUE; @@ -200,7 +201,7 @@ public class TestDFSStartupVersions { return false; } // check #3 - int softwareLV = HdfsServerConstants.DATANODE_LAYOUT_VERSION; + int softwareLV = DataNodeLayoutVersion.getCurrentLayoutVersion(); int storedLV = datanodeVer.getLayoutVersion(); if (softwareLV == storedLV && datanodeVer.getCTime() == namenodeVer.getCTime()) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java index e8bd5cefc7f..ddf5461181b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java @@ -36,12 +36,12 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.apache.hadoop.hdfs.server.namenode.TestParallelImageWrite; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.StringUtils; @@ -303,7 +303,8 @@ public class TestDFSUpgrade { UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current"); cluster = createCluster(); baseDirs = UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "current"); - storageInfo = new StorageInfo(HdfsServerConstants.DATANODE_LAYOUT_VERSION, + storageInfo = new StorageInfo( + DataNodeLayoutVersion.getCurrentLayoutVersion(), UpgradeUtilities.getCurrentNamespaceID(cluster), UpgradeUtilities.getCurrentClusterID(cluster), Long.MAX_VALUE, NodeType.DATA_NODE); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java index 4436fd3cac9..f029ee5298e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java @@ -28,10 +28,10 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -191,8 +191,8 @@ public class TestDatanodeRegistration { .getCTime(); StorageInfo mockStorageInfo = mock(StorageInfo.class); doReturn(nnCTime).when(mockStorageInfo).getCTime(); - doReturn(HdfsServerConstants.DATANODE_LAYOUT_VERSION).when(mockStorageInfo) - .getLayoutVersion(); + doReturn(DataNodeLayoutVersion.getCurrentLayoutVersion()) + .when(mockStorageInfo).getLayoutVersion(); DatanodeRegistration dnReg = new DatanodeRegistration(dnId, mockStorageInfo, null, VersionInfo.getVersion()); rpcServer.registerDatanode(dnReg); @@ -236,7 +236,8 @@ public class TestDatanodeRegistration { doReturn(nnCTime).when(mockStorageInfo).getCTime(); DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class); - doReturn(HdfsServerConstants.DATANODE_LAYOUT_VERSION).when(mockDnReg).getVersion(); + doReturn(DataNodeLayoutVersion.getCurrentLayoutVersion()) + .when(mockDnReg).getVersion(); doReturn("127.0.0.1").when(mockDnReg).getIpAddr(); doReturn(123).when(mockDnReg).getXferPort(); doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid(); @@ -285,7 +286,8 @@ public class TestDatanodeRegistration { doReturn(nnCTime).when(mockStorageInfo).getCTime(); DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class); - doReturn(HdfsServerConstants.DATANODE_LAYOUT_VERSION).when(mockDnReg).getVersion(); + doReturn(DataNodeLayoutVersion.getCurrentLayoutVersion()) + .when(mockDnReg).getVersion(); doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid(); doReturn(mockStorageInfo).when(mockDnReg).getStorageInfo(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java index 04f6ce1f59e..b5af8805fb3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.hdfs.TestRollingUpgrade; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -361,9 +362,8 @@ public class TestDataNodeRollingUpgrade { // Restart the DN with a new layout version to trigger layout upgrade. LOG.info("Shutting down the Datanode"); MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(0); - DFSTestUtil.addDataNodeLayoutVersion( - DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1, - "Test Layout for TestDataNodeRollingUpgrade"); + addDataNodeLayoutVersion( + DataNodeLayoutVersion.getCurrentLayoutVersion() - 1); LOG.info("Restarting the DataNode"); cluster.restartDataNode(dnprop, true); cluster.waitActive(); @@ -422,9 +422,8 @@ public class TestDataNodeRollingUpgrade { // Restart the DN with a new layout version to trigger layout upgrade. LOG.info("Shutting down the Datanode"); MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(0); - DFSTestUtil.addDataNodeLayoutVersion( - DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1, - "Test Layout for TestDataNodeRollingUpgrade"); + addDataNodeLayoutVersion( + DataNodeLayoutVersion.getCurrentLayoutVersion() - 1); LOG.info("Restarting the DataNode"); cluster.restartDataNode(dnprop, true); cluster.waitActive(); @@ -470,4 +469,18 @@ public class TestDataNodeRollingUpgrade { shutdownCluster(); } } + + static void addDataNodeLayoutVersion(final int lv) { + assertTrue(lv < DataNodeLayoutVersion.getCurrentLayoutVersion()); + DataNodeLayoutVersion.setCurrentLayoutVersionForTesting(lv); + + // Inject the feature into the FEATURES map. + final LayoutVersion.FeatureInfo featureInfo = + new LayoutVersion.FeatureInfo(lv, lv + 1, + "Test Layout for TestDataNodeRollingUpgrade", false); + + // Update the FEATURES map with the new layout version. + LayoutVersion.updateMap(DataNodeLayoutVersion.FEATURES, + new LayoutVersion.LayoutFeature[]{() -> featureInfo}); + } }