HDFS-15580. [JDK 12] DFSTestUtil#addDataNodeLayoutVersion fails (#2309)
This commit is contained in:
parent
afaab3d332
commit
3848b44b37
|
@ -27,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
|||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hdfs.DFSUtil;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
|
||||
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
|
||||
import org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext;
|
||||
|
||||
|
@ -61,12 +60,6 @@ public interface HdfsServerConstants {
|
|||
*/
|
||||
int NAMENODE_LAYOUT_VERSION
|
||||
= NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION;
|
||||
/**
|
||||
* Current layout version for DataNode.
|
||||
* Please see {@link DataNodeLayoutVersion.Feature} on adding new layout version.
|
||||
*/
|
||||
int DATANODE_LAYOUT_VERSION
|
||||
= DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION;
|
||||
/**
|
||||
* Path components that are reserved in HDFS.
|
||||
* <p>
|
||||
|
|
|
@ -224,7 +224,8 @@ public class StorageInfo {
|
|||
}
|
||||
|
||||
public int getServiceLayoutVersion() {
|
||||
return storageType == NodeType.DATA_NODE ? HdfsServerConstants.DATANODE_LAYOUT_VERSION
|
||||
return storageType == NodeType.DATA_NODE
|
||||
? DataNodeLayoutVersion.getCurrentLayoutVersion()
|
||||
: HdfsServerConstants.NAMENODE_LAYOUT_VERSION;
|
||||
}
|
||||
|
||||
|
|
|
@ -38,7 +38,6 @@ import org.apache.hadoop.fs.FileUtil;
|
|||
import org.apache.hadoop.fs.HardLink;
|
||||
import org.apache.hadoop.fs.StorageType;
|
||||
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
|
||||
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
|
||||
|
@ -285,7 +284,7 @@ public class BlockPoolSliceStorage extends Storage {
|
|||
LOG.info("Formatting block pool {} directory {}", blockpoolID, bpSdir
|
||||
.getCurrentDir());
|
||||
bpSdir.clearDirectory(); // create directory
|
||||
this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
|
||||
this.layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion();
|
||||
this.cTime = nsInfo.getCTime();
|
||||
this.namespaceID = nsInfo.getNamespaceID();
|
||||
this.blockpoolID = nsInfo.getBlockPoolID();
|
||||
|
@ -388,7 +387,7 @@ public class BlockPoolSliceStorage extends Storage {
|
|||
}
|
||||
readProperties(sd);
|
||||
checkVersionUpgradable(this.layoutVersion);
|
||||
assert this.layoutVersion >= HdfsServerConstants.DATANODE_LAYOUT_VERSION
|
||||
assert this.layoutVersion >= DataNodeLayoutVersion.getCurrentLayoutVersion()
|
||||
: "Future version is not allowed";
|
||||
if (getNamespaceID() != nsInfo.getNamespaceID()) {
|
||||
throw new IOException("Incompatible namespaceIDs in "
|
||||
|
@ -402,17 +401,17 @@ public class BlockPoolSliceStorage extends Storage {
|
|||
+ nsInfo.getBlockPoolID() + "; datanode blockpoolID = "
|
||||
+ blockpoolID);
|
||||
}
|
||||
if (this.layoutVersion == HdfsServerConstants.DATANODE_LAYOUT_VERSION
|
||||
if (this.layoutVersion == DataNodeLayoutVersion.getCurrentLayoutVersion()
|
||||
&& this.cTime == nsInfo.getCTime()) {
|
||||
return false; // regular startup
|
||||
}
|
||||
if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION) {
|
||||
if (this.layoutVersion > DataNodeLayoutVersion.getCurrentLayoutVersion()) {
|
||||
int restored = restoreBlockFilesFromTrash(getTrashRootDir(sd));
|
||||
LOG.info("Restored {} block files from trash " +
|
||||
"before the layout upgrade. These blocks will be moved to " +
|
||||
"the previous directory during the upgrade", restored);
|
||||
}
|
||||
if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION
|
||||
if (this.layoutVersion > DataNodeLayoutVersion.getCurrentLayoutVersion()
|
||||
|| this.cTime < nsInfo.getCTime()) {
|
||||
doUpgrade(sd, nsInfo, callables, conf); // upgrade
|
||||
return true;
|
||||
|
@ -459,8 +458,8 @@ public class BlockPoolSliceStorage extends Storage {
|
|||
final int oldLV = getLayoutVersion();
|
||||
LOG.info("Upgrading block pool storage directory {}.\n old LV = {}; old"
|
||||
+ " CTime = {}.\n new LV = {}; new CTime = {}",
|
||||
bpSd.getRoot(), oldLV, this.getCTime(), HdfsServerConstants
|
||||
.DATANODE_LAYOUT_VERSION, nsInfo.getCTime());
|
||||
bpSd.getRoot(), oldLV, this.getCTime(),
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion(), nsInfo.getCTime());
|
||||
// get <SD>/previous directory
|
||||
String dnRoot = getDataNodeStorageRoot(bpSd.getRoot().getCanonicalPath());
|
||||
StorageDirectory dnSdStorage = new StorageDirectory(new File(dnRoot));
|
||||
|
@ -507,7 +506,7 @@ public class BlockPoolSliceStorage extends Storage {
|
|||
throws IOException {
|
||||
// 3. Create new <SD>/current with block files hardlinks and VERSION
|
||||
linkAllBlocks(bpTmpDir, bpCurDir, oldLV, conf);
|
||||
this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
|
||||
this.layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion();
|
||||
assert this.namespaceID == nsInfo.getNamespaceID()
|
||||
: "Data-node and name-node layout versions must be the same.";
|
||||
this.cTime = nsInfo.getCTime();
|
||||
|
@ -616,13 +615,15 @@ public class BlockPoolSliceStorage extends Storage {
|
|||
// the namespace state or can be further upgraded to it.
|
||||
// In another word, we can only roll back when ( storedLV >= software LV)
|
||||
// && ( DN.previousCTime <= NN.ctime)
|
||||
if (!(prevInfo.getLayoutVersion() >= HdfsServerConstants.DATANODE_LAYOUT_VERSION &&
|
||||
if (!(prevInfo.getLayoutVersion() >=
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion() &&
|
||||
prevInfo.getCTime() <= nsInfo.getCTime())) { // cannot rollback
|
||||
throw new InconsistentFSStateException(bpSd.getRoot(),
|
||||
"Cannot rollback to a newer state.\nDatanode previous state: LV = "
|
||||
+ prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime()
|
||||
+ " is newer than the namespace state: LV = "
|
||||
+ HdfsServerConstants.DATANODE_LAYOUT_VERSION + " CTime = " + nsInfo.getCTime());
|
||||
+ DataNodeLayoutVersion.getCurrentLayoutVersion() + " CTime = "
|
||||
+ nsInfo.getCTime());
|
||||
}
|
||||
|
||||
LOG.info("Rolling back storage directory {}.\n target LV = {}; target "
|
||||
|
|
|
@ -1579,7 +1579,7 @@ public class DataNode extends ReconfigurableBase
|
|||
if (storageInfo == null) {
|
||||
// it's null in the case of SimulatedDataSet
|
||||
storageInfo = new StorageInfo(
|
||||
DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION,
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion(),
|
||||
nsInfo.getNamespaceID(), nsInfo.clusterID, nsInfo.getCTime(),
|
||||
NodeType.DATA_NODE);
|
||||
}
|
||||
|
|
|
@ -26,15 +26,36 @@ import org.apache.hadoop.hdfs.protocol.LayoutVersion;
|
|||
import org.apache.hadoop.hdfs.protocol.LayoutVersion.FeatureInfo;
|
||||
import org.apache.hadoop.hdfs.protocol.LayoutVersion.LayoutFeature;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
@InterfaceAudience.Private
|
||||
public class DataNodeLayoutVersion {
|
||||
/** Build layout version and corresponding feature matrix */
|
||||
public final static Map<Integer, SortedSet<LayoutFeature>> FEATURES =
|
||||
new HashMap<Integer, SortedSet<LayoutFeature>>();
|
||||
|
||||
public static final int CURRENT_LAYOUT_VERSION
|
||||
private static int currentLayoutVersion
|
||||
= LayoutVersion.getCurrentLayoutVersion(Feature.values());
|
||||
|
||||
/**
|
||||
* Method for testing rolling upgrade.
|
||||
* Do not call this method from production.
|
||||
*
|
||||
* @param lv new layout version to set
|
||||
*/
|
||||
@VisibleForTesting
|
||||
static void setCurrentLayoutVersionForTesting(int lv) {
|
||||
currentLayoutVersion = lv;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current layout version of the DataNode.
|
||||
* @return the current layout version of the DataNode
|
||||
*/
|
||||
public static int getCurrentLayoutVersion() {
|
||||
return currentLayoutVersion;
|
||||
}
|
||||
|
||||
static{
|
||||
LayoutVersion.updateMap(FEATURES, LayoutVersion.Feature.values());
|
||||
LayoutVersion.updateMap(FEATURES, DataNodeLayoutVersion.Feature.values());
|
||||
|
|
|
@ -53,7 +53,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|||
import org.apache.hadoop.hdfs.DFSUtilClient;
|
||||
import org.apache.hadoop.hdfs.protocol.Block;
|
||||
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
|
||||
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
|
||||
|
@ -571,7 +570,7 @@ public class DataStorage extends Storage {
|
|||
void format(StorageDirectory sd, NamespaceInfo nsInfo,
|
||||
String newDatanodeUuid, Configuration conf) throws IOException {
|
||||
sd.clearDirectory(); // create directory
|
||||
this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
|
||||
this.layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion();
|
||||
this.clusterID = nsInfo.getClusterID();
|
||||
this.namespaceID = nsInfo.getNamespaceID();
|
||||
this.cTime = 0;
|
||||
|
@ -728,8 +727,9 @@ public class DataStorage extends Storage {
|
|||
}
|
||||
readProperties(sd);
|
||||
checkVersionUpgradable(this.layoutVersion);
|
||||
assert this.layoutVersion >= HdfsServerConstants.DATANODE_LAYOUT_VERSION :
|
||||
"Future version is not allowed";
|
||||
assert this.layoutVersion >=
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion() :
|
||||
"Future version is not allowed";
|
||||
|
||||
boolean federationSupported =
|
||||
DataNodeLayoutVersion.supports(
|
||||
|
@ -752,13 +752,13 @@ public class DataStorage extends Storage {
|
|||
}
|
||||
|
||||
// regular start up.
|
||||
if (this.layoutVersion == HdfsServerConstants.DATANODE_LAYOUT_VERSION) {
|
||||
if (this.layoutVersion == DataNodeLayoutVersion.getCurrentLayoutVersion()) {
|
||||
createStorageID(sd, layoutVersion, conf);
|
||||
return false; // need to write properties
|
||||
}
|
||||
|
||||
// do upgrade
|
||||
if (this.layoutVersion > HdfsServerConstants.DATANODE_LAYOUT_VERSION) {
|
||||
if (this.layoutVersion > DataNodeLayoutVersion.getCurrentLayoutVersion()) {
|
||||
if (federationSupported) {
|
||||
// If the existing on-disk layout version supports federation,
|
||||
// simply update the properties.
|
||||
|
@ -775,7 +775,7 @@ public class DataStorage extends Storage {
|
|||
// failed.
|
||||
throw new IOException("BUG: The stored LV = " + this.getLayoutVersion()
|
||||
+ " is newer than the supported LV = "
|
||||
+ HdfsServerConstants.DATANODE_LAYOUT_VERSION);
|
||||
+ DataNodeLayoutVersion.getCurrentLayoutVersion());
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -807,7 +807,7 @@ public class DataStorage extends Storage {
|
|||
final int oldLV = getLayoutVersion();
|
||||
LOG.info("Upgrading storage directory {}.\n old LV = {}; old CTime = {}"
|
||||
+ ".\n new LV = {}; new CTime = {}", sd.getRoot(), oldLV,
|
||||
this.getCTime(), HdfsServerConstants.DATANODE_LAYOUT_VERSION,
|
||||
this.getCTime(), DataNodeLayoutVersion.getCurrentLayoutVersion(),
|
||||
nsInfo.getCTime());
|
||||
|
||||
final File curDir = sd.getCurrentDir();
|
||||
|
@ -868,9 +868,9 @@ public class DataStorage extends Storage {
|
|||
throws IOException {
|
||||
createStorageID(sd, layoutVersion, conf);
|
||||
LOG.info("Updating layout version from {} to {} for storage {}",
|
||||
layoutVersion, HdfsServerConstants.DATANODE_LAYOUT_VERSION,
|
||||
layoutVersion, DataNodeLayoutVersion.getCurrentLayoutVersion(),
|
||||
sd.getRoot());
|
||||
layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
|
||||
layoutVersion = DataNodeLayoutVersion.getCurrentLayoutVersion();
|
||||
writeProperties(sd);
|
||||
}
|
||||
|
||||
|
@ -922,11 +922,11 @@ public class DataStorage extends Storage {
|
|||
// This is a regular startup or a post-federation rollback
|
||||
if (!prevDir.exists()) {
|
||||
if (DataNodeLayoutVersion.supports(LayoutVersion.Feature.FEDERATION,
|
||||
HdfsServerConstants.DATANODE_LAYOUT_VERSION)) {
|
||||
readProperties(sd, HdfsServerConstants.DATANODE_LAYOUT_VERSION);
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion())) {
|
||||
readProperties(sd, DataNodeLayoutVersion.getCurrentLayoutVersion());
|
||||
writeProperties(sd);
|
||||
LOG.info("Layout version rolled back to {} for storage {}",
|
||||
HdfsServerConstants.DATANODE_LAYOUT_VERSION, sd.getRoot());
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion(), sd.getRoot());
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
@ -935,17 +935,19 @@ public class DataStorage extends Storage {
|
|||
|
||||
// We allow rollback to a state, which is either consistent with
|
||||
// the namespace state or can be further upgraded to it.
|
||||
if (!(prevInfo.getLayoutVersion() >= HdfsServerConstants.DATANODE_LAYOUT_VERSION
|
||||
&& prevInfo.getCTime() <= nsInfo.getCTime())) // cannot rollback
|
||||
if (!(prevInfo.getLayoutVersion() >=
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion()
|
||||
&& prevInfo.getCTime() <= nsInfo.getCTime())) { // cannot rollback
|
||||
throw new InconsistentFSStateException(sd.getRoot(),
|
||||
"Cannot rollback to a newer state.\nDatanode previous state: LV = "
|
||||
+ prevInfo.getLayoutVersion() + " CTime = " + prevInfo.getCTime()
|
||||
+ " is newer than the namespace state: LV = "
|
||||
+ HdfsServerConstants.DATANODE_LAYOUT_VERSION + " CTime = "
|
||||
+ DataNodeLayoutVersion.getCurrentLayoutVersion() + " CTime = "
|
||||
+ nsInfo.getCTime());
|
||||
}
|
||||
LOG.info("Rolling back storage directory {}.\n target LV = {}; target "
|
||||
+ "CTime = {}", sd.getRoot(),
|
||||
HdfsServerConstants.DATANODE_LAYOUT_VERSION, nsInfo.getCTime());
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion(), nsInfo.getCTime());
|
||||
File tmpDir = sd.getRemovedTmp();
|
||||
assert !tmpDir.exists() : "removed.tmp directory must not exist.";
|
||||
// rename current to tmp
|
||||
|
|
|
@ -45,8 +45,6 @@ import java.io.InputStream;
|
|||
import java.io.InterruptedIOException;
|
||||
import java.io.PrintStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Modifier;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.Socket;
|
||||
|
@ -131,7 +129,6 @@ import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
|
|||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
|
||||
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
||||
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
||||
import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
|
||||
|
@ -147,12 +144,10 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
|
|||
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
|
||||
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
|
||||
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
|
||||
import org.apache.hadoop.hdfs.server.common.StorageInfo;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNode;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
|
||||
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
|
||||
import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw;
|
||||
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
|
||||
|
@ -1953,39 +1948,6 @@ public class DFSTestUtil {
|
|||
FsShellRun(cmd, 0, null, conf);
|
||||
}
|
||||
|
||||
public static void addDataNodeLayoutVersion(final int lv, final String description)
|
||||
throws NoSuchFieldException, IllegalAccessException {
|
||||
Preconditions.checkState(lv < DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
|
||||
|
||||
// Override {@link DataNodeLayoutVersion#CURRENT_LAYOUT_VERSION} via reflection.
|
||||
Field modifiersField = Field.class.getDeclaredField("modifiers");
|
||||
modifiersField.setAccessible(true);
|
||||
Field field = DataNodeLayoutVersion.class.getField("CURRENT_LAYOUT_VERSION");
|
||||
field.setAccessible(true);
|
||||
modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
|
||||
field.setInt(null, lv);
|
||||
|
||||
field = HdfsServerConstants.class.getField("DATANODE_LAYOUT_VERSION");
|
||||
field.setAccessible(true);
|
||||
modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
|
||||
field.setInt(null, lv);
|
||||
|
||||
// Inject the feature into the FEATURES map.
|
||||
final LayoutVersion.FeatureInfo featureInfo =
|
||||
new LayoutVersion.FeatureInfo(lv, lv + 1, description, false);
|
||||
final LayoutVersion.LayoutFeature feature =
|
||||
new LayoutVersion.LayoutFeature() {
|
||||
@Override
|
||||
public LayoutVersion.FeatureInfo getInfo() {
|
||||
return featureInfo;
|
||||
}
|
||||
};
|
||||
|
||||
// Update the FEATURES map with the new layout version.
|
||||
LayoutVersion.updateMap(DataNodeLayoutVersion.FEATURES,
|
||||
new LayoutVersion.LayoutFeature[] { feature });
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for datanode to reach alive or dead state for waitTime given in
|
||||
* milliseconds.
|
||||
|
|
|
@ -30,10 +30,10 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
|
||||
import org.apache.hadoop.hdfs.server.common.StorageInfo;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
|
||||
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NameNode;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
|
@ -191,7 +191,7 @@ public class TestDFSRollback {
|
|||
UpgradeUtilities.getCurrentBlockPoolID(cluster));
|
||||
// Put newer layout version in current.
|
||||
storageInfo = new StorageInfo(
|
||||
HdfsServerConstants.DATANODE_LAYOUT_VERSION - 1,
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion() - 1,
|
||||
UpgradeUtilities.getCurrentNamespaceID(cluster),
|
||||
UpgradeUtilities.getCurrentClusterID(cluster),
|
||||
UpgradeUtilities.getCurrentFsscTime(cluster),
|
||||
|
@ -277,7 +277,8 @@ public class TestDFSRollback {
|
|||
|
||||
UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "current");
|
||||
baseDirs = UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "previous");
|
||||
storageInfo = new StorageInfo(HdfsServerConstants.DATANODE_LAYOUT_VERSION,
|
||||
storageInfo = new StorageInfo(
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion(),
|
||||
UpgradeUtilities.getCurrentNamespaceID(cluster),
|
||||
UpgradeUtilities.getCurrentClusterID(cluster), Long.MAX_VALUE,
|
||||
NodeType.DATA_NODE);
|
||||
|
|
|
@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
|
|||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
|
||||
import org.apache.hadoop.hdfs.server.common.Storage;
|
||||
import org.apache.hadoop.hdfs.server.common.StorageInfo;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
|
||||
import org.junit.After;
|
||||
import org.junit.Test;
|
||||
|
||||
|
@ -89,7 +90,7 @@ public class TestDFSStartupVersions {
|
|||
*/
|
||||
private StorageData[] initializeVersions() throws Exception {
|
||||
int layoutVersionOld = Storage.LAST_UPGRADABLE_LAYOUT_VERSION;
|
||||
int layoutVersionCur = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
|
||||
int layoutVersionCur = DataNodeLayoutVersion.getCurrentLayoutVersion();
|
||||
int layoutVersionNew = Integer.MIN_VALUE;
|
||||
int namespaceIdCur = UpgradeUtilities.getCurrentNamespaceID(null);
|
||||
int namespaceIdOld = Integer.MIN_VALUE;
|
||||
|
@ -200,7 +201,7 @@ public class TestDFSStartupVersions {
|
|||
return false;
|
||||
}
|
||||
// check #3
|
||||
int softwareLV = HdfsServerConstants.DATANODE_LAYOUT_VERSION;
|
||||
int softwareLV = DataNodeLayoutVersion.getCurrentLayoutVersion();
|
||||
int storedLV = datanodeVer.getLayoutVersion();
|
||||
if (softwareLV == storedLV &&
|
||||
datanodeVer.getCTime() == namenodeVer.getCTime())
|
||||
|
|
|
@ -36,12 +36,12 @@ import org.slf4j.LoggerFactory;
|
|||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
|
||||
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
|
||||
import org.apache.hadoop.hdfs.server.common.Storage;
|
||||
import org.apache.hadoop.hdfs.server.common.StorageInfo;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
|
||||
import org.apache.hadoop.hdfs.server.namenode.TestParallelImageWrite;
|
||||
import org.apache.hadoop.ipc.RemoteException;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
|
@ -303,7 +303,8 @@ public class TestDFSUpgrade {
|
|||
UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current");
|
||||
cluster = createCluster();
|
||||
baseDirs = UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs, "current");
|
||||
storageInfo = new StorageInfo(HdfsServerConstants.DATANODE_LAYOUT_VERSION,
|
||||
storageInfo = new StorageInfo(
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion(),
|
||||
UpgradeUtilities.getCurrentNamespaceID(cluster),
|
||||
UpgradeUtilities.getCurrentClusterID(cluster), Long.MAX_VALUE,
|
||||
NodeType.DATA_NODE);
|
||||
|
|
|
@ -28,10 +28,10 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
|
|||
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
|
||||
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
|
||||
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
|
||||
import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
|
||||
import org.apache.hadoop.hdfs.server.common.StorageInfo;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNode;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
|
||||
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
|
||||
|
@ -191,8 +191,8 @@ public class TestDatanodeRegistration {
|
|||
.getCTime();
|
||||
StorageInfo mockStorageInfo = mock(StorageInfo.class);
|
||||
doReturn(nnCTime).when(mockStorageInfo).getCTime();
|
||||
doReturn(HdfsServerConstants.DATANODE_LAYOUT_VERSION).when(mockStorageInfo)
|
||||
.getLayoutVersion();
|
||||
doReturn(DataNodeLayoutVersion.getCurrentLayoutVersion())
|
||||
.when(mockStorageInfo).getLayoutVersion();
|
||||
DatanodeRegistration dnReg = new DatanodeRegistration(dnId,
|
||||
mockStorageInfo, null, VersionInfo.getVersion());
|
||||
rpcServer.registerDatanode(dnReg);
|
||||
|
@ -236,7 +236,8 @@ public class TestDatanodeRegistration {
|
|||
doReturn(nnCTime).when(mockStorageInfo).getCTime();
|
||||
|
||||
DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class);
|
||||
doReturn(HdfsServerConstants.DATANODE_LAYOUT_VERSION).when(mockDnReg).getVersion();
|
||||
doReturn(DataNodeLayoutVersion.getCurrentLayoutVersion())
|
||||
.when(mockDnReg).getVersion();
|
||||
doReturn("127.0.0.1").when(mockDnReg).getIpAddr();
|
||||
doReturn(123).when(mockDnReg).getXferPort();
|
||||
doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid();
|
||||
|
@ -285,7 +286,8 @@ public class TestDatanodeRegistration {
|
|||
doReturn(nnCTime).when(mockStorageInfo).getCTime();
|
||||
|
||||
DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class);
|
||||
doReturn(HdfsServerConstants.DATANODE_LAYOUT_VERSION).when(mockDnReg).getVersion();
|
||||
doReturn(DataNodeLayoutVersion.getCurrentLayoutVersion())
|
||||
.when(mockDnReg).getVersion();
|
||||
doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid();
|
||||
doReturn(mockStorageInfo).when(mockDnReg).getStorageInfo();
|
||||
|
||||
|
|
|
@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
|
|||
import org.apache.hadoop.hdfs.TestRollingUpgrade;
|
||||
import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
|
||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
|
||||
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
|
||||
import org.apache.hadoop.hdfs.server.namenode.NameNode;
|
||||
|
@ -361,9 +362,8 @@ public class TestDataNodeRollingUpgrade {
|
|||
// Restart the DN with a new layout version to trigger layout upgrade.
|
||||
LOG.info("Shutting down the Datanode");
|
||||
MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(0);
|
||||
DFSTestUtil.addDataNodeLayoutVersion(
|
||||
DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1,
|
||||
"Test Layout for TestDataNodeRollingUpgrade");
|
||||
addDataNodeLayoutVersion(
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion() - 1);
|
||||
LOG.info("Restarting the DataNode");
|
||||
cluster.restartDataNode(dnprop, true);
|
||||
cluster.waitActive();
|
||||
|
@ -422,9 +422,8 @@ public class TestDataNodeRollingUpgrade {
|
|||
// Restart the DN with a new layout version to trigger layout upgrade.
|
||||
LOG.info("Shutting down the Datanode");
|
||||
MiniDFSCluster.DataNodeProperties dnprop = cluster.stopDataNode(0);
|
||||
DFSTestUtil.addDataNodeLayoutVersion(
|
||||
DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1,
|
||||
"Test Layout for TestDataNodeRollingUpgrade");
|
||||
addDataNodeLayoutVersion(
|
||||
DataNodeLayoutVersion.getCurrentLayoutVersion() - 1);
|
||||
LOG.info("Restarting the DataNode");
|
||||
cluster.restartDataNode(dnprop, true);
|
||||
cluster.waitActive();
|
||||
|
@ -470,4 +469,18 @@ public class TestDataNodeRollingUpgrade {
|
|||
shutdownCluster();
|
||||
}
|
||||
}
|
||||
|
||||
static void addDataNodeLayoutVersion(final int lv) {
|
||||
assertTrue(lv < DataNodeLayoutVersion.getCurrentLayoutVersion());
|
||||
DataNodeLayoutVersion.setCurrentLayoutVersionForTesting(lv);
|
||||
|
||||
// Inject the feature into the FEATURES map.
|
||||
final LayoutVersion.FeatureInfo featureInfo =
|
||||
new LayoutVersion.FeatureInfo(lv, lv + 1,
|
||||
"Test Layout for TestDataNodeRollingUpgrade", false);
|
||||
|
||||
// Update the FEATURES map with the new layout version.
|
||||
LayoutVersion.updateMap(DataNodeLayoutVersion.FEATURES,
|
||||
new LayoutVersion.LayoutFeature[]{() -> featureInfo});
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue