From 8fc0d04517912766a3102f3e611f7d0fabd2f815 Mon Sep 17 00:00:00 2001 From: Inigo Goiri Date: Fri, 7 Dec 2018 18:30:36 -0800 Subject: [PATCH] HDFS-14001. [PROVIDED Storage] bootstrapStandby should manage the InMemoryAliasMap. Contributed by Virajith Jalaparti. --- .../server/aliasmap/InMemoryAliasMap.java | 216 ++++++++++++++++-- .../InMemoryLevelDBAliasMapServer.java | 8 + .../hdfs/server/namenode/ImageServlet.java | 32 ++- .../hadoop/hdfs/server/namenode/NameNode.java | 11 + .../server/namenode/NameNodeHttpServer.java | 15 ++ .../hdfs/server/namenode/TransferFsImage.java | 21 +- .../server/namenode/ha/BootstrapStandby.java | 91 +++++++- .../namenode/metrics/NameNodeMetrics.java | 6 + .../apache/hadoop/hdfs/MiniDFSCluster.java | 23 ++ .../aliasmap/ITestInMemoryAliasMap.java | 30 +++ .../server/aliasmap/TestInMemoryAliasMap.java | 45 ---- .../TestInMemoryLevelDBAliasMapClient.java | 9 - .../namenode/ha/TestBootstrapAliasmap.java | 99 ++++++++ .../namenode/ITestProvidedImplementation.java | 139 ++++++++++- 14 files changed, 669 insertions(+), 76 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java index 142a040ca82..eeee2a3c6b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java @@ -19,10 +19,14 @@ package org.apache.hadoop.hdfs.server.aliasmap; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.protobuf.InvalidProtocolBufferException; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation; @@ -30,17 +34,28 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ProvidedStorageLocationProto; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.server.common.FileRegion; +import org.apache.hadoop.hdfs.server.namenode.ImageServlet; +import org.apache.hadoop.hdfs.server.namenode.TransferFsImage; +import org.apache.hadoop.hdfs.util.DataTransferThrottler; +import org.apache.hadoop.io.IOUtils; import org.fusesource.leveldbjni.JniDBFactory; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBIterator; import org.iq80.leveldb.Options; +import org.iq80.leveldb.ReadOptions; +import org.iq80.leveldb.Snapshot; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import javax.servlet.http.HttpServletResponse; +import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import java.util.Map; import java.util.Optional; @@ -57,6 +72,9 @@ public class InMemoryAliasMap implements InMemoryAliasMapProtocol, private static final Logger LOG = LoggerFactory .getLogger(InMemoryAliasMap.class); + private static final String SNAPSHOT_COPY_DIR = "aliasmap_snapshot"; + private static final String TAR_NAME = "aliasmap.tar.gz"; + private final URI aliasMapURI; private final DB levelDb; private Configuration conf; private String blockPoolID; @@ -71,22 +89,15 @@ public class InMemoryAliasMap implements InMemoryAliasMapProtocol, return this.conf; } - @VisibleForTesting - static String createPathErrorMessage(String directory) { - return new StringBuilder() - .append("Configured directory '") - .append(directory) - .append("' doesn't exist") - .toString(); - } - public static @Nonnull InMemoryAliasMap init(Configuration conf, String blockPoolID) throws IOException { Options options = new Options(); options.createIfMissing(true); String directory = conf.get(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR); - LOG.info("Attempting to load InMemoryAliasMap from \"{}\"", directory); + if (directory == null) { + throw new IOException("InMemoryAliasMap location is null"); + } File levelDBpath; if (blockPoolID != null) { levelDBpath = new File(directory, blockPoolID); @@ -94,17 +105,23 @@ public class InMemoryAliasMap implements InMemoryAliasMapProtocol, levelDBpath = new File(directory); } if (!levelDBpath.exists()) { - String error = createPathErrorMessage(directory); - throw new IOException(error); + LOG.warn("InMemoryAliasMap location {} is missing. Creating it.", + levelDBpath); + if(!levelDBpath.mkdirs()) { + throw new IOException( + "Unable to create missing aliasmap location: " + levelDBpath); + } } DB levelDb = JniDBFactory.factory.open(levelDBpath, options); - InMemoryAliasMap aliasMap = new InMemoryAliasMap(levelDb, blockPoolID); + InMemoryAliasMap aliasMap = new InMemoryAliasMap(levelDBpath.toURI(), + levelDb, blockPoolID); aliasMap.setConf(conf); return aliasMap; } @VisibleForTesting - InMemoryAliasMap(DB levelDb, String blockPoolID) { + InMemoryAliasMap(URI aliasMapURI, DB levelDb, String blockPoolID) { + this.aliasMapURI = aliasMapURI; this.levelDb = levelDb; this.blockPoolID = blockPoolID; } @@ -208,6 +225,177 @@ public class InMemoryAliasMap implements InMemoryAliasMapProtocol, return blockOutputStream.toByteArray(); } + /** + * Transfer this aliasmap for bootstrapping standby Namenodes. The map is + * transferred as a tar.gz archive. This archive needs to be extracted on the + * standby Namenode. + * + * @param response http response. + * @param conf configuration to use. + * @param aliasMap aliasmap to transfer. + * @throws IOException + */ + public static void transferForBootstrap(HttpServletResponse response, + Configuration conf, InMemoryAliasMap aliasMap) throws IOException { + File aliasMapSnapshot = null; + File compressedAliasMap = null; + try { + aliasMapSnapshot = createSnapshot(aliasMap); + // compress the snapshot that is associated with the + // block pool id of the aliasmap. + compressedAliasMap = getCompressedAliasMap( + new File(aliasMapSnapshot, aliasMap.blockPoolID)); + try (FileInputStream fis = new FileInputStream(compressedAliasMap)) { + ImageServlet.setVerificationHeadersForGet(response, compressedAliasMap); + ImageServlet.setFileNameHeaders(response, compressedAliasMap); + // send file + DataTransferThrottler throttler = + ImageServlet.getThrottlerForBootstrapStandby(conf); + TransferFsImage.copyFileToStream(response.getOutputStream(), + compressedAliasMap, fis, throttler); + } + } finally { + // cleanup the temporary snapshot and compressed files. + StringBuilder errMessage = new StringBuilder(); + if (compressedAliasMap != null + && !FileUtil.fullyDelete(compressedAliasMap)) { + errMessage.append("Failed to fully delete compressed aliasmap ") + .append(compressedAliasMap.getAbsolutePath()).append("\n"); + } + if (aliasMapSnapshot != null && !FileUtil.fullyDelete(aliasMapSnapshot)) { + errMessage.append("Failed to fully delete the aliasmap snapshot ") + .append(aliasMapSnapshot.getAbsolutePath()).append("\n"); + } + if (errMessage.length() > 0) { + throw new IOException(errMessage.toString()); + } + } + } + + /** + * Create a new LevelDB store which is a snapshot copy of the original + * aliasmap. + * + * @param aliasMap original aliasmap. + * @return the {@link File} where the snapshot is created. + * @throws IOException + */ + static File createSnapshot(InMemoryAliasMap aliasMap) throws IOException { + File originalAliasMapDir = new File(aliasMap.aliasMapURI); + String bpid = originalAliasMapDir.getName(); + File snapshotDir = + new File(originalAliasMapDir.getParent(), SNAPSHOT_COPY_DIR); + File newLevelDBDir = new File(snapshotDir, bpid); + if (!newLevelDBDir.mkdirs()) { + throw new IOException( + "Unable to create aliasmap snapshot directory " + newLevelDBDir); + } + // get a snapshot for the original DB. + DB originalDB = aliasMap.levelDb; + try (Snapshot snapshot = originalDB.getSnapshot()) { + // create a new DB for the snapshot and copy all K,V pairs. + Options options = new Options(); + options.createIfMissing(true); + try (DB snapshotDB = JniDBFactory.factory.open(newLevelDBDir, options)) { + try (DBIterator iterator = + originalDB.iterator(new ReadOptions().snapshot(snapshot))) { + iterator.seekToFirst(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + snapshotDB.put(entry.getKey(), entry.getValue()); + } + } + } + } + + return snapshotDir; + } + + /** + * Compress the given aliasmap directory as tar.gz. + * + * @return a reference to the compressed aliasmap. + * @throws IOException + */ + private static File getCompressedAliasMap(File aliasMapDir) + throws IOException { + File outCompressedFile = new File(aliasMapDir.getParent(), TAR_NAME); + BufferedOutputStream bOut = null; + GzipCompressorOutputStream gzOut = null; + TarArchiveOutputStream tOut = null; + try { + bOut = new BufferedOutputStream(new FileOutputStream(outCompressedFile)); + gzOut = new GzipCompressorOutputStream(bOut); + tOut = new TarArchiveOutputStream(gzOut); + addFileToTarGzRecursively(tOut, aliasMapDir, "", new Configuration()); + } finally { + if (tOut != null) { + tOut.finish(); + } + IOUtils.cleanupWithLogger(null, tOut, gzOut, bOut); + } + return outCompressedFile; + } + + /** + * Add all contents of the given file to the archive. + * + * @param tOut archive to use. + * @param file file to archive. + * @param prefix path prefix. + * @throws IOException + */ + private static void addFileToTarGzRecursively(TarArchiveOutputStream tOut, + File file, String prefix, Configuration conf) throws IOException { + String entryName = prefix + file.getName(); + TarArchiveEntry tarEntry = new TarArchiveEntry(file, entryName); + tOut.putArchiveEntry(tarEntry); + + LOG.debug("Adding entry {} to alias map archive", entryName); + if (file.isFile()) { + try (FileInputStream in = new FileInputStream(file)) { + IOUtils.copyBytes(in, tOut, conf, false); + } + tOut.closeArchiveEntry(); + } else { + tOut.closeArchiveEntry(); + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + // skip the LOCK file + if (!child.getName().equals("LOCK")) { + addFileToTarGzRecursively(tOut, child, entryName + "/", conf); + } + } + } + } + } + + /** + * Extract the aliasmap archive to complete the bootstrap process. This method + * has to be called after the aliasmap archive is transfered from the primary + * Namenode. + * + * @param aliasMap location of the aliasmap. + * @throws IOException + */ + public static void completeBootstrapTransfer(File aliasMap) + throws IOException { + File tarname = new File(aliasMap, TAR_NAME); + if (!tarname.exists()) { + throw new IOException( + "Aliasmap archive (" + tarname + ") does not exist"); + } + try { + FileUtil.unTar(tarname, aliasMap); + } finally { + // delete the archive. + if(!FileUtil.fullyDelete(tarname)) { + LOG.warn("Failed to fully delete aliasmap archive: " + tarname); + } + } + } + /** * CheckedFunction is akin to {@link java.util.function.Function} but * specifies an IOException. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java index 5c56736be43..cd2ed82d98e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java @@ -146,6 +146,14 @@ public class InMemoryLevelDBAliasMapServer implements InMemoryAliasMapProtocol, return conf; } + /** + * Get the {@link InMemoryAliasMap} used by this server. + * @return the inmemoryaliasmap used. + */ + public InMemoryAliasMap getAliasMap() { + return aliasMap; + } + @Override public void close() { LOG.info("Stopping InMemoryLevelDBAliasMapServer"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java index 7250cca3ad5..fcc0abfbf93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; import org.apache.hadoop.hdfs.server.common.Util; import static org.apache.hadoop.util.Time.monotonicNow; @@ -140,6 +141,16 @@ public class ImageServlet extends HttpServlet { long elapsed = monotonicNow() - start; metrics.addGetEdit(elapsed); } + } else if (parsedParams.isGetAliasMap()) { + InMemoryAliasMap aliasMap = + NameNodeHttpServer.getAliasMapFromContext(context); + long start = monotonicNow(); + InMemoryAliasMap.transferForBootstrap(response, conf, aliasMap); + // Metrics non-null only when used inside name node + if (metrics != null) { + long elapsed = monotonicNow() - start; + metrics.addGetAliasMap(elapsed); + } } return null; } @@ -230,7 +241,7 @@ public class ImageServlet extends HttpServlet { return throttler; } - private static DataTransferThrottler getThrottlerForBootstrapStandby( + public static DataTransferThrottler getThrottlerForBootstrapStandby( Configuration conf) { long transferBandwidth = conf.getLongBytes( @@ -337,6 +348,11 @@ public class ImageServlet extends HttpServlet { remoteStorageInfo.toColonSeparatedString(); } + static String getParamStringForAliasMap( + boolean isBootstrapStandby) { + return "getaliasmap=1&" + IS_BOOTSTRAP_STANDBY + "=" + isBootstrapStandby; + } + static class GetImageParams { private boolean isGetImage; private boolean isGetEdit; @@ -345,6 +361,7 @@ public class ImageServlet extends HttpServlet { private String storageInfoString; private boolean fetchLatest; private boolean isBootstrapStandby; + private boolean isGetAliasMap; /** * @param request the object from which this servlet reads the url contents @@ -385,10 +402,16 @@ public class ImageServlet extends HttpServlet { endTxId = ServletUtil.parseLongParam(request, END_TXID_PARAM); } else if (key.equals(STORAGEINFO_PARAM)) { storageInfoString = val[0]; + } else if (key.equals("getaliasmap")) { + isGetAliasMap = true; + String bootstrapStandby = ServletUtil.getParameter(request, + IS_BOOTSTRAP_STANDBY); + isBootstrapStandby = bootstrapStandby != null && + Boolean.parseBoolean(bootstrapStandby); } } - int numGets = (isGetImage?1:0) + (isGetEdit?1:0); + int numGets = (isGetImage?1:0) + (isGetEdit?1:0) + (isGetAliasMap?1:0); if ((numGets > 1) || (numGets == 0)) { throw new IOException("Illegal parameters to TransferFsImage"); } @@ -429,7 +452,10 @@ public class ImageServlet extends HttpServlet { boolean shouldFetchLatest() { return fetchLatest; } - + + boolean isGetAliasMap() { + return isGetAliasMap; + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index b91e7ba6bd4..471abac4f5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -713,12 +713,20 @@ public class NameNode extends ReconfigurableBase implements if (NamenodeRole.NAMENODE == role) { httpServer.setNameNodeAddress(getNameNodeAddress()); httpServer.setFSImage(getFSImage()); + if (levelDBAliasMapServer != null) { + httpServer.setAliasMap(levelDBAliasMapServer.getAliasMap()); + } } startCommonServices(conf); startMetricsLogger(conf); } + @VisibleForTesting + public InMemoryLevelDBAliasMapServer getAliasMapServer() { + return levelDBAliasMapServer; + } + private void startAliasMapServerIfNecessary(Configuration conf) throws IOException { if (conf.getBoolean(DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED, @@ -795,6 +803,9 @@ public class NameNode extends ReconfigurableBase implements startHttpServer(conf); httpServer.setNameNodeAddress(getNameNodeAddress()); httpServer.setFSImage(getFSImage()); + if (levelDBAliasMapServer != null) { + httpServer.setAliasMap(levelDBAliasMapServer.getAliasMap()); + } } rpcServer.start(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index 1bc43b896ae..4d27bd9261f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; @@ -68,6 +69,7 @@ public class NameNodeHttpServer { public static final String FSIMAGE_ATTRIBUTE_KEY = "name.system.image"; protected static final String NAMENODE_ATTRIBUTE_KEY = "name.node"; public static final String STARTUP_PROGRESS_ATTRIBUTE_KEY = "startup.progress"; + public static final String ALIASMAP_ATTRIBUTE_KEY = "name.system.aliasmap"; NameNodeHttpServer(Configuration conf, NameNode nn, InetSocketAddress bindAddress) { @@ -288,6 +290,15 @@ public class NameNodeHttpServer { httpServer.setAttribute(STARTUP_PROGRESS_ATTRIBUTE_KEY, prog); } + /** + * Sets the aliasmap URI. + * + * @param aliasMap the alias map used. + */ + void setAliasMap(InMemoryAliasMap aliasMap) { + httpServer.setAttribute(ALIASMAP_ATTRIBUTE_KEY, aliasMap); + } + private static void setupServlets(HttpServer2 httpServer, Configuration conf) { httpServer.addInternalServlet("startupProgress", StartupProgressServlet.PATH_SPEC, StartupProgressServlet.class); @@ -312,6 +323,10 @@ public class NameNodeHttpServer { return (Configuration)context.getAttribute(JspHelper.CURRENT_CONF); } + static InMemoryAliasMap getAliasMapFromContext(ServletContext context) { + return (InMemoryAliasMap) context.getAttribute(ALIASMAP_ATTRIBUTE_KEY); + } + public static InetSocketAddress getNameNodeAddressFromContext( ServletContext context) { return (InetSocketAddress)context.getAttribute( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java index c1270c565a3..82cdcfe69d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java @@ -25,6 +25,7 @@ import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URISyntaxException; import java.net.URL; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -32,6 +33,7 @@ import java.util.Map.Entry; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -192,7 +194,24 @@ public class TransferFsImage { } } } - + + /** + * Download the InMemoryAliasMap from the remote NN. + * @param fsName http address of remote NN. + * @param aliasMap location of the alias map. + * @param isBootstrapStandby flag to indicate if for bootstrap of standby. + * @throws IOException + */ + public static void downloadAliasMap(URL fsName, File aliasMap, + boolean isBootstrapStandby) throws IOException { + String paramString = ImageServlet.getParamStringForAliasMap( + isBootstrapStandby); + getFileClient(fsName, paramString, Arrays.asList(aliasMap), null, false); + LOG.info("Downloaded file " + aliasMap.getName() + " size " + + aliasMap.length() + " bytes."); + InMemoryAliasMap.completeBootstrapTransfer(aliasMap); + } + /** * Requests that the NameNode download an image from this node. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java index aed90a5300b..8677b588aa7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java @@ -20,23 +20,27 @@ package org.apache.hadoop.hdfs.server.namenode.ha; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; +import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; import java.security.PrivilegedAction; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.HAUtil; @@ -91,6 +95,9 @@ public class BootstrapStandby implements Tool, Configurable { private boolean interactive = true; private boolean skipSharedEditsCheck = false; + private boolean inMemoryAliasMapEnabled; + private String aliasMapPath; + // Exit/return codes. static final int ERR_CODE_FAILED_CONNECT = 2; static final int ERR_CODE_INVALID_VERSION = 3; @@ -235,6 +242,12 @@ public class BootstrapStandby implements Tool, Configurable { if (!isUpgradeFinalized) { doUpgrade(storage); } + + if (inMemoryAliasMapEnabled) { + return formatAndDownloadAliasMap(aliasMapPath, proxyInfo); + } else { + LOG.info("Skipping InMemoryAliasMap bootstrap as it was not configured"); + } return 0; } @@ -392,7 +405,7 @@ public class BootstrapStandby implements Tool, Configurable { private boolean checkLayoutVersion(NamespaceInfo nsInfo) throws IOException { return (nsInfo.getLayoutVersion() == HdfsServerConstants.NAMENODE_LAYOUT_VERSION); } - + private void parseConfAndFindOtherNN() throws IOException { Configuration conf = getConf(); nsId = DFSUtil.getNamenodeNameServiceId(conf); @@ -433,6 +446,82 @@ public class BootstrapStandby implements Tool, Configurable { editUrisToFormat = FSNamesystem.getNamespaceEditsDirs( conf, false); sharedEditsUris = FSNamesystem.getSharedEditsDirs(conf); + + parseProvidedConfigurations(conf); + } + + private void parseProvidedConfigurations(Configuration configuration) + throws IOException { + // if provided and in-memory aliasmap are enabled, + // get the aliasmap location. + boolean providedEnabled = configuration.getBoolean( + DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED, + DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED_DEFAULT); + boolean inmemoryAliasmapConfigured = configuration.getBoolean( + DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_ENABLED, + DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_ENABLED_DEFAULT); + if (providedEnabled && inmemoryAliasmapConfigured) { + inMemoryAliasMapEnabled = true; + aliasMapPath = configuration.get( + DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR); + } else { + inMemoryAliasMapEnabled = false; + aliasMapPath = null; + } + } + + /** + * A storage directory for aliasmaps. This is primarily used for the + * StorageDirectory#hasSomeData for formatting aliasmap directories. + */ + private static class AliasMapStorageDirectory extends StorageDirectory { + + AliasMapStorageDirectory(File aliasMapDir) { + super(aliasMapDir); + } + + @Override + public String toString() { + return "AliasMap directory = " + this.getRoot(); + } + } + + /** + * Format, if needed, and download the aliasmap. + * @param pathAliasMap the path where the aliasmap should be downloaded. + * @param proxyInfo remote namenode to get the aliasmap from. + * @return 0 on a successful transfer, and error code otherwise. + * @throws IOException + */ + private int formatAndDownloadAliasMap(String pathAliasMap, + RemoteNameNodeInfo proxyInfo) throws IOException { + LOG.info("Bootstrapping the InMemoryAliasMap from " + + proxyInfo.getHttpAddress()); + if (pathAliasMap == null) { + throw new IOException("InMemoryAliasMap enabled with null location"); + } + File aliasMapFile = new File(pathAliasMap); + if (aliasMapFile.exists()) { + AliasMapStorageDirectory aliasMapSD = + new AliasMapStorageDirectory(aliasMapFile); + if (!Storage.confirmFormat( + Arrays.asList(aliasMapSD), force, interactive)) { + return ERR_CODE_ALREADY_FORMATTED; + } else { + if (!FileUtil.fullyDelete(aliasMapFile)) { + throw new IOException( + "Cannot remove current alias map: " + aliasMapFile); + } + } + } + + // create the aliasmap location. + if (!aliasMapFile.mkdirs()) { + throw new IOException("Cannot create directory " + aliasMapFile); + } + TransferFsImage.downloadAliasMap(proxyInfo.getHttpAddress(), aliasMapFile, + true); + return 0; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java index 5d43ebf95fe..c15cdbdd48e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java @@ -151,6 +151,8 @@ public class NameNodeMetrics { MutableRate getEdit; @Metric("GetImageServlet getImage") MutableRate getImage; + @Metric("GetImageServlet getAliasMap") + MutableRate getAliasMap; @Metric("GetImageServlet putImage") MutableRate putImage; @@ -394,6 +396,10 @@ public class NameNodeMetrics { getImage.add(latency); } + public void addGetAliasMap(long latency) { + getAliasMap.add(latency); + } + public void addPutImage(long latency) { putImage.add(latency); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index a579c569123..d04267434ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -67,6 +67,8 @@ import java.util.concurrent.TimeoutException; import com.google.common.base.Supplier; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; +import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap; +import org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.InMemoryLevelDBAliasMapClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -3312,6 +3314,27 @@ public class MiniDFSCluster implements AutoCloseable { } } + /** + * Setup the namenode-level PROVIDED configurations, using the + * {@link InMemoryLevelDBAliasMapClient}. + * + * @param conf Configuration, which is modified, to enable provided storage. + * This cannot be null. + */ + public static void setupNamenodeProvidedConfiguration(Configuration conf) { + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED, true); + conf.setBoolean(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_ENABLED, true); + conf.setClass(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_CLASS, + InMemoryLevelDBAliasMapClient.class, BlockAliasMap.class); + File tempDirectory = new File(GenericTestUtils.getRandomizedTestDir(), + "in-memory-alias-map"); + conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, + tempDirectory.getAbsolutePath()); + conf.setInt(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_LOAD_RETRIES, 10); + conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_LEVELDB_PATH, + tempDirectory.getAbsolutePath()); + } + @Override public void close() { shutdown(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java index 432aefbc73e..2785f68409d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java @@ -125,5 +125,35 @@ public class ITestInMemoryAliasMap { // no more results expected assertFalse(list.getNextBlock().isPresent()); } + + @Test + public void testSnapshot() throws Exception { + Block block1 = new Block(100); + Block block2 = new Block(200); + Path path = new Path("users", "alice"); + ProvidedStorageLocation remoteLocation = + new ProvidedStorageLocation(path, 0, 1000, new byte[0]); + // write the first block + aliasMap.write(block1, remoteLocation); + // create snapshot + File snapshotFile = InMemoryAliasMap.createSnapshot(aliasMap); + // write the 2nd block after the snapshot + aliasMap.write(block2, remoteLocation); + // creata a new aliasmap object from the snapshot + InMemoryAliasMap snapshotAliasMap = null; + Configuration newConf = new Configuration(); + newConf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, + snapshotFile.getAbsolutePath()); + try { + snapshotAliasMap = InMemoryAliasMap.init(newConf, bpid); + // now the snapshot should have the first block but not the second one. + assertTrue(snapshotAliasMap.read(block1).isPresent()); + assertFalse(snapshotAliasMap.read(block2).isPresent()); + } finally { + if (snapshotAliasMap != null) { + snapshotAliasMap.close(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java deleted file mode 100644 index 0f57247cec2..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.aliasmap; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.junit.Test; - -import java.io.IOException; - -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; - -/** - * TestInMemoryAliasMap tests the initialization of an AliasMap. Most of the - * rest of the tests are in ITestInMemoryAliasMap since the tests are not - * thread safe (there is competition for the port). - */ -public class TestInMemoryAliasMap { - - @Test - public void testInit() { - String nonExistingDirectory = "non-existing-directory"; - Configuration conf = new Configuration(); - conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, - nonExistingDirectory); - - assertThatExceptionOfType(IOException.class) - .isThrownBy(() -> InMemoryAliasMap.init(conf, "bpid")).withMessage( - InMemoryAliasMap.createPathErrorMessage(nonExistingDirectory)); - } -} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java index fccb6f2ab38..1e42a8844f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java @@ -357,15 +357,6 @@ public class TestInMemoryLevelDBAliasMapClient { writeRead(); } - @Test - public void testNonExistentFile() throws Exception { - // delete alias map location - FileUtils.deleteDirectory(tempDir); - // expect a RuntimeException when the aliasmap is started. - exception.expect(RuntimeException.class); - levelDBAliasMapServer.setConf(conf); - } - @Test public void testNonExistentBlock() throws Exception { inMemoryLevelDBAliasMapClient.setConf(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java new file mode 100644 index 00000000000..715622645ea --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.ha; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.ProvidedStorageLocation; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.TransferFsImage; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.net.URL; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Test for aliasmap bootstrap. + */ +public class TestBootstrapAliasmap { + + private MiniDFSCluster cluster; + + @Before + public void setup() throws Exception { + Configuration conf = new Configuration(); + MiniDFSCluster.setupNamenodeProvidedConfiguration(conf); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(1) + .build(); + cluster.waitActive(); + } + + @Test + public void testAliasmapBootstrap() throws Exception { + InMemoryLevelDBAliasMapServer aliasMapServer = + cluster.getNameNode().getAliasMapServer(); + // write some blocks to the aliasmap. + Block block1 = new Block(1000), block2 = new Block(1002); + Path path = new Path("/test1.dat"); + aliasMapServer.write(new Block(block1), + new ProvidedStorageLocation(path, 0, 100, new byte[0])); + aliasMapServer.write(new Block(block2), + new ProvidedStorageLocation(path, 101, 200, new byte[0])); + + File newLocation = GenericTestUtils.getRandomizedTestDir(); + NameNode nn = cluster.getNameNode(); + Configuration conf = cluster.getConfiguration(0); + String scheme = DFSUtil.getHttpClientScheme(conf); + URL nnHttpURL = DFSUtil.getInfoServerWithDefaultHost( + nn.getNameNodeAddress().getHostName(), conf, scheme).toURL(); + // transfer the aliasmap. + newLocation.mkdirs(); + TransferFsImage.downloadAliasMap(nnHttpURL, newLocation, true); + + // create config for new aliasmap server at the new location. + Configuration newConfig = new Configuration(); + newConfig.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, + newLocation.getAbsolutePath()); + newConfig.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, + "127.0.0.1:" + NetUtils.getFreeSocketPort()); + String blockPoolId = nn.getNamesystem().getBlockPoolId(); + InMemoryLevelDBAliasMapServer newServer = + new InMemoryLevelDBAliasMapServer(InMemoryAliasMap::init, blockPoolId); + newServer.setConf(newConfig); + newServer.start(); + // the server should have only 2 blocks. + assertEquals(2, newServer.list(Optional.empty()).getFileRegions().size()); + assertNotNull(newServer.read(block1)); + assertNotNull(newServer.read(block2)); + assertEquals(blockPoolId, newServer.getBlockPoolId()); + } +} diff --git a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java index e3f4decd59a..f822cc01234 100644 --- a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java +++ b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java @@ -36,6 +36,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Random; import java.util.Set; @@ -55,9 +56,12 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMapProtocol; +import org.apache.hadoop.hdfs.server.aliasmap.InMemoryLevelDBAliasMapServer; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; @@ -66,6 +70,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.blockmanagement.ProvidedStorageMap; +import org.apache.hadoop.hdfs.server.common.FileRegion; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.blockaliasmap.BlockAliasMap; import org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.InMemoryLevelDBAliasMapClient; @@ -78,6 +83,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; +import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetUtils; @@ -95,6 +101,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMOR import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_BIND_HOST; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PROVIDED_ALIASMAP_LEVELDB_PATH; +import static org.apache.hadoop.hdfs.MiniDFSCluster.HDFS_MINIDFS_BASEDIR; import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI; import static org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.TextFileRegionAliasMap.fileNameFromBlockPoolID; import static org.apache.hadoop.net.NodeBase.PATH_SEPARATOR_STR; @@ -262,9 +269,9 @@ public class ITestProvidedImplementation { if ((topo.isHA() || topo.isFederated()) && !doFormat) { builder.manageNameDfsDirs(true); builder.enableManagedDfsDirsRedundancy(false); - builder.manageNameDfsSharedDirs(false); + builder.manageNameDfsSharedDirs(true); List nnDirs = - getProvidedNamenodeDirs(MiniDFSCluster.getBaseDirectory(), topo); + getProvidedNamenodeDirs(conf.get(HDFS_MINIDFS_BASEDIR), topo); for (File nnDir : nnDirs) { MiniDFSCluster.copyNameDirs( Collections.singletonList(nspath.toUri()), @@ -932,7 +939,7 @@ public class ITestProvidedImplementation { // configure the InMemoryAliasMp. conf.setBoolean(DFS_PROVIDED_ALIASMAP_INMEMORY_ENABLED, true); String directory = conf.get(DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR); - if (directory == null) { + if (directory == null || !new File(directory).exists()) { throw new IllegalArgumentException("In-memory alias map configured" + "with the proper location; Set " + DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR); @@ -948,6 +955,9 @@ public class ITestProvidedImplementation { conf.set(DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, new File(new Path(nnDir, dirName).toUri()).getAbsolutePath()); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_PROVIDED_ENABLED, true); + + // format the shared edits dir with the proper VERSION file. + NameNode.initializeSharedEdits(conf); } else { if (!completedNNs.contains(nnIndex)) { // format the NN directories for non-provided namespaces @@ -1197,4 +1207,127 @@ public class ITestProvidedImplementation { cluster.shutdown(); } } + + @Test + public void testBootstrapAliasMap() throws Exception { + int numNamenodes = 3; + MiniDFSNNTopology topology = + MiniDFSNNTopology.simpleHATopology(numNamenodes); + + createInMemoryAliasMapImage(); + conf.setBoolean(DFS_PROVIDED_ALIASMAP_INMEMORY_ENABLED, true); + conf.setInt(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_LOAD_RETRIES, 10); + providedNameservice = topology.getNameservices().get(0).getId(); + // configure the AliasMap addresses + configureAliasMapAddresses(topology, providedNameservice); + startCluster(nnDirPath, 2, + new StorageType[] {StorageType.PROVIDED, StorageType.DISK}, null, + false, null, topology, new MiniDFSClusterBuilderAliasMap(conf)); + + // make NN with index 0 the active, shutdown and delete the directories + // of others. This will delete the aliasmap on these namenodes as well. + cluster.transitionToActive(0); + verifyFileSystemContents(0); + for (int nnIndex = 1; nnIndex < numNamenodes; nnIndex++) { + cluster.shutdownNameNode(nnIndex); + // delete the namenode directories including alias map. + for (URI u : cluster.getNameDirs(nnIndex)) { + File dir = new File(u.getPath()); + assertTrue(FileUtil.fullyDelete(dir)); + } + } + + // start the other namenodes and bootstrap them + for (int index = 1; index < numNamenodes; index++) { + // add some content to aliasmap dir + File aliasMapDir = new File(fBASE, "aliasmap-" + index); + // create a directory inside aliasMapDir + if (!new File(aliasMapDir, "tempDir").mkdirs()) { + throw new IOException("Unable to create directory " + aliasMapDir); + } + Configuration currNNConf = cluster.getConfiguration(index); + currNNConf.set(DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, + aliasMapDir.getAbsolutePath()); + // without force this should fail as aliasmap is not empty. + int rc = + BootstrapStandby.run(new String[] {"-nonInteractive"}, currNNConf); + assertNotEquals(0, rc); + // force deletes the contents of the aliasmap. + rc = BootstrapStandby.run(new String[] {"-nonInteractive", "-force"}, + currNNConf); + assertEquals(0, rc); + } + + // check if aliasmap files are the same on all NNs + checkInMemoryAliasMapContents(0, numNamenodes); + // restart the killed namenodes. + for (int i = 1; i < numNamenodes; i++) { + cluster.restartNameNode(i, false); + } + + cluster.waitClusterUp(); + cluster.waitActive(); + + // transition to namenode 1 as the active + int nextNN = 1; + cluster.shutdownNameNode(0); + cluster.transitionToActive(nextNN); + // all files must be accessible from nextNN. + verifyFileSystemContents(nextNN); + } + + /** + * Check if the alias map contents of the namenodes are the same as the base. + * + * @param baseNN index of the namenode to compare against. + * @param numNamenodes total number of namenodes in the cluster. + */ + private void checkInMemoryAliasMapContents(int baseNN, int numNamenodes) + throws Exception { + InMemoryLevelDBAliasMapServer baseAliasMap = + cluster.getNameNode(baseNN).getAliasMapServer(); + for (int i = 0; i < numNamenodes; i++) { + if (baseNN == i) { + continue; + } + InMemoryLevelDBAliasMapServer otherAliasMap = + cluster.getNameNode(baseNN).getAliasMapServer(); + verifyAliasMapEquals(baseAliasMap, otherAliasMap); + } + } + + /** + * Verify that the contents of the aliasmaps are the same. + * + * @param aliasMap1 + * @param aliasMap2 + */ + private void verifyAliasMapEquals(InMemoryLevelDBAliasMapServer aliasMap1, + InMemoryLevelDBAliasMapServer aliasMap2) throws Exception { + Set fileRegions1 = getFileRegions(aliasMap1); + Set fileRegions2 = getFileRegions(aliasMap2); + assertTrue(fileRegions1.equals(fileRegions2)); + } + + /** + * Get all the aliases the aliasmap contains. + * + * @param aliasMap aliasmap to explore. + * @return set of all aliases. + * @throws IOException + */ + private Set getFileRegions(InMemoryLevelDBAliasMapServer aliasMap) + throws IOException { + Set fileRegions = new HashSet<>(); + Optional marker = Optional.empty(); + while (true) { + InMemoryAliasMapProtocol.IterationResult result = aliasMap.list(marker); + fileRegions.addAll(result.getFileRegions()); + marker = result.getNextBlock(); + if (!marker.isPresent()) { + break; + } + } + return fileRegions; + } }