From ab7e7dfd7e557ff17f177ae445c27133d81e8b1a Mon Sep 17 00:00:00 2001 From: Jitendra Nath Pandey Date: Mon, 17 Oct 2011 20:42:32 +0000 Subject: [PATCH] Merged r1185354 from trunk for HDFS-2188. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1185363 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/namenode/FSEditLog.java | 58 +++++++++++++++++-- .../hadoop/hdfs/server/namenode/FSImage.java | 5 +- .../hdfs/server/namenode/NNStorage.java | 26 ++++++++- .../hdfs/server/namenode/FSImageTestUtil.java | 20 +++++-- .../hdfs/server/namenode/TestEditLog.java | 20 +++++-- 6 files changed, 112 insertions(+), 20 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index be5f7a3f3ce..b9bc4a22ab0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -97,6 +97,9 @@ Trunk (unreleased changes) HDFS-2298. Fix TestDfsOverAvroRpc by changing ClientProtocol to not include multiple methods of the same name. (cutting) + HDFS-2188. Make FSEditLog create its journals from a list of URIs rather + than NNStorage. (Ivan Kelly via jitendra) + Release 0.23.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index 4a41a2cbd65..aac2a35592e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -18,10 +18,11 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.hdfs.server.common.Util.now; - +import java.net.URI; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.apache.commons.logging.Log; @@ -42,9 +43,11 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.conf.Configuration; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; /** * FSEditLog maintains a log of the namespace modifications. @@ -122,23 +125,68 @@ public class FSEditLog { } }; + final private Collection editsDirs; + + /** + * Construct FSEditLog with default configuration, taking editDirs from NNStorage + * @param storage Storage object used by namenode + */ + @VisibleForTesting FSEditLog(NNStorage storage) { + this(new Configuration(), storage, Collections.emptyList()); + } + + /** + * Constructor for FSEditLog. Add underlying journals are constructed, but + * no streams are opened until open() is called. + * + * @param conf The namenode configuration + * @param storage Storage object used by namenode + * @param editsDirs List of journals to use + */ + FSEditLog(Configuration conf, NNStorage storage, Collection editsDirs) { isSyncRunning = false; this.storage = storage; metrics = NameNode.getNameNodeMetrics(); lastPrintTime = now(); + + if (editsDirs.isEmpty()) { + // if this is the case, no edit dirs have been explictly configured + // image dirs are to be used for edits too + try { + editsDirs = Lists.newArrayList(storage.getEditsDirectories()); + } catch (IOException ioe) { + // cannot get list from storage, so the empty editsDirs + // will be assigned. an error will be thrown on first use + // of the editlog, as no journals will exist + } + this.editsDirs = editsDirs; + } else { + this.editsDirs = Lists.newArrayList(editsDirs); + } this.journalSet = new JournalSet(); - for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) { - journalSet.add(new FileJournalManager(sd)); + for (URI u : this.editsDirs) { + StorageDirectory sd = storage.getStorageDirectory(u); + if (sd != null) { + journalSet.add(new FileJournalManager(sd)); + } } - + if (journalSet.isEmpty()) { LOG.error("No edits directories configured!"); } state = State.BETWEEN_LOG_SEGMENTS; } - + + /** + * Get the list of URIs the editlog is using for storage + * @return collection of URIs in use by the edit log + */ + Collection getEditURIs() { + return editsDirs; + } + /** * Initialize the output stream for logging, opening the first * log segment. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 325e4b04ca9..a9429417134 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -120,7 +120,7 @@ public class FSImage implements Closeable { storage.setRestoreFailedStorage(true); } - this.editLog = new FSEditLog(storage); + this.editLog = new FSEditLog(conf, storage, editsDirs); archivalManager = new NNStorageRetentionManager(conf, storage, editLog); } @@ -150,8 +150,7 @@ public class FSImage implements Closeable { "NameNode formatting should be performed before reading the image"; Collection imageDirs = storage.getImageDirectories(); - Collection editsDirs = storage.getEditsDirectories(); - + Collection editsDirs = editLog.getEditURIs(); // none of the data dirs exist if((imageDirs.size() == 0 || editsDirs.size() == 0) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java index 82096252ac4..a7fa7fb4252 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java @@ -59,6 +59,7 @@ import org.apache.hadoop.net.DNS; import com.google.common.base.Preconditions; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; /** * NNStorage is responsible for management of the StorageDirectories used by @@ -154,7 +155,9 @@ public class NNStorage extends Storage implements Closeable { storageDirs = new CopyOnWriteArrayList(); - setStorageDirectories(imageDirs, editsDirs); + // this may modify the editsDirs, so copy before passing in + setStorageDirectories(imageDirs, + Lists.newArrayList(editsDirs)); } @Override // Storage @@ -298,6 +301,27 @@ public class NNStorage extends Storage implements Closeable { } } + /** + * Return the storage directory corresponding to the passed URI + * @param uri URI of a storage directory + * @return The matching storage directory or null if none found + */ + StorageDirectory getStorageDirectory(URI uri) { + try { + uri = Util.fileAsURI(new File(uri)); + Iterator it = dirIterator(); + for (; it.hasNext(); ) { + StorageDirectory sd = it.next(); + if (Util.fileAsURI(sd.getRoot()).equals(uri)) { + return sd; + } + } + } catch (IOException ioe) { + LOG.warn("Error converting file to URI", ioe); + } + return null; + } + /** * Checks the consistency of a URI, in particular if the scheme * is specified and is supported by a concrete implementation diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java index 39e7db17dc7..aec77ab12ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java @@ -41,10 +41,13 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFil import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.util.MD5FileUtils; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.conf.Configuration; import org.mockito.Mockito; +import org.mockito.Matchers; import com.google.common.base.Joiner; import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.Files; @@ -105,7 +108,7 @@ public abstract class FSImageTestUtil { Mockito.doReturn(type) .when(sd).getStorageDirType(); Mockito.doReturn(currentDir).when(sd).getCurrentDir(); - + Mockito.doReturn(currentDir).when(sd).getRoot(); Mockito.doReturn(mockFile(true)).when(sd).getVersionFile(); Mockito.doReturn(mockFile(false)).when(sd).getPreviousDir(); return sd; @@ -127,7 +130,8 @@ public abstract class FSImageTestUtil { // Version file should always exist doReturn(mockFile(true)).when(sd).getVersionFile(); - + doReturn(mockFile(true)).when(sd).getRoot(); + // Previous dir optionally exists doReturn(mockFile(previousExists)) .when(sd).getPreviousDir(); @@ -142,6 +146,7 @@ public abstract class FSImageTestUtil { doReturn(files).when(mockDir).listFiles(); doReturn(mockDir).when(sd).getCurrentDir(); + return sd; } @@ -169,11 +174,16 @@ public abstract class FSImageTestUtil { assertTrue(logDir.mkdirs() || logDir.exists()); Files.deleteDirectoryContents(logDir); NNStorage storage = Mockito.mock(NNStorage.class); - List sds = Lists.newArrayList( - FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS)); + StorageDirectory sd + = FSImageTestUtil.mockStorageDirectory(logDir, NameNodeDirType.EDITS); + List sds = Lists.newArrayList(sd); Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS); + Mockito.doReturn(sd).when(storage) + .getStorageDirectory(Matchers.anyObject()); - return new FSEditLog(storage); + return new FSEditLog(new Configuration(), + storage, + ImmutableList.of(logDir.toURI())); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java index 123810c9dc4..5c14ab3061e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; @@ -861,8 +862,11 @@ public class TestEditLog extends TestCase { * The syntax [1,] specifies an in-progress log starting at * txid 1. */ - private NNStorage mockStorageWithEdits(String... editsDirSpecs) { + private NNStorage mockStorageWithEdits(String... editsDirSpecs) throws IOException { List sds = Lists.newArrayList(); + List uris = Lists.newArrayList(); + + NNStorage storage = Mockito.mock(NNStorage.class); for (String dirSpec : editsDirSpecs) { List files = Lists.newArrayList(); String[] logSpecs = dirSpec.split("\\|"); @@ -878,13 +882,17 @@ public class TestEditLog extends TestCase { Long.valueOf(m.group(2)))); } } - sds.add(FSImageTestUtil.mockStorageDirectory( + StorageDirectory sd = FSImageTestUtil.mockStorageDirectory( NameNodeDirType.EDITS, false, - files.toArray(new String[0]))); - } - - NNStorage storage = Mockito.mock(NNStorage.class); + files.toArray(new String[0])); + sds.add(sd); + URI u = URI.create("file:///storage"+ Math.random()); + Mockito.doReturn(sd).when(storage).getStorageDirectory(u); + uris.add(u); + } + Mockito.doReturn(sds).when(storage).dirIterable(NameNodeDirType.EDITS); + Mockito.doReturn(uris).when(storage).getEditsDirectories(); return storage; }