From d91c68729c18a2f853beb66b506ef9dff3526a38 Mon Sep 17 00:00:00 2001 From: He Xiaoqiao Date: Thu, 30 May 2019 13:27:48 -0700 Subject: [PATCH] HDFS-14497. Write lock held by metasave impact following RPC processing. Contributed by He Xiaoqiao. Signed-off-by: Wei-Chiu Chuang (cherry picked from commit 33c62f8f4e94442825fe286c2b18518925d980e6) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (cherry picked from commit 80392e94b6dca16229fc35426d107184be68c908) --- .../server/blockmanagement/BlockManager.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 28 ++++++--- .../hdfs/server/namenode/TestMetaSave.java | 60 +++++++++++++++++++ 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 4e450e2d5fc..321f44e9538 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -726,7 +726,7 @@ public class BlockManager implements BlockStatsMXBean { /** Dump meta data to out. */ public void metaSave(PrintWriter out) { - assert namesystem.hasWriteLock(); // TODO: block manager read lock and NS write lock + assert namesystem.hasReadLock(); // TODO: block manager read lock and NS write lock final List live = new ArrayList(); final List dead = new ArrayList(); datanodeManager.fetchDatanodes(live, dead, false); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 7ebc8daef3f..108cfbce8c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -122,6 +122,7 @@ import java.lang.management.ManagementFactory; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -587,6 +588,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, private boolean resourceLowSafeMode = false; private String nameNodeHostName = null; + /** + * HDFS-14497: Concurrency control when many metaSave request to write + * meta to same out stream after switch to read lock. + */ + private Object metaSaveLock = new Object(); + /** * Notify that loading of this FSDirectory is complete, and * it is imageLoaded for use @@ -1757,23 +1764,26 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, String operationName = "metaSave"; checkSuperuserPrivilege(operationName); checkOperation(OperationCategory.READ); - writeLock(); + readLock(); try { checkOperation(OperationCategory.READ); - File file = new File(System.getProperty("hadoop.log.dir"), filename); - PrintWriter out = new PrintWriter(new BufferedWriter( - new OutputStreamWriter(new FileOutputStream(file), Charsets.UTF_8))); - metaSave(out); - out.flush(); - out.close(); + synchronized(metaSaveLock) { + File file = new File(System.getProperty("hadoop.log.dir"), filename); + PrintWriter out = new PrintWriter(new BufferedWriter( + new OutputStreamWriter(Files.newOutputStream(file.toPath()), + Charsets.UTF_8))); + metaSave(out); + out.flush(); + out.close(); + } } finally { - writeUnlock(operationName); + readUnlock(operationName); } logAuditEvent(true, operationName, null); } private void metaSave(PrintWriter out) { - assert hasWriteLock(); + assert hasReadLock(); long totalInodes = this.dir.totalInodes(); long totalBlocks = this.getBlocksTotal(); out.println(totalInodes + " files and directories, " + totalBlocks diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java index 8cc1433cef5..d4748f3d601 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.util.ArrayList; import java.util.concurrent.TimeoutException; import com.google.common.base.Supplier; @@ -215,6 +216,65 @@ public class TestMetaSave { } } + class MetaSaveThread extends Thread { + NamenodeProtocols nnRpc; + String filename; + public MetaSaveThread(NamenodeProtocols nnRpc, String filename) { + this.nnRpc = nnRpc; + this.filename = filename; + } + + @Override + public void run() { + try { + nnRpc.metaSave(filename); + } catch (IOException e) { + } + } + } + + /** + * Tests that metasave concurrent output file (not append). + */ + @Test + public void testConcurrentMetaSave() throws Exception { + ArrayList threads = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + threads.add(new MetaSaveThread(nnRpc, "metaSaveConcurrent.out.txt")); + } + for (int i = 0; i < 10; i++) { + threads.get(i).start(); + } + for (int i = 0; i < 10; i++) { + threads.get(i).join(); + } + // Read output file. + FileInputStream fis = null; + InputStreamReader isr = null; + BufferedReader rdr = null; + try { + fis = new FileInputStream(getLogFile("metaSaveConcurrent.out.txt")); + isr = new InputStreamReader(fis); + rdr = new BufferedReader(isr); + + // Validate that file was overwritten (not appended) by checking for + // presence of only one "Live Datanodes" line. + boolean foundLiveDatanodesLine = false; + String line = rdr.readLine(); + while (line != null) { + if (line.startsWith("Live Datanodes")) { + if (foundLiveDatanodesLine) { + fail("multiple Live Datanodes lines, output file not overwritten"); + } + foundLiveDatanodesLine = true; + } + line = rdr.readLine(); + } + } finally { + IOUtils.cleanup(null, rdr, isr, fis); + } + } + @After public void tearDown() throws IOException { if (fileSys != null)