From b258b344bb76af6492828201959e36b45f0f75b8 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 11 Jun 2015 10:12:31 -0700 Subject: [PATCH] HDFS-8573. Move creation of restartMeta file logic from BlockReceiver to ReplicaInPipeline. Contributed by Eddy Xu. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/datanode/BlockReceiver.java | 10 +--------- .../hdfs/server/datanode/ReplicaInPipeline.java | 15 ++++++++++++++- .../datanode/ReplicaInPipelineInterface.java | 10 ++++++++++ .../hdfs/server/datanode/SimulatedFSDataset.java | 5 +++++ .../extdataset/ExternalReplicaInPipeline.java | 7 +++++++ 6 files changed, 40 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3c857734db4..7b8370e9643 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -620,6 +620,9 @@ Release 2.8.0 - UNRELEASED HDFS-8549. Abort the balancer if an upgrade is in progress. (wang) + HDFS-8573. Move creation of restartMeta file logic from BlockReceiver to + ReplicaInPipeline. (Eddy Xu via wang) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 2e1160049f9..c46892d3e2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -23,7 +23,6 @@ import java.io.BufferedOutputStream; import java.io.Closeable; import java.io.DataInputStream; import java.io.DataOutputStream; -import java.io.File; import java.io.FileDescriptor; import java.io.FileOutputStream; import java.io.IOException; @@ -839,15 +838,8 @@ class BlockReceiver implements Closeable { // In case this datanode is shutting down for quick restart, // send a special ack upstream. if (datanode.isRestarting() && isClient && !isTransfer) { - File blockFile = ((ReplicaInPipeline)replicaInfo).getBlockFile(); - File restartMeta = new File(blockFile.getParent() + - File.pathSeparator + "." + blockFile.getName() + ".restart"); - if (restartMeta.exists() && !restartMeta.delete()) { - LOG.warn("Failed to delete restart meta file: " + - restartMeta.getPath()); - } try (Writer out = new OutputStreamWriter( - new FileOutputStream(restartMeta), "UTF-8")) { + replicaInfo.createRestartMetaStream(), "UTF-8")) { // write out the current time. out.write(Long.toString(Time.now() + restartBudget)); out.flush(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java index 0eb143a4a52..d9406f0f9fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.RandomAccessFile; import org.apache.hadoop.hdfs.protocol.Block; @@ -287,7 +288,19 @@ public class ReplicaInPipeline extends ReplicaInfo throw e; } } - + + @Override + public OutputStream createRestartMetaStream() throws IOException { + File blockFile = getBlockFile(); + File restartMeta = new File(blockFile.getParent() + + File.pathSeparator + "." + blockFile.getName() + ".restart"); + if (restartMeta.exists() && !restartMeta.delete()) { + DataNode.LOG.warn("Failed to delete restart meta file: " + + restartMeta.getPath()); + } + return new FileOutputStream(restartMeta); + } + @Override public String toString() { return super.toString() diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java index 0263d0f9f3f..ef9f3e2e2cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipelineInterface.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.IOException; +import java.io.OutputStream; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; import org.apache.hadoop.util.DataChecksum; @@ -73,4 +74,13 @@ public interface ReplicaInPipelineInterface extends Replica { */ public ReplicaOutputStreams createStreams(boolean isCreate, DataChecksum requestedChecksum) throws IOException; + + /** + * Create an output stream to write restart metadata in case of datanode + * shutting down for quick restart. + * + * @return output stream for writing. + * @throws IOException if any error occurs + */ + public OutputStream createRestartMetaStream() throws IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 778dd2804c1..6e5f07ca0e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -262,6 +262,11 @@ public class SimulatedFSDataset implements FsDatasetSpi { } } + @Override + public OutputStream createRestartMetaStream() throws IOException { + return new SimulatedOutputStream(); + } + @Override synchronized public long getBlockId() { return theBlock.getBlockId(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java index ad44500e502..a0039bcdbe9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalReplicaInPipeline.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.datanode.extdataset; import java.io.IOException; +import java.io.OutputStream; + import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.ChunkChecksum; import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface; @@ -59,6 +61,11 @@ public class ExternalReplicaInPipeline implements ReplicaInPipelineInterface { return new ReplicaOutputStreams(null, null, requestedChecksum, false); } + @Override + public OutputStream createRestartMetaStream() throws IOException { + return null; + } + @Override public long getBlockId() { return 0;