Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-16406. ReadsFromLocalClient counts short-circuit reads (#3847)

Browse Source
This commit is contained in:
secfree 2022-02-09 10:07:05 +08:00 committed by GitHub
parent 3e7a7c3c4a
commit bf0cefb0d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 41 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
View File

@ -415,6 +415,9 @@ public void requestShortCircuitFds(final ExtendedBlock blk,
"Not verifying {}", slotId); "Not verifying {}", slotId);
} }
success = true; success = true;
// update metrics
datanode.metrics.addReadBlockOp(elapsed());
datanode.metrics.incrReadsFromClient(true, blk.getNumBytes());
} }
} finally { } finally {
if ((!success) && (registeredSlotId != null)) { if ((!success) && (registeredSlotId != null)) {

38
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
View File

@ -25,6 +25,7 @@
import static org.junit.Assert.*; import static org.junit.Assert.*;
import java.io.Closeable; import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
@ -38,7 +39,10 @@
import net.jcip.annotations.NotThreadSafe; import net.jcip.annotations.NotThreadSafe;
import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.net.unix.DomainSocket;
import org.apache.hadoop.net.unix.TemporarySocketDirectory;
import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.Lists;
import org.junit.Assume;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -654,4 +658,38 @@ public void testNNRpcMetricsWithFederationAndHA() throws IOException {
assertCounter("HeartbeatsForns1-nn1NumOps", 1L, rb); assertCounter("HeartbeatsForns1-nn1NumOps", 1L, rb);
assertCounter("HeartbeatsNumOps", 4L, rb); assertCounter("HeartbeatsNumOps", 4L, rb);
} }
@Test
public void testNodeLocalMetrics() throws Exception {
Assume.assumeTrue(null == DomainSocket.getLoadingFailureReason());
Configuration conf = new HdfsConfiguration();
conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
DomainSocket.disableBindPathValidation();
conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
new File(sockDir.getDir(),
"testNodeLocalMetrics._PORT.sock").getAbsolutePath());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
try {
cluster.waitActive();
FileSystem fs = cluster.getFileSystem();
Path testFile = new Path("/testNodeLocalMetrics.txt");
DFSTestUtil.createFile(fs, testFile, 10L, (short)1, 1L);
DFSTestUtil.readFile(fs, testFile);
List<DataNode> datanodes = cluster.getDataNodes();
assertEquals(1, datanodes.size());
DataNode datanode = datanodes.get(0);
MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
// Write related metrics
assertCounter("WritesFromLocalClient", 1L, rb);
// Read related metrics
assertCounter("ReadsFromLocalClient", 1L, rb);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
} }