From 36fb90c93982a22df7177809c3b0ddae455ebc07 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Sat, 23 Sep 2017 10:11:00 -0700 Subject: [PATCH] HDFS-12064. Reuse object mapper in HDFS. Contributed by Hanisha Koneru. --- .../server/blockmanagement/SlowDiskTracker.java | 9 +++++++-- .../server/blockmanagement/SlowPeerTracker.java | 8 ++++++-- .../server/diskbalancer/planner/NodePlan.java | 15 ++++++++------- .../blockmanagement/TestSlowDiskTracker.java | 6 ++++-- .../blockmanagement/TestSlowPeerTracker.java | 10 ++++++---- .../org/apache/hadoop/hdfs/web/TestJsonUtil.java | 16 ++++++++-------- 6 files changed, 39 insertions(+), 25 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java index 52fce5d35c0..051121ed1ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -67,6 +68,11 @@ public class SlowDiskTracker { */ private final Timer timer; + /** + * ObjectWriter to convert JSON reports to String. + */ + private static final ObjectWriter WRITER = new ObjectMapper().writer(); + /** * Number of disks to include in JSON report per operation. We will return * disks with the highest latency. @@ -254,12 +260,11 @@ public int compare(DiskLatency o1, DiskLatency o2) { * serialization failed. */ public String getSlowDiskReportAsJsonString() { - ObjectMapper objectMapper = new ObjectMapper(); try { if (slowDisksReport.isEmpty()) { return null; } - return objectMapper.writeValueAsString(slowDisksReport); + return WRITER.writeValueAsString(slowDisksReport); } catch (JsonProcessingException e) { // Failed to serialize. Don't log the exception call stack. LOG.debug("Failed to serialize statistics" + e); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java index 7b24370d7ad..22983ea1b6a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Ints; @@ -69,6 +70,10 @@ public class SlowPeerTracker { */ private final Timer timer; + /** + * ObjectWriter to convert JSON reports to String. + */ + private static final ObjectWriter WRITER = new ObjectMapper().writer(); /** * Number of nodes to include in JSON report. We will return nodes with * the highest number of votes from peers. @@ -188,9 +193,8 @@ private SortedSet filterNodeReports( public String getJson() { Collection validReports = getJsonReports( MAX_NODES_TO_REPORT); - ObjectMapper objectMapper = new ObjectMapper(); try { - return objectMapper.writeValueAsString(validReports); + return WRITER.writeValueAsString(validReports); } catch (JsonProcessingException e) { // Failed to serialize. Don't log the exception call stack. LOG.debug("Failed to serialize statistics" + e); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java index c72bb1f4c13..44039eaa0f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/NodePlan.java @@ -18,8 +18,9 @@ package org.apache.hadoop.hdfs.server.diskbalancer.planner; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.base.Preconditions; import java.io.IOException; @@ -38,6 +39,10 @@ public class NodePlan { private int port; private long timeStamp; + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectReader READER = MAPPER.readerFor(NodePlan.class); + private static final ObjectWriter WRITER = MAPPER.writerFor( + MAPPER.constructType(NodePlan.class)); /** * returns timestamp when this plan was created. * @@ -153,8 +158,7 @@ public void setPort(int port) { * @throws IOException */ public static NodePlan parseJson(String json) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(json, NodePlan.class); + return READER.readValue(json); } /** @@ -164,10 +168,7 @@ public static NodePlan parseJson(String json) throws IOException { * @throws IOException */ public String toJson() throws IOException { - ObjectMapper mapper = new ObjectMapper(); - JavaType planType = mapper.constructType(NodePlan.class); - return mapper.writerFor(planType) - .writeValueAsString(this); + return WRITER.writeValueAsString(this); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java index 172400d46de..78cf927727c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; import com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import static org.apache.hadoop.hdfs.DFSConfigKeys @@ -76,6 +77,8 @@ public class TestSlowDiskTracker { private FakeTimer timer; private long reportValidityMs; private static final long OUTLIERS_REPORT_INTERVAL = 1000; + private static final ObjectReader READER = new ObjectMapper().readerFor( + new TypeReference>() {}); static { conf = new HdfsConfiguration(); @@ -416,8 +419,7 @@ private boolean isDiskInReports(ArrayList reports, private ArrayList getAndDeserializeJson( final String json) throws IOException { - return (new ObjectMapper()).readValue(json, - new TypeReference>() {}); + return READER.readValue(json); } private void addSlowDiskForTesting(String dnID, String disk, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowPeerTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowPeerTracker.java index 15eb3a511e7..fb2928cc486 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowPeerTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowPeerTracker.java @@ -20,9 +20,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.server.blockmanagement.SlowPeerTracker.ReportForJson; +import org.apache.hadoop.hdfs.server.blockmanagement.SlowPeerTracker + .ReportForJson; import org.apache.hadoop.util.FakeTimer; import org.junit.Before; import org.junit.Rule; @@ -39,7 +41,6 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; - /** * Tests for {@link SlowPeerTracker}. */ @@ -57,6 +58,8 @@ public class TestSlowPeerTracker { private SlowPeerTracker tracker; private FakeTimer timer; private long reportValidityMs; + private static final ObjectReader READER = + new ObjectMapper().readerFor(new TypeReference>() {}); @Before public void setup() { @@ -220,7 +223,6 @@ private Set getAndDeserializeJson() throws IOException { final String json = tracker.getJson(); LOG.info("Got JSON: {}", json); - return (new ObjectMapper()).readValue( - json, new TypeReference>() {}); + return READER.readValue(json); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java index edb79d3bb4e..8c1a549b39b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java @@ -52,6 +52,10 @@ import com.google.common.collect.Lists; public class TestJsonUtil { + + private static final ObjectReader READER = + new ObjectMapper().readerFor(Map.class); + static FileStatus toFileStatus(HdfsFileStatus f, String parent) { return new FileStatus(f.getLen(), f.isDirectory(), f.getReplication(), f.getBlockSize(), f.getModificationTime(), f.getAccessTime(), @@ -76,9 +80,8 @@ public void testHdfsFileStatus() throws IOException { System.out.println("fstatus = " + fstatus); final String json = JsonUtil.toJsonString(status, true); System.out.println("json = " + json.replace(",", ",\n ")); - ObjectReader reader = new ObjectMapper().readerFor(Map.class); final HdfsFileStatus s2 = - JsonUtilClient.toFileStatus((Map) reader.readValue(json), true); + JsonUtilClient.toFileStatus((Map) READER.readValue(json), true); final FileStatus fs2 = toFileStatus(s2, parent); System.out.println("s2 = " + s2); System.out.println("fs2 = " + fs2); @@ -164,8 +167,7 @@ public void testToDatanodeInfoWithName() throws Exception { public void testToAclStatus() throws IOException { String jsonString = "{\"AclStatus\":{\"entries\":[\"user::rwx\",\"user:user1:rw-\",\"group::rw-\",\"other::r-x\"],\"group\":\"supergroup\",\"owner\":\"testuser\",\"stickyBit\":false}}"; - ObjectReader reader = new ObjectMapper().readerFor(Map.class); - Map json = reader.readValue(jsonString); + Map json = READER.readValue(jsonString); List aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER, ALL), @@ -224,8 +226,7 @@ public void testToXAttrMap() throws IOException { String jsonString = "{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"}," + "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}"; - ObjectReader reader = new ObjectMapper().readerFor(Map.class); - Map json = reader.readValue(jsonString); + Map json = READER.readValue(jsonString); XAttr xAttr1 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER). setName("a1").setValue(XAttrCodec.decodeValue("0x313233")).build(); XAttr xAttr2 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER). @@ -250,8 +251,7 @@ public void testGetXAttrFromJson() throws IOException { String jsonString = "{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"}," + "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}"; - ObjectReader reader = new ObjectMapper().readerFor(Map.class); - Map json = reader.readValue(jsonString); + Map json = READER.readValue(jsonString); // Get xattr: user.a2 byte[] value = JsonUtilClient.getXAttr(json, "user.a2");