diff --git a/docs/content/operations/api-reference.md b/docs/content/operations/api-reference.md index 473282f19ed..3b49b12c5b1 100644 --- a/docs/content/operations/api-reference.md +++ b/docs/content/operations/api-reference.md @@ -162,7 +162,7 @@ Returns a list of datasource names found in the cluster. * `/druid/coordinator/v1/datasources?simple` -Returns a list of JSON objects containing the name and properties of datasources found in the cluster. Properties include segment count, total segment byte size, minTime, and maxTime. +Returns a list of JSON objects containing the name and properties of datasources found in the cluster. Properties include segment count, total segment byte size, replicated total segment byte size, minTime, and maxTime. * `/druid/coordinator/v1/datasources?full` @@ -170,7 +170,7 @@ Returns a list of datasource names found in the cluster with all metadata about * `/druid/coordinator/v1/datasources/{dataSourceName}` -Returns a JSON object containing the name and properties of a datasource. Properties include segment count, total segment byte size, minTime, and maxTime. +Returns a JSON object containing the name and properties of a datasource. Properties include segment count, total segment byte size, replicated total segment byte size, minTime, and maxTime. * `/druid/coordinator/v1/datasources/{dataSourceName}?full` diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 9255e7c5145..a2f04fffa68 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -538,6 +538,8 @@ public class DataSourcesResource Map> tierDistinctSegments = new HashMap<>(); long totalSegmentSize = 0; + long totalReplicatedSize = 0; + DateTime minTime = DateTimes.MAX; DateTime maxTime = DateTimes.MIN; String tier; @@ -552,6 +554,8 @@ public class DataSourcesResource tierDistinctSegments.computeIfAbsent(tier, k -> new HashSet<>()); long dataSourceSegmentSize = 0; + long replicatedSegmentSize = 0; + for (DataSegment dataSegment : druidDataSource.getSegments()) { // tier segments stats if (!tierDistinctSegments.get(tier).contains(dataSegment.getId())) { @@ -565,6 +569,8 @@ public class DataSourcesResource minTime = DateTimes.min(minTime, dataSegment.getInterval().getStart()); maxTime = DateTimes.max(maxTime, dataSegment.getInterval().getEnd()); } + totalReplicatedSize += dataSegment.getSize(); + replicatedSegmentSize += dataSegment.getSize(); } // tier stats @@ -577,10 +583,14 @@ public class DataSourcesResource long segmentSize = MapUtils.getLong(tierStats, "size", 0L); tierStats.put("size", segmentSize + dataSourceSegmentSize); + + long replicatedSize = MapUtils.getLong(tierStats, "replicatedSize", 0L); + tierStats.put("replicatedSize", replicatedSize + replicatedSegmentSize); } segments.put("count", totalDistinctSegments.size()); segments.put("size", totalSegmentSize); + segments.put("replicatedSize", totalReplicatedSize); segments.put("minTime", minTime); segments.put("maxTime", maxTime); return retVal; diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 02885ff2e03..1ae294bd195 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -373,11 +373,13 @@ public class DataSourcesResourceTest Map> result = (Map>) response.getEntity(); Assert.assertEquals(1, ((Map) (result.get("tiers").get(null))).get("segmentCount")); Assert.assertEquals(10L, ((Map) (result.get("tiers").get(null))).get("size")); + Assert.assertEquals(10L, ((Map) (result.get("tiers").get(null))).get("replicatedSize")); Assert.assertNotNull(result.get("segments")); Assert.assertEquals("2010-01-01T00:00:00.000Z", result.get("segments").get("minTime").toString()); Assert.assertEquals("2010-01-02T00:00:00.000Z", result.get("segments").get("maxTime").toString()); Assert.assertEquals(1, result.get("segments").get("count")); Assert.assertEquals(10L, result.get("segments").get("size")); + Assert.assertEquals(10L, result.get("segments").get("replicatedSize")); EasyMock.verify(inventoryView, server); } @@ -413,6 +415,7 @@ public class DataSourcesResourceTest Map> result = (Map>) response.getEntity(); Assert.assertEquals(2, ((Map) (result.get("tiers").get("cold"))).get("segmentCount")); Assert.assertEquals(30L, ((Map) (result.get("tiers").get("cold"))).get("size")); + Assert.assertEquals(30L, ((Map) (result.get("tiers").get("cold"))).get("replicatedSize")); Assert.assertEquals(1, ((Map) (result.get("tiers").get("hot"))).get("segmentCount")); Assert.assertEquals(20L, ((Map) (result.get("tiers").get("hot"))).get("size")); Assert.assertNotNull(result.get("segments")); @@ -420,9 +423,62 @@ public class DataSourcesResourceTest Assert.assertEquals("2010-01-23T00:00:00.000Z", result.get("segments").get("maxTime").toString()); Assert.assertEquals(2, result.get("segments").get("count")); Assert.assertEquals(30L, result.get("segments").get("size")); + Assert.assertEquals(50L, result.get("segments").get("replicatedSize")); EasyMock.verify(inventoryView, server, server2, server3); } + @Test + public void testSimpleGetTheDataSourceWithReplicatedSegments() + { + server = new DruidServer("server1", "host1", null, 1234, ServerType.HISTORICAL, "tier1", 0); + DruidServer server2 = new DruidServer("server2", "host2", null, 1234, ServerType.HISTORICAL, "tier2", 0); + DruidServer server3 = new DruidServer("server3", "host3", null, 1234, ServerType.HISTORICAL, "tier1", 0); + + server.addDataSegment(dataSegmentList.get(0)); + server.addDataSegment(dataSegmentList.get(1)); + server.addDataSegment(dataSegmentList.get(2)); + server2.addDataSegment(dataSegmentList.get(0)); + server2.addDataSegment(dataSegmentList.get(1)); + server3.addDataSegment(dataSegmentList.get(2)); + + EasyMock.expect(inventoryView.getInventory()).andReturn( + ImmutableList.of(server, server2, server3) + ).atLeastOnce(); + + EasyMock.replay(inventoryView); + + DataSourcesResource dataSourcesResource = + new DataSourcesResource(inventoryView, null, null, null, new AuthConfig(), null); + Response response = dataSourcesResource.getTheDataSource("datasource1", null); + Assert.assertEquals(200, response.getStatus()); + Map> result1 = (Map>) response.getEntity(); + Assert.assertEquals(2, ((Map) (result1.get("tiers").get("tier1"))).get("segmentCount")); + Assert.assertEquals(30L, ((Map) (result1.get("tiers").get("tier1"))).get("size")); + Assert.assertEquals(30L, ((Map) (result1.get("tiers").get("tier1"))).get("replicatedSize")); + Assert.assertEquals(2, ((Map) (result1.get("tiers").get("tier2"))).get("segmentCount")); + Assert.assertEquals(30L, ((Map) (result1.get("tiers").get("tier2"))).get("size")); + Assert.assertNotNull(result1.get("segments")); + Assert.assertEquals("2010-01-01T00:00:00.000Z", result1.get("segments").get("minTime").toString()); + Assert.assertEquals("2010-01-23T00:00:00.000Z", result1.get("segments").get("maxTime").toString()); + Assert.assertEquals(2, result1.get("segments").get("count")); + Assert.assertEquals(30L, result1.get("segments").get("size")); + Assert.assertEquals(60L, result1.get("segments").get("replicatedSize")); + + response = dataSourcesResource.getTheDataSource("datasource2", null); + Assert.assertEquals(200, response.getStatus()); + Map> result2 = (Map>) response.getEntity(); + Assert.assertEquals(1, ((Map) (result2.get("tiers").get("tier1"))).get("segmentCount")); + Assert.assertEquals(30L, ((Map) (result2.get("tiers").get("tier1"))).get("size")); + Assert.assertEquals(60L, ((Map) (result2.get("tiers").get("tier1"))).get("replicatedSize")); + Assert.assertNotNull(result2.get("segments")); + Assert.assertEquals("2010-01-01T00:00:00.000Z", result2.get("segments").get("minTime").toString()); + Assert.assertEquals("2010-01-02T00:00:00.000Z", result2.get("segments").get("maxTime").toString()); + Assert.assertEquals(1, result2.get("segments").get("count")); + Assert.assertEquals(30L, result2.get("segments").get("size")); + Assert.assertEquals(60L, result2.get("segments").get("replicatedSize")); + EasyMock.verify(inventoryView); + } + @Test public void testGetSegmentDataSourceIntervals() {