Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Include replicated segment size property for datasources endpoint (#8039) 

* Add replication size

* Summon comma
This commit is contained in:
Atul Mohan 2019-07-11 03:10:38 -05:00 committed by Clint Wylie
parent 349b743ce0
commit 631cda649b
3 changed files with 68 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/content/operations/api-reference.md
View File

@ -162,7 +162,7 @@ Returns a list of datasource names found in the cluster.
* `/druid/coordinator/v1/datasources?simple`
Returns a list of JSON objects containing the name and properties of datasources found in the cluster. Properties include segment count, total segment byte size, minTime, and maxTime.
Returns a list of JSON objects containing the name and properties of datasources found in the cluster. Properties include segment count, total segment byte size, replicated total segment byte size, minTime, and maxTime.
* `/druid/coordinator/v1/datasources?full`
@ -170,7 +170,7 @@ Returns a list of datasource names found in the cluster with all metadata about
* `/druid/coordinator/v1/datasources/{dataSourceName}`
Returns a JSON object containing the name and properties of a datasource. Properties include segment count, total segment byte size, minTime, and maxTime.
Returns a JSON object containing the name and properties of a datasource. Properties include segment count, total segment byte size, replicated total segment byte size, minTime, and maxTime.
* `/druid/coordinator/v1/datasources/{dataSourceName}?full`

10
server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java
View File

@ -538,6 +538,8 @@ public class DataSourcesResource
Map<String, HashSet<Object>> tierDistinctSegments = new HashMap<>();
long totalSegmentSize = 0;
long totalReplicatedSize = 0;
DateTime minTime = DateTimes.MAX;
DateTime maxTime = DateTimes.MIN;
String tier;
@ -552,6 +554,8 @@ public class DataSourcesResource
tierDistinctSegments.computeIfAbsent(tier, k -> new HashSet<>());
long dataSourceSegmentSize = 0;
long replicatedSegmentSize = 0;
for (DataSegment dataSegment : druidDataSource.getSegments()) {
// tier segments stats
if (!tierDistinctSegments.get(tier).contains(dataSegment.getId())) {
@ -565,6 +569,8 @@ public class DataSourcesResource
minTime = DateTimes.min(minTime, dataSegment.getInterval().getStart());
maxTime = DateTimes.max(maxTime, dataSegment.getInterval().getEnd());
}
totalReplicatedSize += dataSegment.getSize();
replicatedSegmentSize += dataSegment.getSize();
}
// tier stats
@ -577,10 +583,14 @@ public class DataSourcesResource
long segmentSize = MapUtils.getLong(tierStats, "size", 0L);
tierStats.put("size", segmentSize + dataSourceSegmentSize);
long replicatedSize = MapUtils.getLong(tierStats, "replicatedSize", 0L);
tierStats.put("replicatedSize", replicatedSize + replicatedSegmentSize);
}
segments.put("count", totalDistinctSegments.size());
segments.put("size", totalSegmentSize);
segments.put("replicatedSize", totalReplicatedSize);
segments.put("minTime", minTime);
segments.put("maxTime", maxTime);
return retVal;

56
server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java
View File

@ -373,11 +373,13 @@ public class DataSourcesResourceTest
Map<String, Map<String, Object>> result = (Map<String, Map<String, Object>>) response.getEntity();
Assert.assertEquals(1, ((Map) (result.get("tiers").get(null))).get("segmentCount"));
Assert.assertEquals(10L, ((Map) (result.get("tiers").get(null))).get("size"));
Assert.assertEquals(10L, ((Map) (result.get("tiers").get(null))).get("replicatedSize"));
Assert.assertNotNull(result.get("segments"));
Assert.assertEquals("2010-01-01T00:00:00.000Z", result.get("segments").get("minTime").toString());
Assert.assertEquals("2010-01-02T00:00:00.000Z", result.get("segments").get("maxTime").toString());
Assert.assertEquals(1, result.get("segments").get("count"));
Assert.assertEquals(10L, result.get("segments").get("size"));
Assert.assertEquals(10L, result.get("segments").get("replicatedSize"));
EasyMock.verify(inventoryView, server);
}
@ -413,6 +415,7 @@ public class DataSourcesResourceTest
Map<String, Map<String, Object>> result = (Map<String, Map<String, Object>>) response.getEntity();
Assert.assertEquals(2, ((Map) (result.get("tiers").get("cold"))).get("segmentCount"));
Assert.assertEquals(30L, ((Map) (result.get("tiers").get("cold"))).get("size"));
Assert.assertEquals(30L, ((Map) (result.get("tiers").get("cold"))).get("replicatedSize"));
Assert.assertEquals(1, ((Map) (result.get("tiers").get("hot"))).get("segmentCount"));
Assert.assertEquals(20L, ((Map) (result.get("tiers").get("hot"))).get("size"));
Assert.assertNotNull(result.get("segments"));
@ -420,9 +423,62 @@ public class DataSourcesResourceTest
Assert.assertEquals("2010-01-23T00:00:00.000Z", result.get("segments").get("maxTime").toString());
Assert.assertEquals(2, result.get("segments").get("count"));
Assert.assertEquals(30L, result.get("segments").get("size"));
Assert.assertEquals(50L, result.get("segments").get("replicatedSize"));
EasyMock.verify(inventoryView, server, server2, server3);
}
@Test
public void testSimpleGetTheDataSourceWithReplicatedSegments()
{
server = new DruidServer("server1", "host1", null, 1234, ServerType.HISTORICAL, "tier1", 0);
DruidServer server2 = new DruidServer("server2", "host2", null, 1234, ServerType.HISTORICAL, "tier2", 0);
DruidServer server3 = new DruidServer("server3", "host3", null, 1234, ServerType.HISTORICAL, "tier1", 0);
server.addDataSegment(dataSegmentList.get(0));
server.addDataSegment(dataSegmentList.get(1));
server.addDataSegment(dataSegmentList.get(2));
server2.addDataSegment(dataSegmentList.get(0));
server2.addDataSegment(dataSegmentList.get(1));
server3.addDataSegment(dataSegmentList.get(2));
EasyMock.expect(inventoryView.getInventory()).andReturn(
ImmutableList.of(server, server2, server3)
).atLeastOnce();
EasyMock.replay(inventoryView);
DataSourcesResource dataSourcesResource =
new DataSourcesResource(inventoryView, null, null, null, new AuthConfig(), null);
Response response = dataSourcesResource.getTheDataSource("datasource1", null);
Assert.assertEquals(200, response.getStatus());
Map<String, Map<String, Object>> result1 = (Map<String, Map<String, Object>>) response.getEntity();
Assert.assertEquals(2, ((Map) (result1.get("tiers").get("tier1"))).get("segmentCount"));
Assert.assertEquals(30L, ((Map) (result1.get("tiers").get("tier1"))).get("size"));
Assert.assertEquals(30L, ((Map) (result1.get("tiers").get("tier1"))).get("replicatedSize"));
Assert.assertEquals(2, ((Map) (result1.get("tiers").get("tier2"))).get("segmentCount"));
Assert.assertEquals(30L, ((Map) (result1.get("tiers").get("tier2"))).get("size"));
Assert.assertNotNull(result1.get("segments"));
Assert.assertEquals("2010-01-01T00:00:00.000Z", result1.get("segments").get("minTime").toString());
Assert.assertEquals("2010-01-23T00:00:00.000Z", result1.get("segments").get("maxTime").toString());
Assert.assertEquals(2, result1.get("segments").get("count"));
Assert.assertEquals(30L, result1.get("segments").get("size"));
Assert.assertEquals(60L, result1.get("segments").get("replicatedSize"));
response = dataSourcesResource.getTheDataSource("datasource2", null);
Assert.assertEquals(200, response.getStatus());
Map<String, Map<String, Object>> result2 = (Map<String, Map<String, Object>>) response.getEntity();
Assert.assertEquals(1, ((Map) (result2.get("tiers").get("tier1"))).get("segmentCount"));
Assert.assertEquals(30L, ((Map) (result2.get("tiers").get("tier1"))).get("size"));
Assert.assertEquals(60L, ((Map) (result2.get("tiers").get("tier1"))).get("replicatedSize"));
Assert.assertNotNull(result2.get("segments"));
Assert.assertEquals("2010-01-01T00:00:00.000Z", result2.get("segments").get("minTime").toString());
Assert.assertEquals("2010-01-02T00:00:00.000Z", result2.get("segments").get("maxTime").toString());
Assert.assertEquals(1, result2.get("segments").get("count"));
Assert.assertEquals(30L, result2.get("segments").get("size"));
Assert.assertEquals(60L, result2.get("segments").get("replicatedSize"));
EasyMock.verify(inventoryView);
}
@Test
public void testGetSegmentDataSourceIntervals()
{