From 6de51e022d083f68db66460179e36d2cb48f155b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20=C5=81ugowski?= Date: Sat, 24 Oct 2015 17:23:43 +0200 Subject: [PATCH] Coordinator - add intervals route. --- docs/content/design/coordinator.md | 18 ++ .../server/http/DatasourcesResource.java | 43 +--- .../druid/server/http/IntervalsResource.java | 164 +++++++++++++++ .../druid/server/http/InventoryViewUtils.java | 68 +++++++ .../server/http/IntervalsResourceTest.java | 190 ++++++++++++++++++ .../java/io/druid/cli/CliCoordinator.java | 4 + 6 files changed, 451 insertions(+), 36 deletions(-) create mode 100644 server/src/main/java/io/druid/server/http/IntervalsResource.java create mode 100644 server/src/main/java/io/druid/server/http/InventoryViewUtils.java create mode 100644 server/src/test/java/io/druid/server/http/IntervalsResourceTest.java diff --git a/docs/content/design/coordinator.md b/docs/content/design/coordinator.md index 5642c315235..45b01d00886 100644 --- a/docs/content/design/coordinator.md +++ b/docs/content/design/coordinator.md @@ -193,6 +193,24 @@ Returns all rules for a specified datasource and includes default datasource. Returns audit history of rules for a specified datasource. default value of interval can be specified by setting `druid.audit.manager.auditHistoryMillis` (1 week if not configured) in coordinator runtime.properties +#### Intervals + +* `/druid/coordinator/v1/intervals` + +Returns all intervals for all datasources with total size and count. + +* `/druid/coordinator/v1/intervals/{interval}` + +Returns aggregated total size and count for all intervals that intersect given isointerval. + +* `/druid/coordinator/v1/intervals/{interval}?simple` + +Returns total size and count for each interval within given isointerval. + +* `/druid/coordinator/v1/intervals/{interval}?full` + +Returns total size and count for each datasource for each interval within given isointerval. + ### POST diff --git a/server/src/main/java/io/druid/server/http/DatasourcesResource.java b/server/src/main/java/io/druid/server/http/DatasourcesResource.java index e77b240a327..04f7b7db970 100644 --- a/server/src/main/java/io/druid/server/http/DatasourcesResource.java +++ b/server/src/main/java/io/druid/server/http/DatasourcesResource.java @@ -27,12 +27,14 @@ import com.google.inject.Inject; import com.metamx.common.MapUtils; import com.metamx.common.Pair; import com.metamx.common.guava.Comparators; + import io.druid.client.DruidDataSource; import io.druid.client.DruidServer; import io.druid.client.InventoryView; import io.druid.client.indexing.IndexingServiceClient; import io.druid.metadata.MetadataSegmentManager; import io.druid.timeline.DataSegment; + import org.joda.time.DateTime; import org.joda.time.Interval; @@ -47,11 +49,11 @@ import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; + import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeSet; /** */ @@ -82,13 +84,14 @@ public class DatasourcesResource ) { Response.ResponseBuilder builder = Response.ok(); + final Set datasources = InventoryViewUtils.getDataSources(serverInventoryView); if (full != null) { - return builder.entity(getDataSources()).build(); + return builder.entity(datasources).build(); } else if (simple != null) { return builder.entity( Lists.newArrayList( Iterables.transform( - getDataSources(), + datasources, new Function>() { @Override @@ -105,7 +108,7 @@ public class DatasourcesResource return builder.entity( Lists.newArrayList( Iterables.transform( - getDataSources(), + datasources, new Function() { @Override @@ -460,38 +463,6 @@ public class DatasourcesResource ).addSegments(segmentMap); } - private Set getDataSources() - { - TreeSet dataSources = Sets.newTreeSet( - new Comparator() - { - @Override - public int compare(DruidDataSource druidDataSource, DruidDataSource druidDataSource1) - { - return druidDataSource.getName().compareTo(druidDataSource1.getName()); - } - } - ); - dataSources.addAll( - Lists.newArrayList( - Iterables.concat( - Iterables.transform( - serverInventoryView.getInventory(), - new Function>() - { - @Override - public Iterable apply(DruidServer input) - { - return input.getDataSources(); - } - } - ) - ) - ) - ); - return dataSources; - } - private Pair> getSegment(String segmentId) { DataSegment theSegment = null; diff --git a/server/src/main/java/io/druid/server/http/IntervalsResource.java b/server/src/main/java/io/druid/server/http/IntervalsResource.java new file mode 100644 index 00000000000..8adb1a80cff --- /dev/null +++ b/server/src/main/java/io/druid/server/http/IntervalsResource.java @@ -0,0 +1,164 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.http; + +import com.google.common.collect.Maps; +import com.google.inject.Inject; +import com.metamx.common.MapUtils; +import com.metamx.common.guava.Comparators; + +import io.druid.client.DruidDataSource; +import io.druid.client.InventoryView; +import io.druid.timeline.DataSegment; + +import org.joda.time.Interval; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import java.util.Comparator; +import java.util.Map; +import java.util.Set; + +/** + */ +@Path("/druid/coordinator/v1/intervals") +public class IntervalsResource +{ + private final InventoryView serverInventoryView; + + @Inject + public IntervalsResource( + InventoryView serverInventoryView + ) + { + this.serverInventoryView = serverInventoryView; + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + public Response getIntervals() + { + final Comparator comparator = Comparators.inverse(Comparators.intervalsByStartThenEnd()); + final Set datasources = InventoryViewUtils.getDataSources(serverInventoryView); + + final Map>> retVal = Maps.newTreeMap(comparator); + for (DruidDataSource dataSource : datasources) { + for (DataSegment dataSegment : dataSource.getSegments()) { + Map> interval = retVal.get(dataSegment.getInterval()); + if (interval == null) { + Map> tmp = Maps.newHashMap(); + retVal.put(dataSegment.getInterval(), tmp); + } + setProperties(retVal, dataSource, dataSegment); + } + } + + return Response.ok(retVal).build(); + } + + @GET + @Path("/{interval}") + @Produces(MediaType.APPLICATION_JSON) + public Response getSpecificIntervals( + @PathParam("interval") String interval, + @QueryParam("simple") String simple, + @QueryParam("full") String full + ) + { + final Interval theInterval = new Interval(interval.replace("_", "/")); + final Set datasources = InventoryViewUtils.getDataSources(serverInventoryView); + + final Comparator comparator = Comparators.inverse(Comparators.intervalsByStartThenEnd()); + if (full != null) { + final Map>> retVal = Maps.newTreeMap(comparator); + for (DruidDataSource dataSource : datasources) { + for (DataSegment dataSegment : dataSource.getSegments()) { + if (theInterval.contains(dataSegment.getInterval())) { + Map> dataSourceInterval = retVal.get(dataSegment.getInterval()); + if (dataSourceInterval == null) { + Map> tmp = Maps.newHashMap(); + retVal.put(dataSegment.getInterval(), tmp); + } + setProperties(retVal, dataSource, dataSegment); + } + } + } + + return Response.ok(retVal).build(); + } + + if (simple != null) { + final Map> retVal = Maps.newHashMap(); + for (DruidDataSource dataSource : datasources) { + for (DataSegment dataSegment : dataSource.getSegments()) { + if (theInterval.contains(dataSegment.getInterval())) { + Map properties = retVal.get(dataSegment.getInterval()); + if (properties == null) { + properties = Maps.newHashMap(); + properties.put("size", dataSegment.getSize()); + properties.put("count", 1); + + retVal.put(dataSegment.getInterval(), properties); + } else { + properties.put("size", MapUtils.getLong(properties, "size", 0L) + dataSegment.getSize()); + properties.put("count", MapUtils.getInt(properties, "count", 0) + 1); + } + } + } + } + + return Response.ok(retVal).build(); + } + + final Map retVal = Maps.newHashMap(); + for (DruidDataSource dataSource : datasources) { + for (DataSegment dataSegment : dataSource.getSegments()) { + if (theInterval.contains(dataSegment.getInterval())) { + retVal.put("size", MapUtils.getLong(retVal, "size", 0L) + dataSegment.getSize()); + retVal.put("count", MapUtils.getInt(retVal, "count", 0) + 1); + } + } + } + + return Response.ok(retVal).build(); + } + + private void setProperties( + final Map>> retVal, + DruidDataSource dataSource, DataSegment dataSegment) { + Map properties = retVal.get(dataSegment.getInterval()).get(dataSource.getName()); + if (properties == null) { + properties = Maps.newHashMap(); + properties.put("size", dataSegment.getSize()); + properties.put("count", 1); + + retVal.get(dataSegment.getInterval()).put(dataSource.getName(), properties); + } else { + properties.put("size", MapUtils.getLong(properties, "size", 0L) + dataSegment.getSize()); + properties.put("count", MapUtils.getInt(properties, "count", 0) + 1); + } + } +} diff --git a/server/src/main/java/io/druid/server/http/InventoryViewUtils.java b/server/src/main/java/io/druid/server/http/InventoryViewUtils.java new file mode 100644 index 00000000000..6caf90017fe --- /dev/null +++ b/server/src/main/java/io/druid/server/http/InventoryViewUtils.java @@ -0,0 +1,68 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.http; + +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.client.InventoryView; + +import java.util.Comparator; +import java.util.Set; +import java.util.TreeSet; + +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +public class InventoryViewUtils { + + public static Set getDataSources(InventoryView serverInventoryView) + { + TreeSet dataSources = Sets.newTreeSet( + new Comparator() + { + @Override + public int compare(DruidDataSource druidDataSource, DruidDataSource druidDataSource1) + { + return druidDataSource.getName().compareTo(druidDataSource1.getName()); + } + } + ); + dataSources.addAll( + Lists.newArrayList( + Iterables.concat( + Iterables.transform( + serverInventoryView.getInventory(), + new Function>() + { + @Override + public Iterable apply(DruidServer input) + { + return input.getDataSources(); + } + } + ) + ) + ) + ); + return dataSources; + } +} diff --git a/server/src/test/java/io/druid/server/http/IntervalsResourceTest.java b/server/src/test/java/io/druid/server/http/IntervalsResourceTest.java new file mode 100644 index 00000000000..3173c4516d8 --- /dev/null +++ b/server/src/test/java/io/druid/server/http/IntervalsResourceTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.http; + +import com.google.common.collect.ImmutableList; + +import io.druid.client.DruidServer; +import io.druid.client.InventoryView; +import io.druid.timeline.DataSegment; + +import org.easymock.EasyMock; +import org.joda.time.Interval; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.ws.rs.core.Response; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public class IntervalsResourceTest +{ + private InventoryView inventoryView; + private DruidServer server; + private List dataSegmentList; + + @Before + public void setUp() + { + inventoryView = EasyMock.createStrictMock(InventoryView.class); + server = EasyMock.createStrictMock(DruidServer.class); + dataSegmentList = new ArrayList<>(); + dataSegmentList.add( + new DataSegment( + "datasource1", + new Interval("2010-01-01T00:00:00.000Z/P1D"), + null, + null, + null, + null, + null, + 0x9, + 20 + ) + ); + dataSegmentList.add( + new DataSegment( + "datasource1", + new Interval("2010-01-22T00:00:00.000Z/P1D"), + null, + null, + null, + null, + null, + 0x9, + 10 + ) + ); + dataSegmentList.add( + new DataSegment( + "datasource2", + new Interval("2010-01-01T00:00:00.000Z/P1D"), + null, + null, + null, + null, + null, + 0x9, + 5 + ) + ); + server = new DruidServer("who", "host", 1234, "historical", "tier1", 0); + server.addDataSegment(dataSegmentList.get(0).getIdentifier(), dataSegmentList.get(0)); + server.addDataSegment(dataSegmentList.get(1).getIdentifier(), dataSegmentList.get(1)); + server.addDataSegment(dataSegmentList.get(2).getIdentifier(), dataSegmentList.get(2)); + } + + @Test + public void testGetIntervals() + { + EasyMock.expect(inventoryView.getInventory()).andReturn( + ImmutableList.of(server) + ).atLeastOnce(); + EasyMock.replay(inventoryView); + + List expectedIntervals = new ArrayList<>(); + expectedIntervals.add(new Interval("2010-01-01T00:00:00.000Z/2010-01-02T00:00:00.000Z")); + expectedIntervals.add(new Interval("2010-01-22T00:00:00.000Z/2010-01-23T00:00:00.000Z")); + IntervalsResource intervalsResource = new IntervalsResource(inventoryView); + + Response response = intervalsResource.getIntervals(); + TreeMap>> actualIntervals = (TreeMap) response.getEntity(); + Assert.assertEquals(2, actualIntervals.size()); + Assert.assertEquals(expectedIntervals.get(1), actualIntervals.firstKey()); + Assert.assertEquals(10L, actualIntervals.get(expectedIntervals.get(1)).get("datasource1").get("size")); + Assert.assertEquals(1, actualIntervals.get(expectedIntervals.get(1)).get("datasource1").get("count")); + Assert.assertEquals(expectedIntervals.get(0), actualIntervals.lastKey()); + Assert.assertEquals(20L, actualIntervals.get(expectedIntervals.get(0)).get("datasource1").get("size")); + Assert.assertEquals(1, actualIntervals.get(expectedIntervals.get(0)).get("datasource1").get("count")); + Assert.assertEquals(5L, actualIntervals.get(expectedIntervals.get(0)).get("datasource2").get("size")); + Assert.assertEquals(1, actualIntervals.get(expectedIntervals.get(0)).get("datasource2").get("count")); + + EasyMock.verify(inventoryView); + } + + @Test + public void testSimpleGetSpecificIntervals() + { + EasyMock.expect(inventoryView.getInventory()).andReturn( + ImmutableList.of(server) + ).atLeastOnce(); + EasyMock.replay(inventoryView); + + List expectedIntervals = new ArrayList<>(); + expectedIntervals.add(new Interval("2010-01-01T00:00:00.000Z/2010-01-02T00:00:00.000Z")); + IntervalsResource intervalsResource = new IntervalsResource(inventoryView); + + Response response = intervalsResource.getSpecificIntervals("2010-01-01T00:00:00.000Z/P1D", "simple", null); + Map> actualIntervals = (Map) response.getEntity(); + Assert.assertEquals(1, actualIntervals.size()); + Assert.assertTrue(actualIntervals.containsKey(expectedIntervals.get(0))); + Assert.assertEquals(25L, actualIntervals.get(expectedIntervals.get(0)).get("size")); + Assert.assertEquals(2, actualIntervals.get(expectedIntervals.get(0)).get("count")); + + EasyMock.verify(inventoryView); + } + + @Test + public void testFullGetSpecificIntervals() + { + EasyMock.expect(inventoryView.getInventory()).andReturn( + ImmutableList.of(server) + ).atLeastOnce(); + EasyMock.replay(inventoryView); + + List expectedIntervals = new ArrayList<>(); + expectedIntervals.add(new Interval("2010-01-01T00:00:00.000Z/2010-01-02T00:00:00.000Z")); + IntervalsResource intervalsResource = new IntervalsResource(inventoryView); + + Response response = intervalsResource.getSpecificIntervals("2010-01-01T00:00:00.000Z/P1D", null, "full"); + TreeMap>> actualIntervals = (TreeMap) response.getEntity(); + Assert.assertEquals(1, actualIntervals.size()); + Assert.assertEquals(expectedIntervals.get(0), actualIntervals.firstKey()); + Assert.assertEquals(20L, actualIntervals.get(expectedIntervals.get(0)).get("datasource1").get("size")); + Assert.assertEquals(1, actualIntervals.get(expectedIntervals.get(0)).get("datasource1").get("count")); + Assert.assertEquals(5L, actualIntervals.get(expectedIntervals.get(0)).get("datasource2").get("size")); + Assert.assertEquals(1, actualIntervals.get(expectedIntervals.get(0)).get("datasource2").get("count")); + + EasyMock.verify(inventoryView); + } + + @Test + public void testGetSpecificIntervals() + { + EasyMock.expect(inventoryView.getInventory()).andReturn( + ImmutableList.of(server) + ).atLeastOnce(); + EasyMock.replay(inventoryView); + + IntervalsResource intervalsResource = new IntervalsResource(inventoryView); + + Response response = intervalsResource.getSpecificIntervals("2010-01-01T00:00:00.000Z/P1D", null, null); + Map actualIntervals = (Map) response.getEntity(); + Assert.assertEquals(2, actualIntervals.size()); + Assert.assertEquals(25L, actualIntervals.get("size")); + Assert.assertEquals(2, actualIntervals.get("count")); + + EasyMock.verify(inventoryView); + } +} diff --git a/services/src/main/java/io/druid/cli/CliCoordinator.java b/services/src/main/java/io/druid/cli/CliCoordinator.java index 3911e0f9696..7f5b254a829 100644 --- a/services/src/main/java/io/druid/cli/CliCoordinator.java +++ b/services/src/main/java/io/druid/cli/CliCoordinator.java @@ -25,6 +25,7 @@ import com.google.inject.Provides; import com.google.inject.name.Names; import com.metamx.common.concurrent.ScheduledExecutorFactory; import com.metamx.common.logger.Logger; + import io.airlift.airline.Command; import io.druid.client.indexing.IndexingServiceClient; import io.druid.guice.ConfigProvider; @@ -50,6 +51,7 @@ import io.druid.server.http.CoordinatorDynamicConfigsResource; import io.druid.server.http.CoordinatorRedirectInfo; import io.druid.server.http.CoordinatorResource; import io.druid.server.http.DatasourcesResource; +import io.druid.server.http.IntervalsResource; import io.druid.server.http.MetadataResource; import io.druid.server.http.RedirectFilter; import io.druid.server.http.RedirectInfo; @@ -58,6 +60,7 @@ import io.druid.server.http.ServersResource; import io.druid.server.http.TiersResource; import io.druid.server.initialization.jetty.JettyServerInitializer; import io.druid.server.router.TieredBrokerConfig; + import org.apache.curator.framework.CuratorFramework; import org.eclipse.jetty.server.Server; @@ -132,6 +135,7 @@ public class CliCoordinator extends ServerRunnable Jerseys.addResource(binder, ServersResource.class); Jerseys.addResource(binder, DatasourcesResource.class); Jerseys.addResource(binder, MetadataResource.class); + Jerseys.addResource(binder, IntervalsResource.class); LifecycleModule.register(binder, Server.class); }