Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #363 from metamx/moar-endpoints 

More Coordinator endpoints for getting info out of druid
This commit is contained in:
fjy 2014-01-20 13:07:37 -08:00
parent c1f0027a36 215584891e
commit 08966c424a
18 changed files with 979 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/content/Broker.md
View File

@ -37,7 +37,7 @@ The broker module uses several of the default modules in [Configuration](Configu
|Property|Possible Values|Description|Default|
|--------|---------------|-----------|-------|
|`druid.broker.cache.type`|`local`, `memcache`|The type of cache to use for queries.|`local`|
|`druid.broker.cache.type`|`local`, `memcached`|The type of cache to use for queries.|`local`|
|`druid.broker.balancer.type`|`random`, `connectionCount`|Determines how the broker balances connections to compute nodes. `random` choose randomly, `connectionCount` picks the node with the fewest number of active connections to|`random`|
#### Local Cache

70
server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java
View File

@ -44,6 +44,7 @@ import io.druid.client.DruidDataSource;
import io.druid.client.DruidServer;
import io.druid.client.ServerInventoryView;
import io.druid.client.indexing.IndexingServiceClient;
import io.druid.collections.CountingMap;
import io.druid.common.config.JacksonConfigManager;
import io.druid.concurrent.Execs;
import io.druid.curator.discovery.ServiceAnnouncer;
@ -53,6 +54,8 @@ import io.druid.guice.ManageLifecycle;
import io.druid.guice.annotations.Self;
import io.druid.segment.IndexIO;
import io.druid.server.DruidNode;
import io.druid.server.coordinator.rules.LoadRule;
import io.druid.server.coordinator.rules.Rule;
import io.druid.server.initialization.ZkPathsConfig;
import io.druid.timeline.DataSegment;
import org.apache.curator.framework.CuratorFramework;
@ -63,7 +66,6 @@ import org.apache.curator.utils.ZKPaths;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
@ -72,6 +74,7 @@ import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
/**
@ -183,6 +186,53 @@ public class DruidCoordinator
return leader;
}
public Map<String, LoadQueuePeon> getLoadManagementPeons()
{
return loadManagementPeons;
}
public Map<String, Double> getReplicationStatus()
{
// find expected load per datasource
final CountingMap<String> expectedSegmentsInCluster = new CountingMap<>();
final DateTime now = new DateTime();
for (DataSegment segment : getAvailableDataSegments()) {
List<Rule> rules = databaseRuleManager.getRulesWithDefault(segment.getDataSource());
for (Rule rule : rules) {
if (rule instanceof LoadRule && rule.appliesTo(segment, now)) {
expectedSegmentsInCluster.add(segment.getDataSource(), ((LoadRule) rule).getReplicants());
//Integer count = expectedSegmentsInCluster.get(segment.getDataSource());
//if (count == null) {
// count = 0;
//}
//expectedSegmentsInCluster.put(segment.getDataSource(), count + ((LoadRule) rule).getReplicants());
break;
}
}
}
// find segments currently loaded per datasource
Map<String, Integer> segmentsInCluster = Maps.newHashMap();
for (DruidServer druidServer : serverInventoryView.getInventory()) {
for (DataSegment segment : druidServer.getSegments().values()) {
Integer count = segmentsInCluster.get(segment.getDataSource());
if (count == null) {
count = 0;
}
segmentsInCluster.put(segment.getDataSource(), count + 1);
}
}
// compare available segments with currently loaded
Map<String, Double> loadStatus = Maps.newHashMap();
for (Map.Entry<String, AtomicLong> entry : expectedSegmentsInCluster.entrySet()) {
Integer actual = segmentsInCluster.get(entry.getKey());
loadStatus.put(entry.getKey(), 100 * (actual == null ? 0.0D : (double) actual) / entry.getValue().get());
}
return loadStatus;
}
public Map<String, Double> getLoadStatus()
{
// find available segments
@ -687,11 +737,11 @@ public class DruidCoordinator
// Do coordinator stuff.
DruidCoordinatorRuntimeParams params =
DruidCoordinatorRuntimeParams.newBuilder()
.withStartTime(startTime)
.withDatasources(databaseSegmentManager.getInventory())
.withDynamicConfigs(dynamicConfigs.get())
.withEmitter(emitter)
.build();
.withStartTime(startTime)
.withDatasources(databaseSegmentManager.getInventory())
.withDynamicConfigs(dynamicConfigs.get())
.withEmitter(emitter)
.build();
for (DruidCoordinatorHelper helper : helpers) {
@ -724,7 +774,7 @@ public class DruidCoordinator
{
@Override
public boolean apply(
@Nullable DruidServer input
DruidServer input
)
{
return input.getType().equalsIgnoreCase("historical");
@ -760,11 +810,11 @@ public class DruidCoordinator
SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(cluster);
// Stop peons for servers that aren't there anymore.
final Set<String> disdappearedServers = Sets.newHashSet(loadManagementPeons.keySet());
final Set<String> disappeared = Sets.newHashSet(loadManagementPeons.keySet());
for (DruidServer server : servers) {
disdappearedServers.remove(server.getName());
disappeared.remove(server.getName());
}
for (String name : disdappearedServers) {
for (String name : disappeared) {
log.info("Removing listener for server[%s] which is no longer there.", name);
LoadQueuePeon peon = loadManagementPeons.remove(name);
peon.stop();

3
server/src/main/java/io/druid/server/coordinator/LoadQueuePeon.java
View File

@ -19,6 +19,7 @@
package io.druid.server.coordinator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
@ -103,6 +104,7 @@ public class LoadQueuePeon
this.config = config;
}
@JsonProperty
public Set<DataSegment> getSegmentsToLoad()
{
return new ConcurrentSkipListSet<DataSegment>(
@ -120,6 +122,7 @@ public class LoadQueuePeon
);
}
@JsonProperty
public Set<DataSegment> getSegmentsToDrop()
{
return new ConcurrentSkipListSet<DataSegment>(

2
server/src/main/java/io/druid/server/http/CoordinatorDynamicConfigsResource.java
View File

@ -32,7 +32,7 @@ import javax.ws.rs.core.Response;
/**
*/
@Path("/coordinator/config")
@Path("/druid/coordinator/v1/config")
public class CoordinatorDynamicConfigsResource
{
private final JacksonConfigManager manager;

124
server/src/main/java/io/druid/server/http/CoordinatorResource.java
View File

@ -19,21 +19,23 @@
package io.druid.server.http;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.inject.Inject;
import io.druid.server.coordinator.DruidCoordinator;
import io.druid.server.coordinator.LoadPeonCallback;
import io.druid.server.coordinator.LoadQueuePeon;
import io.druid.timeline.DataSegment;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
import java.util.List;
/**
*/
@Path("/coordinator")
@Path("/druid/coordinator/v1")
public class CoordinatorResource
{
private final DruidCoordinator coordinator;
@ -46,74 +48,64 @@ public class CoordinatorResource
this.coordinator = coordinator;
}
@POST
@Path("/move")
@Consumes("application/json")
public Response moveSegment(List<SegmentToMove> segmentsToMove)
@GET
@Path("/leader")
@Produces("application/json")
public Response getLeader()
{
Response resp = Response.status(Response.Status.OK).build();
for (SegmentToMove segmentToMove : segmentsToMove) {
try {
coordinator.moveSegment(
segmentToMove.getFromServer(),
segmentToMove.getToServer(),
segmentToMove.getSegmentName(),
new LoadPeonCallback()
{
@Override
protected void execute()
{
return;
}
}
);
}
catch (Exception e) {
resp = Response
.status(Response.Status.BAD_REQUEST)
.entity(e.getMessage())
.build();
break;
}
}
return resp;
}
@POST
@Path("/drop")
@Consumes("application/json")
public Response dropSegment(List<SegmentToDrop> segmentsToDrop)
{
Response resp = Response.status(Response.Status.OK).build();
for (SegmentToDrop segmentToDrop : segmentsToDrop) {
try {
coordinator.dropSegment(
segmentToDrop.getFromServer(), segmentToDrop.getSegmentName(), new LoadPeonCallback()
{
@Override
protected void execute()
{
return;
}
}
);
}
catch (Exception e) {
resp = Response
.status(Response.Status.BAD_REQUEST)
.entity(e.getMessage())
.build();
break;
}
}
return resp;
return Response.ok(coordinator.getCurrentLeader()).build();
}
@GET
@Path("/loadstatus")
@Produces("application/json")
public Response getLoadStatus()
public Response getLoadStatus(
@QueryParam("full") String full
)
{
if (full != null) {
return Response.ok(coordinator.getReplicationStatus()).build();
}
return Response.ok(coordinator.getLoadStatus()).build();
}
@GET
@Path("loadqueue")
@Produces("application/json")
public Response getLoadQueue(
@QueryParam("simple") String simple
)
{
if (simple != null) {
return Response.ok(
Maps.transformValues(
coordinator.getLoadManagementPeons(),
new Function<LoadQueuePeon, Object>()
{
@Override
public Object apply(LoadQueuePeon input)
{
long loadSize = 0;
for (DataSegment dataSegment : input.getSegmentsToLoad()) {
loadSize += dataSegment.getSize();
}
long dropSize = 0;
for (DataSegment dataSegment : input.getSegmentsToDrop()) {
dropSize += dataSegment.getSize();
}
return new ImmutableMap.Builder<>()
.put("segmentsToLoad", input.getSegmentsToLoad().size())
.put("segmentsToDrop", input.getSegmentsToDrop().size())
.put("segmentsToLoadSize", loadSize)
.put("segmentsToDropSize", dropSize)
.build();
}
}
)
).build();
}
return Response.ok(coordinator.getLoadManagementPeons()).build();
}
}

159
server/src/main/java/io/druid/server/http/DBResource.java Normal file
View File

@ -0,0 +1,159 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.server.http;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.inject.Inject;
import io.druid.client.DruidDataSource;
import io.druid.db.DatabaseSegmentManager;
import io.druid.timeline.DataSegment;
import javax.annotation.Nullable;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
import java.util.Collections;
import java.util.List;
/**
*/
@Path("/druid/coordinator/v1/db")
public class DBResource
{
private final DatabaseSegmentManager databaseSegmentManager;
@Inject
public DBResource(
DatabaseSegmentManager databaseSegmentManager
)
{
this.databaseSegmentManager = databaseSegmentManager;
}
@GET
@Path("/datasources")
@Produces("application/json")
public Response getDatabaseDataSources(
@QueryParam("full") String full,
@QueryParam("includeDisabled") String includeDisabled
)
{
Response.ResponseBuilder builder = Response.status(Response.Status.OK);
if (includeDisabled != null) {
return builder.entity(databaseSegmentManager.getAllDatasourceNames()).build();
}
if (full != null) {
return builder.entity(databaseSegmentManager.getInventory()).build();
}
List<String> dataSourceNames = Lists.newArrayList(
Iterables.transform(
databaseSegmentManager.getInventory(),
new Function<DruidDataSource, String>()
{
@Override
public String apply(@Nullable DruidDataSource dataSource)
{
return dataSource.getName();
}
}
)
);
Collections.sort(dataSourceNames);
return builder.entity(dataSourceNames).build();
}
@GET
@Path("/datasources/{dataSourceName}")
@Produces("application/json")
public Response getDatabaseSegmentDataSource(
@PathParam("dataSourceName") final String dataSourceName
)
{
DruidDataSource dataSource = databaseSegmentManager.getInventoryValue(dataSourceName);
if (dataSource == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
return Response.status(Response.Status.OK).entity(dataSource).build();
}
@GET
@Path("/datasources/{dataSourceName}/segments")
@Produces("application/json")
public Response getDatabaseSegmentDataSourceSegments(
@PathParam("dataSourceName") String dataSourceName,
@QueryParam("full") String full
)
{
DruidDataSource dataSource = databaseSegmentManager.getInventoryValue(dataSourceName);
if (dataSource == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
Response.ResponseBuilder builder = Response.status(Response.Status.OK);
if (full != null) {
return builder.entity(dataSource.getSegments()).build();
}
return builder.entity(
Iterables.transform(
dataSource.getSegments(),
new Function<DataSegment, Object>()
{
@Override
public Object apply(@Nullable DataSegment segment)
{
return segment.getIdentifier();
}
}
)
).build();
}
@GET
@Path("/datasources/{dataSourceName}/segments/{segmentId}")
@Produces("application/json")
public Response getDatabaseSegmentDataSourceSegment(
@PathParam("dataSourceName") String dataSourceName,
@PathParam("segmentId") String segmentId
)
{
DruidDataSource dataSource = databaseSegmentManager.getInventoryValue(dataSourceName);
if (dataSource == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
for (DataSegment segment : dataSource.getSegments()) {
if (segment.getIdentifier().equalsIgnoreCase(segmentId)) {
return Response.status(Response.Status.OK).entity(segment).build();
}
}
return Response.status(Response.Status.NOT_FOUND).build();
}
}

325
server/src/main/java/io/druid/server/http/DatasourcesResource.java Normal file
View File

@ -0,0 +1,325 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.server.http;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
import io.druid.client.DruidDataSource;
import io.druid.client.DruidServer;
import io.druid.client.InventoryView;
import io.druid.client.indexing.IndexingServiceClient;
import io.druid.db.DatabaseSegmentManager;
import io.druid.segment.IndexGranularity;
import io.druid.timeline.DataSegment;
import org.joda.time.Interval;
import javax.annotation.Nullable;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
/**
*/
@Path("/druid/coordinator/v1/datasources")
public class DatasourcesResource
{
private static Map<String, Object> makeSimpleDatasource(DruidDataSource input)
{
return new ImmutableMap.Builder<String, Object>()
.put("name", input.getName())
.put("properties", input.getProperties())
.build();
}
private final InventoryView serverInventoryView;
private final DatabaseSegmentManager databaseSegmentManager;
private final IndexingServiceClient indexingServiceClient;
@Inject
public DatasourcesResource(
InventoryView serverInventoryView,
DatabaseSegmentManager databaseSegmentManager,
@Nullable IndexingServiceClient indexingServiceClient
)
{
this.serverInventoryView = serverInventoryView;
this.databaseSegmentManager = databaseSegmentManager;
this.indexingServiceClient = indexingServiceClient;
}
@GET
@Produces("application/json")
public Response getQueryableDataSources(
@QueryParam("full") String full,
@QueryParam("simple") String simple,
@QueryParam("gran") String gran
)
{
Response.ResponseBuilder builder = Response.status(Response.Status.OK);
if (full != null) {
return builder.entity(getDataSources()).build();
} else if (simple != null) {
return builder.entity(
Lists.newArrayList(
Iterables.transform(
getDataSources(),
new Function<DruidDataSource, Map<String, Object>>()
{
@Override
public Map<String, Object> apply(DruidDataSource dataSource)
{
return makeSimpleDatasource(dataSource);
}
}
)
)
).build();
} else if (gran != null) {
IndexGranularity granularity = IndexGranularity.fromString(gran);
// TODO
}
return builder.entity(
Lists.newArrayList(
Iterables.transform(
getDataSources(),
new Function<DruidDataSource, String>()
{
@Override
public String apply(DruidDataSource dataSource)
{
return dataSource.getName();
}
}
)
)
).build();
}
@DELETE
@Path("/{dataSourceName}")
public Response deleteDataSource(
@PathParam("dataSourceName") final String dataSourceName,
@QueryParam("kill") final String kill,
@QueryParam("interval") final String interval
)
{
if (indexingServiceClient == null) {
return Response.status(Response.Status.OK).entity(ImmutableMap.of("error", "no indexing service found")).build();
}
if (kill != null && Boolean.valueOf(kill)) {
indexingServiceClient.killSegments(dataSourceName, new Interval(interval));
} else {
if (!databaseSegmentManager.removeDatasource(dataSourceName)) {
return Response.status(Response.Status.NOT_FOUND).build();
}
}
return Response.status(Response.Status.OK).build();
}
@POST
@Path("/{dataSourceName}")
@Consumes("application/json")
public Response enableDataSource(
@PathParam("dataSourceName") final String dataSourceName
)
{
if (!databaseSegmentManager.enableDatasource(dataSourceName)) {
return Response.status(Response.Status.NOT_FOUND).build();
}
return Response.status(Response.Status.OK).build();
}
@GET
@Path("/{dataSourceName}/segments")
@Produces("application/json")
public Response getSegmentDataSourceSegments(
@PathParam("dataSourceName") String dataSourceName,
@QueryParam("full") String full
)
{
DruidDataSource dataSource = getDataSource(dataSourceName.toLowerCase());
if (dataSource == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
Response.ResponseBuilder builder = Response.status(Response.Status.OK);
if (full != null) {
return builder.entity(dataSource.getSegments()).build();
}
return builder.entity(
Iterables.transform(
dataSource.getSegments(),
new Function<DataSegment, Object>()
{
@Override
public Object apply(@Nullable DataSegment segment)
{
return segment.getIdentifier();
}
}
)
).build();
}
@GET
@Path("/{dataSourceName}/segments/{segmentId}")
@Produces("application/json")
public Response getSegmentDataSourceSegment(
@PathParam("dataSourceName") String dataSourceName,
@PathParam("segmentId") String segmentId
)
{
DruidDataSource dataSource = getDataSource(dataSourceName.toLowerCase());
if (dataSource == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
for (DataSegment segment : dataSource.getSegments()) {
if (segment.getIdentifier().equalsIgnoreCase(segmentId)) {
return Response.status(Response.Status.OK).entity(segment).build();
}
}
return Response.status(Response.Status.NOT_FOUND).build();
}
@DELETE
@Path("/{dataSourceName}/segments/{segmentId}")
public Response deleteDatasourceSegment(
@PathParam("dataSourceName") String dataSourceName,
@PathParam("segmentId") String segmentId
)
{
if (!databaseSegmentManager.removeSegment(dataSourceName, segmentId)) {
return Response.status(Response.Status.NOT_FOUND).build();
}
return Response.status(Response.Status.OK).build();
}
@POST
@Path("/{dataSourceName}/segments/{segmentId}")
@Consumes("application/json")
public Response enableDatasourceSegment(
@PathParam("dataSourceName") String dataSourceName,
@PathParam("segmentId") String segmentId
)
{
if (!databaseSegmentManager.enableSegment(segmentId)) {
return Response.status(Response.Status.NOT_FOUND).build();
}
return Response.status(Response.Status.OK).build();
}
private DruidDataSource getDataSource(final String dataSourceName)
{
Iterable<DruidDataSource> dataSources =
Iterables.concat(
Iterables.transform(
serverInventoryView.getInventory(),
new Function<DruidServer, DruidDataSource>()
{
@Override
public DruidDataSource apply(DruidServer input)
{
return input.getDataSource(dataSourceName);
}
}
)
);
List<DruidDataSource> validDataSources = Lists.newArrayList();
for (DruidDataSource dataSource : dataSources) {
if (dataSource != null) {
validDataSources.add(dataSource);
}
}
if (validDataSources.isEmpty()) {
return null;
}
Map<String, DataSegment> segmentMap = Maps.newHashMap();
for (DruidDataSource dataSource : validDataSources) {
if (dataSource != null) {
Iterable<DataSegment> segments = dataSource.getSegments();
for (DataSegment segment : segments) {
segmentMap.put(segment.getIdentifier(), segment);
}
}
}
return new DruidDataSource(
dataSourceName,
ImmutableMap.<String, String>of()
).addSegments(segmentMap);
}
private Set<DruidDataSource> getDataSources()
{
TreeSet<DruidDataSource> dataSources = Sets.newTreeSet(
new Comparator<DruidDataSource>()
{
@Override
public int compare(DruidDataSource druidDataSource, DruidDataSource druidDataSource1)
{
return druidDataSource.getName().compareTo(druidDataSource1.getName());
}
}
);
dataSources.addAll(
Lists.newArrayList(
Iterables.concat(
Iterables.transform(
serverInventoryView.getInventory(),
new Function<DruidServer, Iterable<DruidDataSource>>()
{
@Override
public Iterable<DruidDataSource> apply(DruidServer input)
{
return input.getDataSources();
}
}
)
)
)
);
return dataSources;
}
}

2
server/src/main/java/io/druid/server/http/InfoResource.java
View File

@ -57,6 +57,7 @@ import java.util.TreeSet;
/**
*/
@Deprecated
@Path("/info")
public class InfoResource
{
@ -102,7 +103,6 @@ public class InfoResource
private final DatabaseRuleManager databaseRuleManager;
private final IndexingServiceClient indexingServiceClient;
@Inject
public InfoResource(
DruidCoordinator coordinator,

70
server/src/main/java/io/druid/server/http/RulesResource.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.server.http;
import com.google.inject.Inject;
import io.druid.db.DatabaseRuleManager;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
/**
*/
@Path("/druid/coordinator/v1/rules")
public class RulesResource
{
private final DatabaseRuleManager databaseRuleManager;
@Inject
public RulesResource(
DatabaseRuleManager databaseRuleManager
)
{
this.databaseRuleManager = databaseRuleManager;
}
@GET
@Produces("application/json")
public Response getRules()
{
return Response.ok(databaseRuleManager.getAllRules()).build();
}
@GET
@Path("/{dataSourceName}")
@Produces("application/json")
public Response getDatasourceRules(
@PathParam("dataSourceName") final String dataSourceName,
@QueryParam("full") final String full
)
{
if (full != null) {
return Response.ok(databaseRuleManager.getRulesWithDefault(dataSourceName))
.build();
}
return Response.ok(databaseRuleManager.getRules(dataSourceName))
.build();
}
}

188
server/src/main/java/io/druid/server/http/ServersResource.java Normal file
View File

@ -0,0 +1,188 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.server.http;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.inject.Inject;
import io.druid.client.DruidServer;
import io.druid.client.InventoryView;
import io.druid.timeline.DataSegment;
import javax.annotation.Nullable;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
import java.util.Map;
/**
*/
@Path("/druid/coordinator/v1/servers")
public class ServersResource
{
private static Map<String, Object> makeSimpleServer(DruidServer input)
{
return new ImmutableMap.Builder<String, Object>()
.put("host", input.getHost())
.put("tier", input.getTier())
.put("currSize", input.getCurrSize())
.put("maxSize", input.getMaxSize())
.build();
}
private final InventoryView serverInventoryView;
@Inject
public ServersResource(
InventoryView serverInventoryView
)
{
this.serverInventoryView = serverInventoryView;
}
@GET
@Produces("application/json")
public Response getClusterServers(
@QueryParam("full") String full,
@QueryParam("simple") String simple
)
{
Response.ResponseBuilder builder = Response.status(Response.Status.OK);
if (full != null) {
return builder.entity(Lists.newArrayList(serverInventoryView.getInventory())).build();
} else if (simple != null) {
return builder.entity(
Lists.newArrayList(
Iterables.transform(
serverInventoryView.getInventory(),
new Function<DruidServer, Map<String, Object>>()
{
@Override
public Map<String, Object> apply(DruidServer input)
{
return makeSimpleServer(input);
}
}
)
)
).build();
}
return builder.entity(
Lists.newArrayList(
Iterables.transform(
serverInventoryView.getInventory(),
new Function<DruidServer, String>()
{
@Override
public String apply(DruidServer druidServer)
{
return druidServer.getHost();
}
}
)
)
).build();
}
@GET
@Path("/{serverName}")
@Produces("application/json")
public Response getServer(
@PathParam("serverName") String serverName,
@QueryParam("simple") String simple
)
{
DruidServer server = serverInventoryView.getInventoryValue(serverName);
if (server == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
Response.ResponseBuilder builder = Response.status(Response.Status.OK);
if (simple != null) {
return builder.entity(makeSimpleServer(server)).build();
}
return builder.entity(server)
.build();
}
@GET
@Path("/{serverName}/segments")
@Produces("application/json")
public Response getServerSegments(
@PathParam("serverName") String serverName,
@QueryParam("full") String full
)
{
Response.ResponseBuilder builder = Response.status(Response.Status.OK);
DruidServer server = serverInventoryView.getInventoryValue(serverName);
if (server == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
if (full != null) {
return builder.entity(server.getSegments().values()).build();
}
return builder.entity(
Collections2.transform(
server.getSegments().values(),
new Function<DataSegment, String>()
{
@Override
public String apply(@Nullable DataSegment segment)
{
return segment.getIdentifier();
}
}
)
).build();
}
@GET
@Path("/{serverName}/segments/{segmentId}")
@Produces("application/json")
public Response getServerSegment(
@PathParam("serverName") String serverName,
@PathParam("segmentId") String segmentId
)
{
DruidServer server = serverInventoryView.getInventoryValue(serverName);
if (server == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
DataSegment segment = server.getSegment(segmentId);
if (segment == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
return Response.status(Response.Status.OK).entity(segment).build();
}
}

88
server/src/main/java/io/druid/server/http/TiersResource.java Normal file
View File

@ -0,0 +1,88 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.server.http;
import com.google.api.client.util.Maps;
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import com.google.common.collect.Table;
import com.google.inject.Inject;
import io.druid.client.DruidServer;
import io.druid.client.InventoryView;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
import java.util.Map;
import java.util.Set;
/**
*/
@Path("/druid/coordinator/v1/tiers")
public class TiersResource
{
private final InventoryView serverInventoryView;
@Inject
public TiersResource(
InventoryView serverInventoryView
)
{
this.serverInventoryView = serverInventoryView;
}
@GET
@Produces("application/json")
public Response getTiers(
@QueryParam("simple") String simple
)
{
Response.ResponseBuilder builder = Response.status(Response.Status.OK);
if (simple != null) {
Map<String, Map<String, Long>> metadata = Maps.newHashMap();
for (DruidServer druidServer : serverInventoryView.getInventory()) {
Map<String, Long> tierMetadata = metadata.get(druidServer.getTier());
if (tierMetadata == null) {
tierMetadata = Maps.newHashMap();
metadata.put(druidServer.getTier(), tierMetadata);
}
Long currSize = tierMetadata.get("currSize");
tierMetadata.put("currSize", (currSize == null) ? 0 : currSize + druidServer.getCurrSize());
Long maxSize = tierMetadata.get("maxSize");
tierMetadata.put("maxSize", (maxSize == null) ? 0 : maxSize + druidServer.getMaxSize());
}
return builder.entity(metadata).build();
}
Set<String> tiers = Sets.newHashSet();
for (DruidServer server : serverInventoryView.getInventory()) {
tiers.add(server.getTier());
}
return builder.entity(tiers).build();
}
}

8
server/src/main/resources/static/js/enable-0.0.1.js
View File

@ -21,7 +21,7 @@ $(document).ready(function() {
var selected = $('#datasources option:selected').text();
$.ajax({
type: 'POST',
url:'/info/datasources/' + selected,
url:'/druid/coordinator/v1/datasources/' + selected,
data: JSON.stringify(selected),
contentType:"application/json; charset=utf-8",
dataType:"json",
@ -50,7 +50,7 @@ $(document).ready(function() {
var selected = $('#datasources option:selected').text();
$.ajax({
type: 'DELETE',
url:'/info/datasources/' + selected,
url:'/druid/coordinator/v1/datasources/' + selected,
data: JSON.stringify(selected),
contentType:"application/json; charset=utf-8",
dataType:"json",
@ -70,12 +70,12 @@ $(document).ready(function() {
}
});
$.getJSON("/info/db/datasources", function(enabled_datasources) {
$.getJSON("/druid/coordinator/v1/db/datasources", function(enabled_datasources) {
$.each(enabled_datasources, function(index, datasource) {
$('#enabled_datasources').append($('<li>' + datasource + '</li>'));
});
$.getJSON("/info/db/datasources?includeDisabled", function(db_datasources) {
$.getJSON("/druid/coordinator/v1/db/datasources?includeDisabled", function(db_datasources) {
var disabled_datasources = _.difference(db_datasources, enabled_datasources);
$.each(disabled_datasources, function(index, datasource) {
$('#disabled_datasources').append($('<li>' + datasource + '</li>'));

2
server/src/main/resources/static/js/handlers-0.0.1.js
View File

@ -2,7 +2,7 @@
$(document).ready(function() {
var basePath = "/info/";
var basePath = "/druid/coordinator/v1/";
var type = $('#select_type').attr('value') + '';
var view = $('#select_view').attr('value') + '';

74
server/src/main/resources/static/js/init-0.0.2.js
View File

@ -100,8 +100,8 @@ $(document).ready(function() {
}
// Execution stuff
$.get('/info/coordinator', function(data) {
$("#coordinator").html('Current Cluster Coordinator: ' + data.host);
$.get('/druid/coordinator/v1/leader', function(data) {
$("#coordinator").html('Current Cluster Coordinator Leader: ' + data.host);
});
$('#move_segment').submit(function() {
@ -118,57 +118,10 @@ $(document).ready(function() {
});
}
/*
$.ajax({
url:"/coordinator/move",
type: "POST",
data: JSON.stringify(data),
contentType:"application/json; charset=utf-8",
dataType:"json",
error: function(xhr, status, error) {
alert(error + ": " + xhr.responseText);
},
success: function(data, status, xhr) {
for (seg in CONSOLE.selected_segments) {
CONSOLE.selected_segments[seg].children('.server_host').text($('#move_segment > .to').val());
}
}
});
*/
return false;
});
/*$
('#drop_segment').submit(function() {
var data = [];
if ($.isEmptyObject(CONSOLE.selected_segments)) {
alert("Please select at least one segment");
}
for (seg in CONSOLE.selected_segments) {
data.push({
'segmentName' : seg,
'from' : CONSOLE.selected_segments[seg]
});
}
$.ajax({
url:"/coordinator/drop",
type: "POST",
data: JSON.stringify(data),
contentType:"application/json; charset=utf-8",
dataType:"json",
error: function(xhr, status, error) {
alert(error + ": " + xhr.responseText);
}
});
return false;
});
*/
$.get('/info/cluster', function(data) {
$.get('/druid/coordinator/v1/servers?full', function(data) {
$('.loading').hide();
initTables(data);
@ -176,26 +129,5 @@ $(document).ready(function() {
var oTable = [];
initDataTable($('#servers'), oTable);
initDataTable($('#segments'), oTable);
// init select segments
/*$("#segments tbody").click(function(event) {
var el = $(event.target.parentNode);
var key = el.children('.segment_name').text();
if (el.is("tr")) {
if (el.hasClass('row_selected')) {
el.removeClass('row_selected');
delete CONSOLE.selected_segments[key];
} else {
el.addClass('row_selected');
CONSOLE.selected_segments[key] = el;
}
var html ="";
for (segment in CONSOLE.selected_segments) {
html += segment + ' on ' + CONSOLE.selected_segments[segment].children('.server_host').text() + '<br/>';
}
$('#selected_segments').html(html);
}
});*/
});
});

4
server/src/main/resources/static/js/kill-0.0.1.js
View File

@ -22,7 +22,7 @@ $(document).ready(function() {
var interval = $('#interval').val();
$.ajax({
type: 'DELETE',
url:'/info/datasources/' + selected +'?kill=true&interval=' + interval,
url:'/druid/coordinator/v1/datasources/' + selected +'?kill=true&interval=' + interval,
contentType:"application/json; charset=utf-8",
dataType:"json",
error: function(xhr, status, error) {
@ -41,7 +41,7 @@ $(document).ready(function() {
}
});
$.getJSON("/info/db/datasources?includeDisabled", function(data) {
$.getJSON("/druid/coordinator/v1/db/datasources?includeDisabled", function(data) {
$.each(data, function(index, datasource) {
$('#datasources').append($('<option></option>').attr("value", datasource).text(datasource));
});

8
server/src/main/resources/static/js/rules-0.0.1.js
View File

@ -115,7 +115,7 @@ function makeTiersDropdown(rule) {
function getRules() {
var selected = $('#datasources option:selected').text();
if (selected !== "") {
$.getJSON("/info/rules/" + selected, function(data) {
$.getJSON("/druid/coordinator/v1/rules/" + selected, function(data) {
$('#rules_list').empty();
if (!$.isEmptyObject(data)) {
$.each(data, function(index, rule) {
@ -189,7 +189,7 @@ $(document).ready(function() {
var selected = $('#datasources option:selected').text();
$.ajax({
type: 'POST',
url:'/info/rules/' + selected,
url:'/druid/coordinator/v1/rules/' + selected,
data: JSON.stringify(rules),
contentType:"application/json; charset=utf-8",
dataType:"json",
@ -209,11 +209,11 @@ $(document).ready(function() {
}
});
$.getJSON("/info/tiers", function(theTiers) {
$.getJSON("/druid/coordinator/v1/tiers", function(theTiers) {
tiers = theTiers;
});
$.getJSON("/info/db/datasources", function(data) {
$.getJSON("/druid/coordinator/v1/db/datasources", function(data) {
$.each(data, function(index, datasource) {
$('#datasources').append($('<option></option>').attr("value", datasource).text(datasource));
});

10
services/src/main/java/io/druid/cli/CliCoordinator.java
View File

@ -47,9 +47,14 @@ import io.druid.server.http.BackwardsCompatibleInfoResource;
import io.druid.server.http.CoordinatorDynamicConfigsResource;
import io.druid.server.http.CoordinatorRedirectInfo;
import io.druid.server.http.CoordinatorResource;
import io.druid.server.http.DBResource;
import io.druid.server.http.DatasourcesResource;
import io.druid.server.http.InfoResource;
import io.druid.server.http.RedirectFilter;
import io.druid.server.http.RedirectInfo;
import io.druid.server.http.RulesResource;
import io.druid.server.http.ServersResource;
import io.druid.server.http.TiersResource;
import io.druid.server.initialization.JettyServerInitializer;
import org.apache.curator.framework.CuratorFramework;
import org.eclipse.jetty.server.Server;
@ -107,6 +112,11 @@ public class CliCoordinator extends ServerRunnable
Jerseys.addResource(binder, InfoResource.class);
Jerseys.addResource(binder, CoordinatorResource.class);
Jerseys.addResource(binder, CoordinatorDynamicConfigsResource.class);
Jerseys.addResource(binder, TiersResource.class);
Jerseys.addResource(binder, RulesResource.class);
Jerseys.addResource(binder, ServersResource.class);
Jerseys.addResource(binder, DatasourcesResource.class);
Jerseys.addResource(binder, DBResource.class);
LifecycleModule.register(binder, Server.class);
}

2
services/src/main/java/io/druid/cli/CoordinatorJettyServerInitializer.java
View File

@ -53,7 +53,7 @@ class CoordinatorJettyServerInitializer implements JettyServerInitializer
// The coordinator really needs a standarized api path
root.addFilter(GuiceFilter.class, "/status/*", null);
root.addFilter(GuiceFilter.class, "/info/*", null);
root.addFilter(GuiceFilter.class, "/coordinator/*", null);
root.addFilter(GuiceFilter.class, "/druid/coordinator/*", null);
HandlerList handlerList = new HandlerList();
handlerList.setHandlers(new Handler[]{root});