Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

first set of changes

This commit is contained in:
Dhruv Parthasarathy 2013-07-29 11:39:24 -07:00
parent b0090a1de6
commit 31ffed466c
8 changed files with 234 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
server/src/main/java/com/metamx/druid/http/InfoResource.java
View File

@ -31,9 +31,11 @@ import com.metamx.druid.client.DruidDataSource;
import com.metamx.druid.client.DruidServer;
import com.metamx.druid.client.InventoryView;
import com.metamx.druid.client.indexing.IndexingServiceClient;
import com.metamx.druid.config.JacksonConfigManager;
import com.metamx.druid.db.DatabaseRuleManager;
import com.metamx.druid.db.DatabaseSegmentManager;
import com.metamx.druid.master.DruidMaster;
import com.metamx.druid.master.DynamicConfigs;
import com.metamx.druid.master.rules.Rule;
import org.joda.time.Interval;
@ -101,6 +103,8 @@ public class InfoResource
private final DatabaseSegmentManager databaseSegmentManager;
private final DatabaseRuleManager databaseRuleManager;
private final IndexingServiceClient indexingServiceClient;
private final JacksonConfigManager configManager;
@Inject
public InfoResource(
@ -108,6 +112,7 @@ public class InfoResource
InventoryView serverInventoryView,
DatabaseSegmentManager databaseSegmentManager,
DatabaseRuleManager databaseRuleManager,
JacksonConfigManager configManager,
@Nullable
IndexingServiceClient indexingServiceClient
)
@ -116,6 +121,7 @@ public class InfoResource
this.serverInventoryView = serverInventoryView;
this.databaseSegmentManager = databaseSegmentManager;
this.databaseRuleManager = databaseRuleManager;
this.configManager=configManager;
this.indexingServiceClient = indexingServiceClient;
}
@ -129,6 +135,15 @@ public class InfoResource
.build();
}
@GET
@Path("/master/dynamicConfigs")
@Produces("application/json")
public Response getDynamicConfigs()
{
Response.ResponseBuilder builder = Response.status(Response.Status.OK);
return builder.build();
}
@GET
@Path("/cluster")
@Produces("application/json")
@ -373,6 +388,20 @@ public class InfoResource
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
}
@POST
@Path("master/setDynamicConfigs")
@Produces("application/json")
public Response setDynamicConfigs(
final DynamicConfigs dynamicConfigs)
{
if (!configManager.set(DynamicConfigs.CONFIG_KEY, dynamicConfigs))
{
return Response.status(Response.Status.BAD_REQUEST).build();
}
return Response.status(Response.Status.OK).build();
}
@GET
@Path("/datasources")
@Produces("application/json")

14
server/src/main/java/com/metamx/druid/master/DruidMaster.java
View File

@ -80,6 +80,7 @@ public class DruidMaster
private volatile boolean started = false;
private volatile boolean master = false;
private volatile boolean defaultConfigsSet=false;
private final DruidMasterConfig config;
private final ZkPathsConfig zkPaths;
@ -465,7 +466,11 @@ public class DruidMaster
serverInventoryView.start();
final List<Pair<? extends MasterRunnable, Duration>> masterRunnables = Lists.newArrayList();
if (!defaultConfigsSet)
{
configManager.set(DynamicConfigs.CONFIG_KEY, new DynamicConfigs());
defaultConfigsSet=true;
}
masterRunnables.add(Pair.of(new MasterComputeManagerRunnable(), config.getMasterPeriod()));
if (indexingServiceClient != null) {
@ -650,18 +655,17 @@ public class DruidMaster
}
// Do master stuff.
DynamicConfigs dynamicConfigs = configManager.watch(DynamicConfigs.CONFIG_KEY,DynamicConfigs.class).get();
DruidMasterRuntimeParams params =
DruidMasterRuntimeParams.newBuilder()
.withStartTime(startTime)
.withDatasources(databaseSegmentManager.getInventory())
.withMillisToWaitBeforeDeleting(config.getMillisToWaitBeforeDeleting())
.withDynamicConfigs(dynamicConfigs)
.withEmitter(emitter)
.withMergeBytesLimit(config.getMergeBytesLimit())
.withMergeSegmentsLimit(config.getMergeSegmentsLimit())
.withMaxSegmentsToMove(config.getMaxSegmentsToMove())
.build();
for (DruidMasterHelper helper : helpers) {
params = helper.run(params);
}

75
server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java
View File

@ -46,11 +46,8 @@ public class DruidMasterRuntimeParams
private final Map<String, LoadQueuePeon> loadManagementPeons;
private final ReplicationThrottler replicationManager;
private final ServiceEmitter emitter;
private final long millisToWaitBeforeDeleting;
private final DynamicConfigs dynamicConfigs;
private final MasterStats stats;
private final long mergeBytesLimit;
private final int mergeSegmentsLimit;
private final int maxSegmentsToMove;
private final DateTime balancerReferenceTimestamp;
public DruidMasterRuntimeParams(
@ -63,11 +60,8 @@ public class DruidMasterRuntimeParams
Map<String, LoadQueuePeon> loadManagementPeons,
ReplicationThrottler replicationManager,
ServiceEmitter emitter,
long millisToWaitBeforeDeleting,
DynamicConfigs dynamicConfigs,
MasterStats stats,
long mergeBytesLimit,
int mergeSegmentsLimit,
int maxSegmentsToMove,
DateTime balancerReferenceTimestamp
)
{
@ -80,11 +74,8 @@ public class DruidMasterRuntimeParams
this.loadManagementPeons = loadManagementPeons;
this.replicationManager = replicationManager;
this.emitter = emitter;
this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting;
this.dynamicConfigs = dynamicConfigs;
this.stats = stats;
this.mergeBytesLimit = mergeBytesLimit;
this.mergeSegmentsLimit = mergeSegmentsLimit;
this.maxSegmentsToMove = maxSegmentsToMove;
this.balancerReferenceTimestamp = balancerReferenceTimestamp;
}
@ -135,7 +126,7 @@ public class DruidMasterRuntimeParams
public long getMillisToWaitBeforeDeleting()
{
return millisToWaitBeforeDeleting;
return dynamicConfigs.getMillisToWaitBeforeDeleting();
}
public MasterStats getMasterStats()
@ -145,17 +136,17 @@ public class DruidMasterRuntimeParams
public long getMergeBytesLimit()
{
return mergeBytesLimit;
return dynamicConfigs.getMergeBytesLimit();
}
public int getMergeSegmentsLimit()
{
return mergeSegmentsLimit;
return dynamicConfigs.getMergeSegmentsLimit();
}
public int getMaxSegmentsToMove()
{
return maxSegmentsToMove;
return dynamicConfigs.getMaxSegmentsToMove();
}
public DateTime getBalancerReferenceTimestamp()
@ -190,11 +181,8 @@ public class DruidMasterRuntimeParams
loadManagementPeons,
replicationManager,
emitter,
millisToWaitBeforeDeleting,
dynamicConfigs,
stats,
mergeBytesLimit,
mergeSegmentsLimit,
maxSegmentsToMove,
balancerReferenceTimestamp
);
}
@ -210,11 +198,8 @@ public class DruidMasterRuntimeParams
private final Map<String, LoadQueuePeon> loadManagementPeons;
private ReplicationThrottler replicationManager;
private ServiceEmitter emitter;
private long millisToWaitBeforeDeleting;
private DynamicConfigs dynamicConfigs;
private MasterStats stats;
private long mergeBytesLimit;
private int mergeSegmentsLimit;
private int maxSegmentsToMove;
private DateTime balancerReferenceTimestamp;
Builder()
@ -228,11 +213,8 @@ public class DruidMasterRuntimeParams
this.loadManagementPeons = Maps.newHashMap();
this.replicationManager = null;
this.emitter = null;
this.millisToWaitBeforeDeleting = 0;
this.stats = new MasterStats();
this.mergeBytesLimit = 0;
this.mergeSegmentsLimit = 0;
this.maxSegmentsToMove = 0;
this.dynamicConfigs = new DynamicConfigs();
this.balancerReferenceTimestamp = null;
}
@ -246,11 +228,8 @@ public class DruidMasterRuntimeParams
Map<String, LoadQueuePeon> loadManagementPeons,
ReplicationThrottler replicationManager,
ServiceEmitter emitter,
long millisToWaitBeforeDeleting,
DynamicConfigs dynamicConfigs,
MasterStats stats,
long mergeBytesLimit,
int mergeSegmentsLimit,
int maxSegmentsToMove,
DateTime balancerReferenceTimestamp
)
{
@ -263,11 +242,8 @@ public class DruidMasterRuntimeParams
this.loadManagementPeons = loadManagementPeons;
this.replicationManager = replicationManager;
this.emitter = emitter;
this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting;
this.dynamicConfigs=dynamicConfigs;
this.stats = stats;
this.mergeBytesLimit = mergeBytesLimit;
this.mergeSegmentsLimit = mergeSegmentsLimit;
this.maxSegmentsToMove = maxSegmentsToMove;
this.balancerReferenceTimestamp = balancerReferenceTimestamp;
}
@ -283,11 +259,8 @@ public class DruidMasterRuntimeParams
loadManagementPeons,
replicationManager,
emitter,
millisToWaitBeforeDeleting,
dynamicConfigs,
stats,
mergeBytesLimit,
mergeSegmentsLimit,
maxSegmentsToMove,
balancerReferenceTimestamp
);
}
@ -346,33 +319,15 @@ public class DruidMasterRuntimeParams
return this;
}
public Builder withMillisToWaitBeforeDeleting(long millisToWaitBeforeDeleting)
{
this.millisToWaitBeforeDeleting = millisToWaitBeforeDeleting;
return this;
}
public Builder withMasterStats(MasterStats stats)
{
this.stats.accumulate(stats);
return this;
}
public Builder withMergeBytesLimit(long mergeBytesLimit)
public Builder withDynamicConfigs(DynamicConfigs configs)
{
this.mergeBytesLimit = mergeBytesLimit;
return this;
}
public Builder withMergeSegmentsLimit(int mergeSegmentsLimit)
{
this.mergeSegmentsLimit = mergeSegmentsLimit;
return this;
}
public Builder withMaxSegmentsToMove(int maxSegmentsToMove)
{
this.maxSegmentsToMove = maxSegmentsToMove;
this.dynamicConfigs = configs;
return this;
}

85
server/src/main/java/com/metamx/druid/master/DynamicConfigs.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package com.metamx.druid.master;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class DynamicConfigs
{
public static final String CONFIG_KEY = "master.dynamicConfigs";
private long millisToWaitBeforeDeleting=15 * 60 * 1000L;
private long mergeBytesLimit= 100000000L;
private int mergeSegmentsLimit = Integer.MAX_VALUE;
private int maxSegmentsToMove = 5;
@JsonCreator
public DynamicConfigs(@JsonProperty("millisToWaitBeforeDeleting") Integer millisToWaitBeforeDeleting,
@JsonProperty("mergeBytesLimit") Long mergeBytesLimit,
@JsonProperty("mergeSegmentsLimit") Integer mergeSegmentsLimit,
@JsonProperty("maxSegmentsToMove") Integer maxSegmentsToMove
)
{
if (maxSegmentsToMove!=null)
{
this.maxSegmentsToMove=maxSegmentsToMove;
}
if (millisToWaitBeforeDeleting!=null)
{
this.millisToWaitBeforeDeleting=millisToWaitBeforeDeleting;
}
if (mergeSegmentsLimit!=null)
{
this.mergeSegmentsLimit=mergeSegmentsLimit;
}
if (mergeBytesLimit!=null)
{
this.mergeBytesLimit=mergeBytesLimit;
}
}
public DynamicConfigs()
{
}
public static String getConfigKey()
{
return CONFIG_KEY;
}
public long getMillisToWaitBeforeDeleting()
{
return millisToWaitBeforeDeleting;
}
public long getMergeBytesLimit()
{
return mergeBytesLimit;
}
public int getMergeSegmentsLimit()
{
return mergeSegmentsLimit;
}
public int getMaxSegmentsToMove()
{
return maxSegmentsToMove;
}
}

30
server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java
View File

@ -167,7 +167,6 @@ public class DruidMasterBalancerTest
LoadQueuePeonTester fromPeon = new LoadQueuePeonTester();
LoadQueuePeonTester toPeon = new LoadQueuePeonTester();
DruidMasterRuntimeParams params =
DruidMasterRuntimeParams.newBuilder()
.withDruidCluster(
@ -184,9 +183,25 @@ public class DruidMasterBalancerTest
)
)
)
.withLoadManagementPeons(ImmutableMap.<String, LoadQueuePeon>of("from", fromPeon, "to", toPeon))
.withLoadManagementPeons(
ImmutableMap.<String, LoadQueuePeon>of(
"from",
fromPeon,
"to",
toPeon
)
)
.withAvailableSegments(segments.values())
.withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE)
.withDynamicConfigs(
new DynamicConfigs()
{
@Override
public int getMaxSegmentsToMove()
{
return MAX_SEGMENTS_TO_MOVE;
}
}
)
.withBalancerReferenceTimestamp(new DateTime("2013-01-01"))
.build();
@ -264,7 +279,14 @@ public class DruidMasterBalancerTest
)
.withLoadManagementPeons(ImmutableMap.<String, LoadQueuePeon>of("1", peon1, "2", peon2, "3", peon3, "4", peon4))
.withAvailableSegments(segments.values())
.withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE)
.withDynamicConfigs(new DynamicConfigs()
{
@Override
public int getMaxSegmentsToMove()
{
return MAX_SEGMENTS_TO_MOVE;
}
})
.withBalancerReferenceTimestamp(new DateTime("2013-01-01"))
.build();

63
server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java
View File

@ -177,7 +177,14 @@ public class DruidMasterRuleRunnerTest
.withAvailableSegments(availableSegments)
.withDatabaseRuleManager(databaseRuleManager)
.withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster()))
.withMaxSegmentsToMove(5)
.withDynamicConfigs(new DynamicConfigs()
{
@Override
public int getMaxSegmentsToMove()
{
return 5;
}
})
.withBalancerReferenceTimestamp(new DateTime("2013-01-01"))
.build();
@ -523,7 +530,14 @@ public class DruidMasterRuleRunnerTest
DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
.withDruidCluster(druidCluster)
.withMillisToWaitBeforeDeleting(0L)
.withDynamicConfigs( new DynamicConfigs()
{
@Override
public long getMillisToWaitBeforeDeleting()
{
return 0L;
}
})
.withAvailableSegments(availableSegments)
.withDatabaseRuleManager(databaseRuleManager)
.withSegmentReplicantLookup(segmentReplicantLookup)
@ -597,7 +611,14 @@ public class DruidMasterRuleRunnerTest
DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
.withDruidCluster(druidCluster)
.withMillisToWaitBeforeDeleting(0L)
.withDynamicConfigs( new DynamicConfigs()
{
@Override
public long getMillisToWaitBeforeDeleting()
{
return 0L;
}
})
.withAvailableSegments(availableSegments)
.withDatabaseRuleManager(databaseRuleManager)
.withSegmentReplicantLookup(segmentReplicantLookup)
@ -678,7 +699,14 @@ public class DruidMasterRuleRunnerTest
DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
.withDruidCluster(druidCluster)
.withMillisToWaitBeforeDeleting(0L)
.withDynamicConfigs( new DynamicConfigs()
{
@Override
public long getMillisToWaitBeforeDeleting()
{
return 0L;
}
})
.withAvailableSegments(availableSegments)
.withDatabaseRuleManager(databaseRuleManager)
.withSegmentReplicantLookup(segmentReplicantLookup)
@ -755,7 +783,14 @@ public class DruidMasterRuleRunnerTest
DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
.withDruidCluster(druidCluster)
.withMillisToWaitBeforeDeleting(0L)
.withDynamicConfigs( new DynamicConfigs()
{
@Override
public long getMillisToWaitBeforeDeleting()
{
return 0L;
}
})
.withAvailableSegments(availableSegments)
.withDatabaseRuleManager(databaseRuleManager)
.withSegmentReplicantLookup(segmentReplicantLookup)
@ -844,7 +879,14 @@ public class DruidMasterRuleRunnerTest
DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
.withDruidCluster(druidCluster)
.withMillisToWaitBeforeDeleting(0L)
.withDynamicConfigs( new DynamicConfigs()
{
@Override
public long getMillisToWaitBeforeDeleting()
{
return 0L;
}
})
.withAvailableSegments(availableSegments)
.withDatabaseRuleManager(databaseRuleManager)
.withSegmentReplicantLookup(segmentReplicantLookup)
@ -1030,7 +1072,14 @@ public class DruidMasterRuleRunnerTest
DruidMasterRuntimeParams params = new DruidMasterRuntimeParams.Builder()
.withDruidCluster(druidCluster)
.withMillisToWaitBeforeDeleting(0L)
.withDynamicConfigs( new DynamicConfigs()
{
@Override
public long getMillisToWaitBeforeDeleting()
{
return 0L;
}
})
.withAvailableSegments(longerAvailableSegments)
.withDatabaseRuleManager(databaseRuleManager)
.withSegmentReplicantLookup(segmentReplicantLookup)

15
server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java
View File

@ -419,10 +419,21 @@ public class DruidMasterSegmentMergerTest
final AtomicReference<MergerWhitelist> whitelistRef = new AtomicReference<MergerWhitelist>(null);
final DruidMasterSegmentMerger merger = new DruidMasterSegmentMerger(indexingServiceClient, whitelistRef);
final DynamicConfigs dynamicConfigs = new DynamicConfigs(null,mergeBytesLimit,mergeSegmentsLimit,null);
final DruidMasterRuntimeParams params = DruidMasterRuntimeParams.newBuilder()
.withAvailableSegments(ImmutableSet.copyOf(segments))
.withMergeBytesLimit(mergeBytesLimit)
.withMergeSegmentsLimit(mergeSegmentsLimit)
.withDynamicConfigs(
new DynamicConfigs()
{
@Override
public long getMergeBytesLimit(){
return mergeBytesLimit;
}
@Override
public int getMergeSegmentsLimit(){
return mergeSegmentsLimit;
}
})
.build();
merger.run(params);

1
server/src/test/java/com/metamx/druid/master/DruidMasterTest.java
View File

@ -188,4 +188,5 @@ public class DruidMasterTest
master.moveSegment("from", "to", "dummySegment", null);
}
}