Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'druid-0.7.x' into offheap-incremental-index

This commit is contained in:
fjy 2014-08-18 11:14:19 -07:00
parent 8fabc541f0 74b8adf5b0
commit 77e514688a
16 changed files with 175 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
indexing-service/src/main/java/io/druid/indexing/overlord/config/ForkingTaskRunnerConfig.java
View File

@ -49,7 +49,7 @@ public class ForkingTaskRunnerConfig
@JsonProperty
@Min(1024)
@Max(65535)
private int startPort = 8081;
private int startPort = 8100;
@JsonProperty
@NotNull

21
server/src/main/java/io/druid/server/DruidNode.java
View File

@ -19,8 +19,10 @@
package io.druid.server;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.inject.name.Named;
import io.druid.common.utils.SocketUtil;
import javax.validation.constraints.Max;
@ -31,15 +33,17 @@ import javax.validation.constraints.NotNull;
*/
public class DruidNode
{
public static final String DEFAULT_HOST = "localhost";
private String hostNoPort;
@JsonProperty("service")
@NotNull
private String serviceName = null;
private String serviceName;
@JsonProperty
@NotNull
private String host = null;
private String host;
@JsonProperty
@Min(0) @Max(0xffff)
@ -47,16 +51,21 @@ public class DruidNode
@JsonCreator
public DruidNode(
@JsonProperty("service") String serviceName,
@JacksonInject @Named("serviceName") @JsonProperty("service") String serviceName,
@JsonProperty("host") String host,
@JsonProperty("port") Integer port
@JacksonInject @Named("servicePort") @JsonProperty("port") Integer port
)
{
init(serviceName, host, port);
}
private void init(String serviceName, String host, Integer port)
{
this.serviceName = serviceName;
if (port == null) {
if (host == null) {
setHostAndPort(null, -1, null);
setHostAndPort(DEFAULT_HOST, -1, DEFAULT_HOST);
}
else if (host.contains(":")) {
final String[] hostParts = host.split(":");
@ -74,7 +83,7 @@ public class DruidNode
}
else {
if (host == null || host.contains(":")) {
setHostAndPort(host, port, host == null ? null : host.split(":")[0]);
setHostAndPort(host == null ? DEFAULT_HOST : host, port, host == null ? DEFAULT_HOST : host.split(":")[0]);
}
else {
setHostAndPort(String.format("%s:%d", host, port), port, host);

2
server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java
View File

@ -454,7 +454,7 @@ public class DruidCoordinator
private LeaderLatch createNewLeaderLatch()
{
final LeaderLatch newLeaderLatch = new LeaderLatch(
curator, ZKPaths.makePath(zkPaths.getCoordinatorPath(), COORDINATOR_OWNER_NODE), config.getHost()
curator, ZKPaths.makePath(zkPaths.getCoordinatorPath(), COORDINATOR_OWNER_NODE), self.getHost()
);
newLeaderLatch.addListener(

3
server/src/main/java/io/druid/server/coordinator/DruidCoordinatorConfig.java
View File

@ -27,9 +27,6 @@ import org.skife.config.Default;
*/
public abstract class DruidCoordinatorConfig
{
@Config("druid.host")
public abstract String getHost();
@Config("druid.coordinator.startDelay")
@Default("PT300s")
public abstract Duration getCoordinatorStartDelay();

2
server/src/test/java/io/druid/initialization/InitializationTest.java
View File

@ -109,7 +109,7 @@ public class InitializationTest
public void configure(Binder binder)
{
JsonConfigProvider.bindInstance(
binder, Key.get(DruidNode.class, Self.class), new DruidNode("hadoop-indexer", "localhost", -1)
binder, Key.get(DruidNode.class, Self.class), new DruidNode("test-inject", "localhost", -1)
);
}
}

6
server/src/test/java/io/druid/server/coordinator/DruidCoordinatorTest.java
View File

@ -77,12 +77,6 @@ public class DruidCoordinatorTest
coordinator = new DruidCoordinator(
new DruidCoordinatorConfig()
{
@Override
public String getHost()
{
return null;
}
@Override
public Duration getCoordinatorStartDelay()
{

4
services/src/main/java/io/druid/cli/CliBridge.java
View File

@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Module;
import com.google.inject.Provides;
import com.google.inject.name.Names;
import com.metamx.common.lifecycle.Lifecycle;
import com.metamx.common.logger.Logger;
import io.airlift.command.Command;
@ -69,6 +70,9 @@ public class CliBridge extends ServerRunnable
@Override
public void configure(Binder binder)
{
binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/bridge");
binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8081);
ConfigProvider.bind(binder, BridgeCuratorConfig.class);
binder.bind(BridgeZkCoordinator.class).in(ManageLifecycle.class);

4
services/src/main/java/io/druid/cli/CliBroker.java
View File

@ -22,6 +22,7 @@ package io.druid.cli;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Module;
import com.google.inject.name.Names;
import com.metamx.common.logger.Logger;
import io.airlift.command.Command;
import io.druid.client.BrokerServerView;
@ -78,6 +79,9 @@ public class CliBroker extends ServerRunnable
@Override
public void configure(Binder binder)
{
binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/broker");
binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8082);
binder.bind(QueryToolChestWarehouse.class).to(MapQueryToolChestWarehouse.class);
binder.bind(CachingClusteredClient.class).in(LazySingleton.class);

4
services/src/main/java/io/druid/cli/CliCoordinator.java
View File

@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Module;
import com.google.inject.Provides;
import com.google.inject.name.Names;
import com.metamx.common.concurrent.ScheduledExecutorFactory;
import com.metamx.common.logger.Logger;
import io.airlift.command.Command;
@ -87,6 +88,9 @@ public class CliCoordinator extends ServerRunnable
@Override
public void configure(Binder binder)
{
binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/coordinator");
binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8081);
ConfigProvider.bind(binder, DruidCoordinatorConfig.class);
JsonConfigProvider.bind(binder, "druid.manager.segments", DatabaseSegmentManagerConfig.class);

4
services/src/main/java/io/druid/cli/CliHistorical.java
View File

@ -22,6 +22,7 @@ package io.druid.cli;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Module;
import com.google.inject.name.Names;
import com.metamx.common.logger.Logger;
import io.airlift.command.Command;
import io.druid.client.cache.Cache;
@ -68,6 +69,9 @@ public class CliHistorical extends ServerRunnable
@Override
public void configure(Binder binder)
{
binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/historical");
binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8083);
binder.bind(ServerManager.class).in(LazySingleton.class);
binder.bind(ZkCoordinator.class).in(ManageLifecycle.class);
binder.bind(QuerySegmentWalker.class).to(ServerManager.class).in(LazySingleton.class);

4
services/src/main/java/io/druid/cli/CliMiddleManager.java
View File

@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Module;
import com.google.inject.Provides;
import com.google.inject.name.Names;
import com.google.inject.util.Providers;
import com.metamx.common.logger.Logger;
import io.airlift.command.Command;
@ -74,6 +75,9 @@ public class CliMiddleManager extends ServerRunnable
@Override
public void configure(Binder binder)
{
binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/middlemanager");
binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8091);
IndexingServiceModuleHelper.configureTaskRunnerConfigs(binder);
JsonConfigProvider.bind(binder, "druid.indexer.task", TaskConfig.class);

4
services/src/main/java/io/druid/cli/CliOverlord.java
View File

@ -26,6 +26,7 @@ import com.google.inject.Key;
import com.google.inject.Module;
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.MapBinder;
import com.google.inject.name.Names;
import com.google.inject.servlet.GuiceFilter;
import com.google.inject.util.Providers;
import com.metamx.common.logger.Logger;
@ -114,6 +115,9 @@ public class CliOverlord extends ServerRunnable
@Override
public void configure(Binder binder)
{
binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/overlord");
binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8090);
JsonConfigProvider.bind(binder, "druid.indexer.queue", TaskQueueConfig.class);
JsonConfigProvider.bind(binder, "druid.indexer.task", TaskConfig.class);

13
services/src/main/java/io/druid/cli/CliRealtime.java
View File

@ -20,6 +20,9 @@
package io.druid.cli;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Module;
import com.google.inject.name.Names;
import com.metamx.common.logger.Logger;
import io.airlift.command.Command;
import io.druid.guice.RealtimeModule;
@ -45,7 +48,15 @@ public class CliRealtime extends ServerRunnable
protected List<Object> getModules()
{
return ImmutableList.<Object>of(
new RealtimeModule()
new RealtimeModule(),
new Module() {
@Override
public void configure(Binder binder)
{
binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/realtime");
binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8084);
}
}
);
}
}

4
services/src/main/java/io/druid/cli/CliRouter.java
View File

@ -24,6 +24,7 @@ import com.google.inject.Binder;
import com.google.inject.Module;
import com.google.inject.Provides;
import com.google.inject.TypeLiteral;
import com.google.inject.name.Names;
import com.metamx.common.logger.Logger;
import io.airlift.command.Command;
import io.druid.curator.discovery.DiscoveryModule;
@ -72,6 +73,9 @@ public class CliRouter extends ServerRunnable
@Override
public void configure(Binder binder)
{
binder.bindConstant().annotatedWith(Names.named("serviceName")).to("druid/router");
binder.bindConstant().annotatedWith(Names.named("servicePort")).to(8888);
JsonConfigProvider.bind(binder, "druid.router", TieredBrokerConfig.class);
binder.bind(CoordinatorRuleManager.class);

116
services/src/main/java/io/druid/cli/CreateTables.java Normal file
View File

@ -0,0 +1,116 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.cli;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Module;
import com.metamx.common.logger.Logger;
import io.airlift.command.Command;
import io.airlift.command.Option;
import io.druid.db.DbConnector;
import io.druid.db.DbConnectorConfig;
import io.druid.db.DbTablesConfig;
import io.druid.guice.JsonConfigProvider;
import io.druid.guice.annotations.Self;
import io.druid.server.DruidNode;
import java.util.List;
@Command(
name = "metadata-init",
description = "Initialize Metadata Storage"
)
public class CreateTables extends GuiceRunnable
{
@Option(name = "--connectURI", description = "Database JDBC connection string", required = true)
private String connectURI;
@Option(name = "--user", description = "Database username", required = true)
private String user;
@Option(name = "--password", description = "Database password", required = true)
private String password;
@Option(name = "--base", description = "Base table name")
private String base;
private static final Logger log = new Logger(CreateTables.class);
public CreateTables()
{
super(log);
}
@Override
protected List<Object> getModules()
{
return ImmutableList.<Object>of(
new Module()
{
@Override
public void configure(Binder binder)
{
JsonConfigProvider.bindInstance(
binder, Key.get(DbConnectorConfig.class), new DbConnectorConfig()
{
@Override
public String getConnectURI()
{
return connectURI;
}
@Override
public String getUser()
{
return user;
}
@Override
public String getPassword()
{
return password;
}
}
);
JsonConfigProvider.bindInstance(
binder, Key.get(DbTablesConfig.class), DbTablesConfig.fromBase(base)
);
JsonConfigProvider.bindInstance(
binder, Key.get(DruidNode.class, Self.class), new DruidNode("tools", "localhost", -1)
);
}
}
);
}
@Override
public void run()
{
final Injector injector = makeInjector();
DbConnector dbConnector = injector.getInstance(DbConnector.class);
dbConnector.createSegmentTable();
dbConnector.createRulesTable();
dbConnector.createConfigTable();
dbConnector.createTaskTables();
}
}

2
services/src/main/java/io/druid/cli/Main.java
View File

@ -61,7 +61,7 @@ public class Main
builder.withGroup("tools")
.withDescription("Various tools for working with Druid")
.withDefaultCommand(Help.class)
.withCommands(ConvertProperties.class, DruidJsonValidator.class, PullDependencies.class);
.withCommands(ConvertProperties.class, DruidJsonValidator.class, PullDependencies.class, CreateTables.class);
builder.withGroup("index")
.withDescription("Run indexing for druid")