diff --git a/common/src/main/java/io/druid/guice/DruidSecondaryModule.java b/common/src/main/java/io/druid/guice/DruidSecondaryModule.java index fbafb29d42b..7ace75f97a6 100644 --- a/common/src/main/java/io/druid/guice/DruidSecondaryModule.java +++ b/common/src/main/java/io/druid/guice/DruidSecondaryModule.java @@ -68,6 +68,7 @@ public class DruidSecondaryModule implements Module @Override public void configure(Binder binder) { + binder.requireExplicitBindings(); binder.install(new DruidGuiceExtensions()); binder.bind(Properties.class).toInstance(properties); binder.bind(ConfigurationObjectFactory.class).toInstance(factory); diff --git a/server/src/main/java/io/druid/server/initialization/Initialization.java b/server/src/main/java/io/druid/server/initialization/Initialization.java index 6de6612ae51..96b20ae077a 100644 --- a/server/src/main/java/io/druid/server/initialization/Initialization.java +++ b/server/src/main/java/io/druid/server/initialization/Initialization.java @@ -22,117 +22,126 @@ package io.druid.server.initialization; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.common.io.ByteStreams; -import com.google.inject.Binder; import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; import com.metamx.common.ISE; import com.metamx.common.logger.Logger; -import io.druid.guice.DruidGuiceExtensions; import io.druid.guice.DruidSecondaryModule; -import io.druid.guice.JsonConfigProvider; import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; import io.druid.initialization.DruidModule; -import io.druid.jackson.JacksonModule; import io.tesla.aether.TeslaAether; import io.tesla.aether.internal.DefaultTeslaAether; import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.collection.CollectRequest; +import org.eclipse.aether.graph.Dependency; +import org.eclipse.aether.graph.DependencyFilter; +import org.eclipse.aether.graph.DependencyNode; +import org.eclipse.aether.resolution.DependencyRequest; +import org.eclipse.aether.util.artifact.JavaScopes; +import org.eclipse.aether.util.filter.DependencyFilterUtils; import java.io.PrintStream; import java.net.URL; import java.net.URLClassLoader; -import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.ServiceLoader; +import java.util.Set; /** */ public class Initialization { private static final Logger log = new Logger(Initialization.class); + private static final Map loadersMap = Maps.newHashMap(); - private static final List exclusions = Arrays.asList( + private static final Set exclusions = Sets.newHashSet( "io.druid", "com.metamx.druid" ); - - public static Injector makeInjector(final Object... modules) - { - final Injector baseInjector = Guice.createInjector( - new DruidGuiceExtensions(), - new JacksonModule(), - new PropertiesModule("runtime.properties"), - new ConfigModule(), - new Module() - { - @Override - public void configure(Binder binder) - { - binder.bind(DruidSecondaryModule.class); - JsonConfigProvider.bind(binder, "druid.extensions", ExtensionsConfig.class); - - for (Object module : modules) { - if (module instanceof Class) { - binder.bind((Class) module); - } - } - } - } - ); - - ModuleList actualModules = new ModuleList(baseInjector); - actualModules.addModule(DruidSecondaryModule.class); - for (Object module : modules) { - actualModules.addModule(module); - } - - addExtensionModules(baseInjector.getInstance(ExtensionsConfig.class), actualModules); - - return Guice.createInjector(actualModules.getModules()); - } - - private static void addExtensionModules(ExtensionsConfig config, ModuleList actualModules) + public synchronized static List getFromExtensions(ExtensionsConfig config, Class clazz) { final TeslaAether aether = getAetherClient(config); + List retVal = Lists.newArrayList(); for (String coordinate : config.getCoordinates()) { log.info("Loading extension[%s]", coordinate); try { - final List artifacts = aether.resolveArtifacts(coordinate); - List urls = Lists.newArrayListWithExpectedSize(artifacts.size()); - for (Artifact artifact : artifacts) { - if (!exclusions.contains(artifact.getGroupId())) { - urls.add(artifact.getFile().toURI().toURL()); + ClassLoader loader = loadersMap.get(coordinate); + if (loader == null) { + final CollectRequest collectRequest = new CollectRequest(); + collectRequest.setRoot(new Dependency(new DefaultArtifact(coordinate), JavaScopes.RUNTIME)); + DependencyRequest dependencyRequest = new DependencyRequest( + collectRequest, + DependencyFilterUtils.andFilter( + DependencyFilterUtils.classpathFilter(JavaScopes.RUNTIME), + new DependencyFilter() + { + @Override + public boolean accept(DependencyNode node, List parents) + { + if (accept(node.getArtifact())) { + return false; + } + + for (DependencyNode parent : parents) { + if (accept(parent.getArtifact())) { + return false; + } + } + + return true; + } + + private boolean accept(final Artifact artifact) + { + return exclusions.contains(artifact.getGroupId()); + } + } + ) + ); + + final List artifacts = aether.resolveArtifacts(dependencyRequest); + List urls = Lists.newArrayListWithExpectedSize(artifacts.size()); + for (Artifact artifact : artifacts) { + if (!exclusions.contains(artifact.getGroupId())) { + urls.add(artifact.getFile().toURI().toURL()); + } + else { + log.error("Skipped Artifact[%s]", artifact); + } } - else { - log.debug("Skipped Artifact[%s]", artifact); + + for (URL url : urls) { + log.error("Added URL[%s]", url); } + + loader = new URLClassLoader(urls.toArray(new URL[urls.size()]), Initialization.class.getClassLoader()); + loadersMap.put(coordinate, loader); } - for (URL url : urls) { - log.debug("Added URL[%s]", url); - } + final ServiceLoader serviceLoader = ServiceLoader.load(clazz, loader); - ClassLoader loader = new URLClassLoader( - urls.toArray(new URL[urls.size()]), Initialization.class.getClassLoader() - ); - - final ServiceLoader serviceLoader = ServiceLoader.load(DruidModule.class, loader); - - for (DruidModule module : serviceLoader) { + for (T module : serviceLoader) { log.info("Adding extension module[%s]", module.getClass()); - actualModules.addModule(module); + retVal.add(module); } } catch (Exception e) { throw Throwables.propagate(e); } } + + return retVal; } private static DefaultTeslaAether getAetherClient(ExtensionsConfig config) @@ -162,6 +171,22 @@ public class Initialization } } + public static Injector makeInjectorWithModules(final Injector baseInjector, List modules) + { + ModuleList actualModules = new ModuleList(baseInjector); + actualModules.addModule(DruidSecondaryModule.class); + for (Object module : modules) { + actualModules.addModule(module); + } + + final ExtensionsConfig config = baseInjector.getInstance(ExtensionsConfig.class); + for (DruidModule module : Initialization.getFromExtensions(config, DruidModule.class)) { + actualModules.addModule(module); + } + + return Guice.createInjector(actualModules.getModules()); + } + private static class ModuleList { private final Injector baseInjector; @@ -217,5 +242,4 @@ public class Initialization return module; } } - - } +} diff --git a/services/src/main/java/io/druid/cli/CliBroker.java b/services/src/main/java/io/druid/cli/CliBroker.java index c843cbefc01..b88835e125b 100644 --- a/services/src/main/java/io/druid/cli/CliBroker.java +++ b/services/src/main/java/io/druid/cli/CliBroker.java @@ -19,7 +19,7 @@ package io.druid.cli; -import com.google.inject.Injector; +import com.google.common.collect.ImmutableList; import com.metamx.common.logger.Logger; import io.airlift.command.Command; import io.druid.client.cache.CacheMonitor; @@ -35,10 +35,11 @@ import io.druid.guice.annotations.Client; import io.druid.server.ClientQuerySegmentWalker; import io.druid.server.StatusResource; import io.druid.server.initialization.EmitterModule; -import io.druid.server.initialization.Initialization; import io.druid.server.initialization.JettyServerModule; import io.druid.server.metrics.MetricsModule; +import java.util.List; + /** */ @Command( @@ -55,22 +56,22 @@ public class CliBroker extends ServerRunnable } @Override - protected Injector getInjector() + protected List getModules() { - return Initialization.makeInjector( - new LifecycleModule(), - EmitterModule.class, - HttpClientModule.global(), - CuratorModule.class, - new MetricsModule().register(CacheMonitor.class), - new ServerModule(), - new JettyServerModule(new QueryJettyServerInitializer()) - .addResource(StatusResource.class), - new QueryableModule(ClientQuerySegmentWalker.class), - new QueryToolChestModule(), - new ServerViewModule(), - new HttpClientModule("druid.broker.http", Client.class), - new BrokerModule() + return ImmutableList.of( + new LifecycleModule(), + EmitterModule.class, + HttpClientModule.global(), + CuratorModule.class, + new MetricsModule().register(CacheMonitor.class), + new ServerModule(), + new JettyServerModule(new QueryJettyServerInitializer()) + .addResource(StatusResource.class), + new QueryableModule(ClientQuerySegmentWalker.class), + new QueryToolChestModule(), + new ServerViewModule(), + new HttpClientModule("druid.broker.http", Client.class), + new BrokerModule() ); } } diff --git a/services/src/main/java/io/druid/cli/CliCoordinator.java b/services/src/main/java/io/druid/cli/CliCoordinator.java index af435b77545..039284ab032 100644 --- a/services/src/main/java/io/druid/cli/CliCoordinator.java +++ b/services/src/main/java/io/druid/cli/CliCoordinator.java @@ -19,6 +19,7 @@ package io.druid.cli; +import com.google.common.collect.ImmutableList; import com.google.inject.Injector; import com.google.inject.servlet.GuiceFilter; import com.metamx.common.logger.Logger; @@ -39,7 +40,6 @@ import io.druid.server.http.InfoResource; import io.druid.server.http.MasterResource; import io.druid.server.http.RedirectFilter; import io.druid.server.initialization.EmitterModule; -import io.druid.server.initialization.Initialization; import io.druid.server.initialization.JettyServerInitializer; import io.druid.server.initialization.JettyServerModule; import io.druid.server.master.DruidMaster; @@ -55,6 +55,8 @@ import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.servlets.GzipFilter; +import java.util.List; + /** */ @Command( @@ -71,9 +73,9 @@ public class CliCoordinator extends ServerRunnable } @Override - protected Injector getInjector() + protected List getModules() { - return Initialization.makeInjector( + return ImmutableList.of( new LifecycleModule().register(DruidMaster.class), EmitterModule.class, HttpClientModule.global(), diff --git a/services/src/main/java/io/druid/cli/CliHistorical.java b/services/src/main/java/io/druid/cli/CliHistorical.java index 2035990d9c1..7b493b639f3 100644 --- a/services/src/main/java/io/druid/cli/CliHistorical.java +++ b/services/src/main/java/io/druid/cli/CliHistorical.java @@ -19,7 +19,7 @@ package io.druid.cli; -import com.google.inject.Injector; +import com.google.common.collect.ImmutableList; import com.metamx.common.logger.Logger; import io.airlift.command.Command; import io.druid.curator.CuratorModule; @@ -38,11 +38,12 @@ import io.druid.server.StatusResource; import io.druid.server.coordination.ServerManager; import io.druid.server.coordination.ZkCoordinator; import io.druid.server.initialization.EmitterModule; -import io.druid.server.initialization.Initialization; import io.druid.server.initialization.JettyServerModule; import io.druid.server.metrics.MetricsModule; import io.druid.server.metrics.ServerMonitor; +import java.util.List; + /** */ @Command( @@ -59,9 +60,9 @@ public class CliHistorical extends ServerRunnable } @Override - protected Injector getInjector() + protected List getModules() { - return Initialization.makeInjector( + return ImmutableList.of( new LifecycleModule().register(ZkCoordinator.class), EmitterModule.class, HttpClientModule.global(), diff --git a/services/src/main/java/io/druid/cli/CliMiddleManager.java b/services/src/main/java/io/druid/cli/CliMiddleManager.java index 16aec9d2460..7b40d812d41 100644 --- a/services/src/main/java/io/druid/cli/CliMiddleManager.java +++ b/services/src/main/java/io/druid/cli/CliMiddleManager.java @@ -19,6 +19,7 @@ package io.druid.cli; +import com.google.common.collect.ImmutableList; import com.google.inject.Injector; import com.google.inject.servlet.GuiceFilter; import com.metamx.common.logger.Logger; @@ -34,7 +35,6 @@ import io.druid.indexing.worker.WorkerTaskMonitor; import io.druid.indexing.worker.http.WorkerResource; import io.druid.server.StatusResource; import io.druid.server.initialization.EmitterModule; -import io.druid.server.initialization.Initialization; import io.druid.server.initialization.JettyServerInitializer; import io.druid.server.initialization.JettyServerModule; import io.druid.server.metrics.MetricsModule; @@ -47,6 +47,8 @@ import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.servlets.GzipFilter; +import java.util.List; + /** */ @Command( @@ -63,9 +65,9 @@ public class CliMiddleManager extends ServerRunnable } @Override - protected Injector getInjector() + protected List getModules() { - return Initialization.makeInjector( + return ImmutableList.of( new LifecycleModule().register(WorkerTaskMonitor.class), EmitterModule.class, HttpClientModule.global(), diff --git a/services/src/main/java/io/druid/cli/CliOverlord.java b/services/src/main/java/io/druid/cli/CliOverlord.java index 5c91c645488..a7842673c44 100644 --- a/services/src/main/java/io/druid/cli/CliOverlord.java +++ b/services/src/main/java/io/druid/cli/CliOverlord.java @@ -19,6 +19,7 @@ package io.druid.cli; +import com.google.common.collect.ImmutableList; import com.google.inject.Injector; import com.google.inject.servlet.GuiceFilter; import com.metamx.common.logger.Logger; @@ -38,7 +39,6 @@ import io.druid.indexing.coordinator.http.IndexerCoordinatorResource; import io.druid.server.StatusResource; import io.druid.server.http.RedirectFilter; import io.druid.server.initialization.EmitterModule; -import io.druid.server.initialization.Initialization; import io.druid.server.initialization.JettyServerInitializer; import io.druid.server.initialization.JettyServerModule; import io.druid.server.metrics.MetricsModule; @@ -54,6 +54,8 @@ import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.servlets.GzipFilter; import org.eclipse.jetty.util.resource.ResourceCollection; +import java.util.List; + /** */ @Command( @@ -70,9 +72,9 @@ public class CliOverlord extends ServerRunnable } @Override - protected Injector getInjector() + protected List getModules() { - return Initialization.makeInjector( + return ImmutableList.of( new LifecycleModule(), EmitterModule.class, HttpClientModule.global(), diff --git a/services/src/main/java/io/druid/cli/CliPeon.java b/services/src/main/java/io/druid/cli/CliPeon.java index 95ac2c42d4c..4147b249775 100644 --- a/services/src/main/java/io/druid/cli/CliPeon.java +++ b/services/src/main/java/io/druid/cli/CliPeon.java @@ -20,6 +20,8 @@ package io.druid.cli; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; import com.google.inject.Injector; import com.metamx.common.lifecycle.Lifecycle; import com.metamx.common.logger.Logger; @@ -70,34 +72,45 @@ public class CliPeon implements Runnable @Option(name = "--nodeType", title = "nodeType", description = "Set the node type to expose on ZK") public String nodeType = "indexer-executor"; + private Injector injector; + + @Inject + public void configure(Injector injector) + { + this.injector = injector; + } + private static final Logger log = new Logger(CliPeon.class); protected Injector getInjector() { - return Initialization.makeInjector( - new LifecycleModule(), - EmitterModule.class, - HttpClientModule.global(), - CuratorModule.class, - new MetricsModule(), - new ServerModule(), - new JettyServerModule(new QueryJettyServerInitializer()) - .addResource(StatusResource.class) - .addResource(ChatHandlerResource.class), - new DiscoveryModule(), - new ServerViewModule(), - new StorageNodeModule(nodeType), - new DataSegmentPusherModule(), - new AnnouncerModule(), - new DruidProcessingModule(), - new QueryableModule(ThreadPoolTaskRunner.class), - new QueryRunnerFactoryModule(), - new IndexingServiceDiscoveryModule(), - new AWSModule(), - new PeonModule( - new ExecutorLifecycleConfig() - .setTaskFile(new File(taskAndStatusFile.get(0))) - .setStatusFile(new File(taskAndStatusFile.get(1))) + return Initialization.makeInjectorWithModules( + injector, + ImmutableList.of( + new LifecycleModule(), + EmitterModule.class, + HttpClientModule.global(), + CuratorModule.class, + new MetricsModule(), + new ServerModule(), + new JettyServerModule(new QueryJettyServerInitializer()) + .addResource(StatusResource.class) + .addResource(ChatHandlerResource.class), + new DiscoveryModule(), + new ServerViewModule(), + new StorageNodeModule(nodeType), + new DataSegmentPusherModule(), + new AnnouncerModule(), + new DruidProcessingModule(), + new QueryableModule(ThreadPoolTaskRunner.class), + new QueryRunnerFactoryModule(), + new IndexingServiceDiscoveryModule(), + new AWSModule(), + new PeonModule( + new ExecutorLifecycleConfig() + .setTaskFile(new File(taskAndStatusFile.get(0))) + .setStatusFile(new File(taskAndStatusFile.get(1))) + ) ) ); } diff --git a/services/src/main/java/io/druid/cli/CliRealtime.java b/services/src/main/java/io/druid/cli/CliRealtime.java index fa07364339c..c6eab365cba 100644 --- a/services/src/main/java/io/druid/cli/CliRealtime.java +++ b/services/src/main/java/io/druid/cli/CliRealtime.java @@ -19,7 +19,7 @@ package io.druid.cli; -import com.google.inject.Injector; +import com.google.common.collect.ImmutableList; import com.metamx.common.logger.Logger; import io.airlift.command.Command; import io.druid.curator.CuratorModule; @@ -39,10 +39,11 @@ import io.druid.guice.StorageNodeModule; import io.druid.segment.realtime.RealtimeManager; import io.druid.server.StatusResource; import io.druid.server.initialization.EmitterModule; -import io.druid.server.initialization.Initialization; import io.druid.server.initialization.JettyServerModule; import io.druid.server.metrics.MetricsModule; +import java.util.List; + /** */ @Command( @@ -59,9 +60,9 @@ public class CliRealtime extends ServerRunnable } @Override - protected Injector getInjector() + protected List getModules() { - return Initialization.makeInjector( + return ImmutableList.of( new LifecycleModule(), EmitterModule.class, DbConnectorModule.class, diff --git a/services/src/main/java/io/druid/cli/CliRealtimeExample.java b/services/src/main/java/io/druid/cli/CliRealtimeExample.java index 52b258898c3..adc6f78ccba 100644 --- a/services/src/main/java/io/druid/cli/CliRealtimeExample.java +++ b/services/src/main/java/io/druid/cli/CliRealtimeExample.java @@ -19,7 +19,7 @@ package io.druid.cli; -import com.google.inject.Injector; +import com.google.common.collect.ImmutableList; import com.metamx.common.logger.Logger; import druid.examples.guice.RealtimeExampleModule; import io.airlift.command.Command; @@ -32,9 +32,10 @@ import io.druid.guice.StorageNodeModule; import io.druid.segment.realtime.RealtimeManager; import io.druid.server.StatusResource; import io.druid.server.initialization.EmitterModule; -import io.druid.server.initialization.Initialization; import io.druid.server.initialization.JettyServerModule; +import java.util.List; + /** */ @Command( @@ -51,9 +52,9 @@ public class CliRealtimeExample extends ServerRunnable } @Override - protected Injector getInjector() + protected List getModules() { - return Initialization.makeInjector( + return ImmutableList.of( new LifecycleModule(), EmitterModule.class, DruidProcessingModule.class, diff --git a/services/src/main/java/io/druid/cli/Main.java b/services/src/main/java/io/druid/cli/Main.java index f7a264f6b4d..1f27ed215d3 100644 --- a/services/src/main/java/io/druid/cli/Main.java +++ b/services/src/main/java/io/druid/cli/Main.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -19,9 +19,23 @@ package io.druid.cli; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Module; import io.airlift.command.Cli; import io.airlift.command.Help; import io.airlift.command.ParseException; +import io.druid.guice.DruidGuiceExtensions; +import io.druid.guice.DruidSecondaryModule; +import io.druid.guice.JsonConfigProvider; +import io.druid.jackson.JacksonModule; +import io.druid.server.initialization.ConfigModule; +import io.druid.server.initialization.ExtensionsConfig; +import io.druid.server.initialization.Initialization; +import io.druid.server.initialization.PropertiesModule; + +import java.util.List; /** */ @@ -54,9 +68,19 @@ public class Main .withDefaultCommand(Help.class) .withCommands(CliPeon.class); + final Injector injector = makeStartupInjector(); + final ExtensionsConfig config = injector.getInstance(ExtensionsConfig.class); + final List extensionCommands = Initialization.getFromExtensions(config, CliCommandCreator.class); + + for (CliCommandCreator creator : extensionCommands) { + creator.addCommands(builder); + } + final Cli cli = builder.build(); try { - cli.parse(args).run(); + final Runnable command = cli.parse(args); + injector.injectMembers(command); + command.run(); } catch (ParseException e) { System.out.println("ERROR!!!!"); @@ -65,4 +89,23 @@ public class Main cli.parse(new String[]{"help"}).run(); } } + + public static Injector makeStartupInjector() + { + return Guice.createInjector( + new DruidGuiceExtensions(), + new JacksonModule(), + new PropertiesModule("runtime.properties"), + new ConfigModule(), + new Module() + { + @Override + public void configure(Binder binder) + { + binder.bind(DruidSecondaryModule.class); + JsonConfigProvider.bind(binder, "druid.extensions", ExtensionsConfig.class); + } + } + ); + } } diff --git a/services/src/main/java/io/druid/cli/ServerRunnable.java b/services/src/main/java/io/druid/cli/ServerRunnable.java index a7ba9febbbd..aa2508d6007 100644 --- a/services/src/main/java/io/druid/cli/ServerRunnable.java +++ b/services/src/main/java/io/druid/cli/ServerRunnable.java @@ -20,10 +20,14 @@ package io.druid.cli; import com.google.common.base.Throwables; +import com.google.inject.Inject; import com.google.inject.Injector; import com.metamx.common.lifecycle.Lifecycle; import com.metamx.common.logger.Logger; import io.druid.initialization.LogLevelAdjuster; +import io.druid.server.initialization.Initialization; + +import java.util.List; /** */ @@ -31,12 +35,20 @@ public abstract class ServerRunnable implements Runnable { private final Logger log; + private Injector baseInjector; + public ServerRunnable(Logger log) { this.log = log; } - protected abstract Injector getInjector(); + @Inject + public void configure(Injector injector) + { + this.baseInjector = injector; + } + + protected abstract List getModules(); @Override public void run() @@ -44,7 +56,8 @@ public abstract class ServerRunnable implements Runnable try { LogLevelAdjuster.register(); - final Lifecycle lifecycle = getInjector().getInstance(Lifecycle.class); + final Injector injector = Initialization.makeInjectorWithModules(baseInjector, getModules()); + final Lifecycle lifecycle = injector.getInstance(Lifecycle.class); try { lifecycle.start();