Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Tidy up construction of the Guice Injectors (#12816) 

* Refactor Guice initialization

Builders for various module collections
Revise the extensions loader
Injector builders for server startup
Move Hadoop init to indexer
Clean up server node role filtering
Calcite test injector builder

* Revisions from review comments

* Build fixes

* Revisions from review comments
This commit is contained in:
Paul Rogers 2022-08-04 00:05:07 -07:00 committed by GitHub
parent ef6811ef88
commit a618458bf0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
59 changed files with 2168 additions and 1529 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java
View File

@ -22,16 +22,12 @@ package org.apache.druid.common.aws;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.AmazonEC2Client;
import com.fasterxml.jackson.databind.Module;
import com.google.inject.Binder;
import com.google.inject.Provides;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.guice.LazySingleton;
import org.apache.druid.initialization.DruidModule;
import java.util.Collections;
import java.util.List;
public class AWSModule implements DruidModule
{
@Override
@ -56,10 +52,4 @@ public class AWSModule implements DruidModule
{
return new AmazonEC2Client(credentials);
}
@Override
public List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
}

8
cloud/gcp-common/pom.xml
View File

@ -52,18 +52,10 @@
<artifactId>jackson-module-guice</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>

10
cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java
View File

@ -19,14 +19,12 @@
package org.apache.druid.common.gcp;
import com.fasterxml.jackson.databind.Module;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Provides;
import org.apache.druid.guice.LazySingleton;
@ -35,23 +33,15 @@ import org.apache.druid.initialization.DruidModule;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.Collections;
import java.util.List;
public class GcpModule implements DruidModule
{
@Override
public List<? extends Module> getJacksonModules()
{
return ImmutableList.of();
}
@Override
public void configure(Binder binder)
{
// Nothing to proactively bind
}
@Provides
@LazySingleton
public HttpRequestInitializer getHttpRequestInitializer(HttpTransport transport, JsonFactory factory)

12
cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpMockModule.java
View File

@ -19,36 +19,24 @@
package org.apache.druid.common.gcp;
import com.fasterxml.jackson.databind.Module;
import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.testing.http.MockHttpTransport;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Provides;
import org.apache.druid.guice.LazySingleton;
import org.apache.druid.initialization.DruidModule;
import java.util.List;
public class GcpMockModule implements DruidModule
{
@Override
public List<? extends Module> getJacksonModules()
{
return ImmutableList.of();
}
@Override
public void configure(Binder binder)
{
}
@Provides
@LazySingleton
public HttpRequestInitializer mockRequestInitializer(

12
core/src/main/java/org/apache/druid/initialization/DruidModule.java
View File

@ -22,12 +22,22 @@ package org.apache.druid.initialization;
import com.fasterxml.jackson.databind.Module;
import org.apache.druid.guice.annotations.ExtensionPoint;
import java.util.Collections;
import java.util.List;
/**
* A Guice module which also provides Jackson modules.
* Extension modules must implement this interface.
* (Enforced in {@code ExtensionInjectorBuilder}).
* Built-in implementations that do not provide Jackson modules can
* implement the simpler {@link com.google.inject.Module Guice Module}
* interface instead.
*/
@ExtensionPoint
public interface DruidModule extends com.google.inject.Module
{
List<? extends Module> getJacksonModules();
default List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
}

7
extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java
View File

@ -1172,16 +1172,9 @@ public class S3InputSourceTest extends InitializedNullHandlingTest
return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(null);
}
@Override
public List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
@Override
public void configure(Binder binder)
{
}
}
);

10
extensions-core/simple-client-sslcontext/src/main/java/org/apache/druid/https/SSLContextModule.java
View File

@ -19,8 +19,6 @@
package org.apache.druid.https;
import com.fasterxml.jackson.databind.Module;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.guice.annotations.Client;
@ -31,17 +29,9 @@ import org.apache.druid.initialization.DruidModule;
import org.apache.druid.server.router.Router;
import javax.net.ssl.SSLContext;
import java.util.List;
public class SSLContextModule implements DruidModule
{
@Override
public List<? extends Module> getJacksonModules()
{
return ImmutableList.of();
}
@Override
public void configure(Binder binder)
{

10
extensions-core/testing-tools/src/main/java/org/apache/druid/guice/SleepModule.java
View File

@ -19,24 +19,14 @@
package org.apache.druid.guice;
import com.fasterxml.jackson.databind.Module;
import com.google.inject.Binder;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.query.expressions.SleepExprMacro;
import org.apache.druid.query.sql.SleepOperatorConversion;
import org.apache.druid.sql.guice.SqlBindings;
import java.util.Collections;
import java.util.List;
public class SleepModule implements DruidModule
{
@Override
public List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
@Override
public void configure(Binder binder)
{

4
indexing-service/pom.xml
View File

@ -212,6 +212,10 @@
<artifactId>commons-collections4</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.eclipse.aether</groupId>
<artifactId>aether-api</artifactId>
</dependency>
<!-- Tests -->
<dependency>
<groupId>junit</groupId>

28
indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopTask.java
View File

@ -25,13 +25,14 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.inject.Injector;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.guice.GuiceInjectors;
import org.apache.druid.guice.ExtensionsLoader;
import org.apache.druid.guice.StartupInjectorBuilder;
import org.apache.druid.indexing.common.TaskToolbox;
import org.apache.druid.initialization.Initialization;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.utils.JvmUtils;
import javax.annotation.Nullable;
import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@ -48,13 +49,9 @@ import java.util.Map;
public abstract class HadoopTask extends AbstractBatchIndexTask
{
private static final Logger log = new Logger(HadoopTask.class);
private static final ExtensionsConfig EXTENSIONS_CONFIG;
static final Injector INJECTOR = GuiceInjectors.makeStartupInjector();
static {
EXTENSIONS_CONFIG = INJECTOR.getInstance(ExtensionsConfig.class);
}
static final Injector INJECTOR = new StartupInjectorBuilder().withExtensions().build();
private static final ExtensionsLoader EXTENSIONS_LOADER = ExtensionsLoader.instance(INJECTOR);
private final List<String> hadoopDependencyCoordinates;
@ -152,8 +149,8 @@ public abstract class HadoopTask extends AbstractBatchIndexTask
}
final List<URL> extensionURLs = new ArrayList<>();
for (final File extension : Initialization.getExtensionFilesToLoad(EXTENSIONS_CONFIG)) {
final URLClassLoader extensionLoader = Initialization.getClassLoaderForExtension(extension, false);
for (final File extension : EXTENSIONS_LOADER.getExtensionFilesToLoad()) {
final URLClassLoader extensionLoader = EXTENSIONS_LOADER.getClassLoaderForExtension(extension, false);
extensionURLs.addAll(Arrays.asList(extensionLoader.getURLs()));
}
@ -165,9 +162,9 @@ public abstract class HadoopTask extends AbstractBatchIndexTask
for (final File hadoopDependency :
Initialization.getHadoopDependencyFilesToLoad(
finalHadoopDependencyCoordinates,
EXTENSIONS_CONFIG
EXTENSIONS_LOADER.config()
)) {
final URLClassLoader hadoopLoader = Initialization.getClassLoaderForExtension(hadoopDependency, false);
final URLClassLoader hadoopLoader = EXTENSIONS_LOADER.getClassLoaderForExtension(hadoopDependency, false);
localClassLoaderURLs.addAll(Arrays.asList(hadoopLoader.getURLs()));
}
@ -187,15 +184,16 @@ public abstract class HadoopTask extends AbstractBatchIndexTask
);
final String hadoopContainerDruidClasspathJars;
if (EXTENSIONS_CONFIG.getHadoopContainerDruidClasspath() == null) {
ExtensionsConfig extnConfig = EXTENSIONS_LOADER.config();
if (extnConfig.getHadoopContainerDruidClasspath() == null) {
hadoopContainerDruidClasspathJars = Joiner.on(File.pathSeparator).join(jobURLs);
} else {
List<URL> hadoopContainerURLs = Lists.newArrayList(
Initialization.getURLsForClasspath(EXTENSIONS_CONFIG.getHadoopContainerDruidClasspath())
ExtensionsLoader.getURLsForClasspath(extnConfig.getHadoopContainerDruidClasspath())
);
if (EXTENSIONS_CONFIG.getAddExtensionsToHadoopContainer()) {
if (extnConfig.getAddExtensionsToHadoopContainer()) {
hadoopContainerURLs.addAll(extensionURLs);
}

62
indexing-service/src/main/java/org/apache/druid/indexing/common/task/Initialization.java Normal file
View File

@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.indexing.common.task;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.java.util.common.ISE;
import org.eclipse.aether.artifact.DefaultArtifact;
import java.io.File;
import java.util.List;
public class Initialization
{
/**
* Find all the Hadoop dependencies that should be loaded by Druid.
*
* @param hadoopDependencyCoordinates e.g.["org.apache.hadoop:hadoop-client:2.3.0"]
* @param extensionsConfig ExtensionsConfig configured by druid.extensions.xxx
*
* @return an array of Hadoop dependency files that will be loaded by the Druid process.
*/
public static File[] getHadoopDependencyFilesToLoad(
List<String> hadoopDependencyCoordinates,
ExtensionsConfig extensionsConfig
)
{
final File rootHadoopDependenciesDir = new File(extensionsConfig.getHadoopDependenciesDir());
if (rootHadoopDependenciesDir.exists() && !rootHadoopDependenciesDir.isDirectory()) {
throw new ISE("Root Hadoop dependencies directory [%s] is not a directory!?", rootHadoopDependenciesDir);
}
final File[] hadoopDependenciesToLoad = new File[hadoopDependencyCoordinates.size()];
int i = 0;
for (final String coordinate : hadoopDependencyCoordinates) {
final DefaultArtifact artifact = new DefaultArtifact(coordinate);
final File hadoopDependencyDir = new File(rootHadoopDependenciesDir, artifact.getArtifactId());
final File versionDir = new File(hadoopDependencyDir, artifact.getVersion());
// find the hadoop dependency with the version specified in coordinate
if (!hadoopDependencyDir.isDirectory() || !versionDir.isDirectory()) {
throw new ISE("Hadoop dependency [%s] didn't exist!?", versionDir.getAbsolutePath());
}
hadoopDependenciesToLoad[i++] = versionDir;
}
return hadoopDependenciesToLoad;
}
}

2
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java
View File

@ -140,7 +140,7 @@ public class ParallelIndexSupervisorTask extends AbstractBatchIndexTask implemen
// reproduce but looking at the code around where the following constant is used one
// possibility is that the sketch's estimate is negative. If that case happens
// code has been added to log it and to set the estimate to the value of the
// following constant. It is not necessary to parametize this value since if this
// following constant. It is not necessary to parameterize this value since if this
// happens it is a bug and the new logging may now provide some evidence to reproduce
// and fix
private static final long DEFAULT_NUM_SHARDS_WHEN_ESTIMATE_GOES_NEGATIVE = 7L;

95
indexing-service/src/test/java/org/apache/druid/indexing/common/task/InitializationTest.java Normal file
View File

@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.indexing.common.task;
import com.google.common.collect.ImmutableList;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.java.util.common.ISE;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
public class InitializationTest
{
@Rule
public final TemporaryFolder temporaryFolder = new TemporaryFolder();
@Test(expected = ISE.class)
public void testGetHadoopDependencyFilesToLoad_wrong_type_root_hadoop_depenencies_dir() throws IOException
{
final File rootHadoopDependenciesDir = temporaryFolder.newFile();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getHadoopDependenciesDir()
{
return rootHadoopDependenciesDir.getAbsolutePath();
}
};
Initialization.getHadoopDependencyFilesToLoad(ImmutableList.of(), config);
}
@Test(expected = ISE.class)
public void testGetHadoopDependencyFilesToLoad_non_exist_version_dir() throws IOException
{
final File rootHadoopDependenciesDir = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getHadoopDependenciesDir()
{
return rootHadoopDependenciesDir.getAbsolutePath();
}
};
final File hadoopClient = new File(rootHadoopDependenciesDir, "hadoop-client");
hadoopClient.mkdir();
Initialization.getHadoopDependencyFilesToLoad(ImmutableList.of("org.apache.hadoop:hadoop-client:2.3.0"), config);
}
@Test
public void testGetHadoopDependencyFilesToLoad_with_hadoop_coordinates() throws IOException
{
final File rootHadoopDependenciesDir = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getHadoopDependenciesDir()
{
return rootHadoopDependenciesDir.getAbsolutePath();
}
};
final File hadoopClient = new File(rootHadoopDependenciesDir, "hadoop-client");
final File versionDir = new File(hadoopClient, "2.3.0");
hadoopClient.mkdir();
versionDir.mkdir();
final File[] expectedFileList = new File[]{versionDir};
final File[] actualFileList = Initialization.getHadoopDependencyFilesToLoad(
ImmutableList.of(
"org.apache.hadoop:hadoop-client:2.3.0"
), config
);
Assert.assertArrayEquals(expectedFileList, actualFileList);
}
}

1
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java
View File

@ -923,7 +923,6 @@ public class AbstractParallelIndexSupervisorTaskTest extends IngestionTestBase
this.transientApiCallFailureRate = transientApiCallFailureRate;
}
@Override
public ParallelIndexSupervisorTaskClient build(String supervisorTaskId, Duration httpTimeout, long numRetries)
{

3
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/MultiPhaseParallelIndexingRowStatsTest.java
View File

@ -33,6 +33,7 @@ import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.incremental.RowIngestionMetersTotals;
import org.joda.time.Interval;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
@ -107,6 +108,7 @@ public class MultiPhaseParallelIndexingRowStatsTest extends AbstractMultiPhasePa
}
@Test
@Ignore("assumes record rates, to be fixed PR #12852")
public void testHashPartitionRowStats_concurrentSubTasks_1()
{
testHashPartitionRowStats(1);
@ -175,5 +177,4 @@ public class MultiPhaseParallelIndexingRowStatsTest extends AbstractMultiPhasePa
Map<String, Object> actualReports = runTaskAndGetReports(task, TaskState.SUCCESS);
compareTaskReports(expectedReports, actualReports);
}
}

11
integration-tests/src/main/java/org/apache/druid/testing/guice/ITTLSCertificateCheckerModule.java
View File

@ -19,16 +19,12 @@
package org.apache.druid.testing.guice;
import com.fasterxml.jackson.databind.Module;
import com.google.inject.Binder;
import com.google.inject.name.Names;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.server.security.TLSCertificateChecker;
import org.apache.druid.testing.utils.ITTLSCertificateChecker;
import java.util.Collections;
import java.util.List;
public class ITTLSCertificateCheckerModule implements DruidModule
{
private final ITTLSCertificateChecker INSTANCE = new ITTLSCertificateChecker();
@ -42,11 +38,4 @@ public class ITTLSCertificateCheckerModule implements DruidModule
.annotatedWith(Names.named(IT_CHECKER_TYPE))
.toInstance(INSTANCE);
}
@Override
public List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
}

68
processing/src/main/java/org/apache/druid/guice/BaseInjectorBuilder.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.guice;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* Utilities for building a Guice injector. Defined as a parameterized
* type so that this class can be used in derived fluent builders.
*/
public class BaseInjectorBuilder<T extends BaseInjectorBuilder<?>>
{
private final List<Module> modules = new ArrayList<>();
@SuppressWarnings("unchecked")
public T add(Module... modules)
{
// Done this way because IntelliJ inspections complains if we
// try to iterate, because it thinks addAll() accepts an array,
// which it does not.
this.modules.addAll(Arrays.asList(modules));
return (T) this;
}
@SuppressWarnings("unchecked")
public T addAll(List<Module> modules)
{
this.modules.addAll(modules);
return (T) this;
}
@SuppressWarnings("unchecked")
public T addAll(Iterable<? extends Module> modules)
{
for (Module m : modules) {
this.modules.add(m);
}
return (T) this;
}
public Injector build()
{
return Guice.createInjector(modules);
}
}

2
server/src/main/java/org/apache/druid/initialization/ExtensionFirstClassLoader.java → processing/src/main/java/org/apache/druid/guice/ExtensionFirstClassLoader.java
View File

@ -17,7 +17,7 @@
* under the License.
*/
package org.apache.druid.initialization;
package org.apache.druid.guice;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;

337
processing/src/main/java/org/apache/druid/guice/ExtensionsLoader.java Normal file
View File

@ -0,0 +1,337 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.guice;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Injector;
import org.apache.commons.io.FileUtils;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.logger.Logger;
import javax.inject.Inject;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* Manages the loading of Druid extensions. Used in two cases: for
* CLI extensions from {@code Main}, and for {@code DruidModule}
* extensions during initialization. The design, however, should support
* any kind of extension that may be needed in the future.
* The extensions are cached so that they can be reported by various REST APIs.
*/
public class ExtensionsLoader
{
private static final Logger log = new Logger(ExtensionsLoader.class);
private final ExtensionsConfig extensionsConfig;
private final ConcurrentHashMap<Pair<File, Boolean>, URLClassLoader> loaders = new ConcurrentHashMap<>();
/**
* Map of loaded extensions, keyed by class (or interface).
*/
private final ConcurrentHashMap<Class<?>, Collection<?>> extensions = new ConcurrentHashMap<>();
@Inject
public ExtensionsLoader(ExtensionsConfig config)
{
this.extensionsConfig = config;
}
public static ExtensionsLoader instance(Injector injector)
{
return injector.getInstance(ExtensionsLoader.class);
}
public ExtensionsConfig config()
{
return extensionsConfig;
}
/**
* Returns a collection of implementations loaded.
*
* @param clazz service class
* @param <T> the service type
*/
public <T> Collection<T> getLoadedImplementations(Class<T> clazz)
{
@SuppressWarnings("unchecked")
Collection<T> retVal = (Collection<T>) extensions.get(clazz);
if (retVal == null) {
return Collections.emptySet();
}
return retVal;
}
/**
* @return a collection of implementations loaded.
*/
public Collection<DruidModule> getLoadedModules()
{
return getLoadedImplementations(DruidModule.class);
}
@VisibleForTesting
public Map<Pair<File, Boolean>, URLClassLoader> getLoadersMap()
{
return loaders;
}
/**
* Look for implementations for the given class from both classpath and extensions directory, using {@link
* ServiceLoader}. A user should never put the same two extensions in classpath and extensions directory, if he/she
* does that, the one that is in the classpath will be loaded, the other will be ignored.
*
* @param serviceClass The class to look the implementations of (e.g., DruidModule)
*
* @return A collection that contains implementations (of distinct concrete classes) of the given class. The order of
* elements in the returned collection is not specified and not guaranteed to be the same for different calls to
* getFromExtensions().
*/
@SuppressWarnings("unchecked")
public <T> Collection<T> getFromExtensions(Class<T> serviceClass)
{
// Classes are loaded once upon first request. Since the class path does
// not change during a run, the set of extension classes cannot change once
// computed.
//
// In practice, it appears the only place this matters is with DruidModule:
// initialization gets the list of extensions, and two REST API calls later
// ask for the same list.
Collection<?> modules = extensions.computeIfAbsent(
serviceClass,
serviceC -> new ServiceLoadingFromExtensions<>(serviceC).implsToLoad
);
//noinspection unchecked
return (Collection<T>) modules;
}
public Collection<DruidModule> getModules()
{
return getFromExtensions(DruidModule.class);
}
/**
* Find all the extension files that should be loaded by druid.
* <p/>
* If user explicitly specifies druid.extensions.loadList, then it will look for those extensions under root
* extensions directory. If one of them is not found, druid will fail loudly.
* <p/>
* If user doesn't specify druid.extension.toLoad (or its value is empty), druid will load all the extensions
* under the root extensions directory.
*
* @return an array of druid extension files that will be loaded by druid process
*/
public File[] getExtensionFilesToLoad()
{
final File rootExtensionsDir = new File(extensionsConfig.getDirectory());
if (rootExtensionsDir.exists() && !rootExtensionsDir.isDirectory()) {
throw new ISE("Root extensions directory [%s] is not a directory!?", rootExtensionsDir);
}
File[] extensionsToLoad;
final LinkedHashSet<String> toLoad = extensionsConfig.getLoadList();
if (toLoad == null) {
extensionsToLoad = rootExtensionsDir.listFiles();
} else {
int i = 0;
extensionsToLoad = new File[toLoad.size()];
for (final String extensionName : toLoad) {
File extensionDir = new File(extensionName);
if (!extensionDir.isAbsolute()) {
extensionDir = new File(rootExtensionsDir, extensionName);
}
if (!extensionDir.isDirectory()) {
throw new ISE(
"Extension [%s] specified in \"druid.extensions.loadList\" didn't exist!?",
extensionDir.getAbsolutePath()
);
}
extensionsToLoad[i++] = extensionDir;
}
}
return extensionsToLoad == null ? new File[]{} : extensionsToLoad;
}
/**
* @param extension The File instance of the extension we want to load
*
* @return a URLClassLoader that loads all the jars on which the extension is dependent
*/
public URLClassLoader getClassLoaderForExtension(File extension, boolean useExtensionClassloaderFirst)
{
return loaders.computeIfAbsent(
Pair.of(extension, useExtensionClassloaderFirst),
k -> makeClassLoaderForExtension(k.lhs, k.rhs)
);
}
private static URLClassLoader makeClassLoaderForExtension(
final File extension,
final boolean useExtensionClassloaderFirst
)
{
final Collection<File> jars = FileUtils.listFiles(extension, new String[]{"jar"}, false);
final URL[] urls = new URL[jars.size()];
try {
int i = 0;
for (File jar : jars) {
final URL url = jar.toURI().toURL();
log.debug("added URL [%s] for extension [%s]", url, extension.getName());
urls[i++] = url;
}
}
catch (MalformedURLException e) {
throw new RuntimeException(e);
}
if (useExtensionClassloaderFirst) {
return new ExtensionFirstClassLoader(urls, ExtensionsLoader.class.getClassLoader());
} else {
return new URLClassLoader(urls, ExtensionsLoader.class.getClassLoader());
}
}
public static List<URL> getURLsForClasspath(String cp)
{
try {
String[] paths = cp.split(File.pathSeparator);
List<URL> urls = new ArrayList<>();
for (String path : paths) {
File f = new File(path);
if ("*".equals(f.getName())) {
File parentDir = f.getParentFile();
if (parentDir.isDirectory()) {
File[] jars = parentDir.listFiles(
new FilenameFilter()
{
@Override
public boolean accept(File dir, String name)
{
return name != null && (name.endsWith(".jar") || name.endsWith(".JAR"));
}
}
);
for (File jar : jars) {
urls.add(jar.toURI().toURL());
}
}
} else {
urls.add(new File(path).toURI().toURL());
}
}
return urls;
}
catch (IOException ex) {
throw new RuntimeException(ex);
}
}
private class ServiceLoadingFromExtensions<T>
{
private final Class<T> serviceClass;
private final List<T> implsToLoad = new ArrayList<>();
private final Set<String> implClassNamesToLoad = new HashSet<>();
private ServiceLoadingFromExtensions(Class<T> serviceClass)
{
this.serviceClass = serviceClass;
if (extensionsConfig.searchCurrentClassloader()) {
addAllFromCurrentClassLoader();
}
addAllFromFileSystem();
}
private void addAllFromCurrentClassLoader()
{
ServiceLoader
.load(serviceClass, Thread.currentThread().getContextClassLoader())
.forEach(impl -> tryAdd(impl, "classpath"));
}
private void addAllFromFileSystem()
{
for (File extension : getExtensionFilesToLoad()) {
log.debug("Loading extension [%s] for class [%s]", extension.getName(), serviceClass);
try {
final URLClassLoader loader = getClassLoaderForExtension(
extension,
extensionsConfig.isUseExtensionClassloaderFirst()
);
log.info(
"Loading extension [%s], jars: %s",
extension.getName(),
Arrays.stream(loader.getURLs())
.map(u -> new File(u.getPath()).getName())
.collect(Collectors.joining(", "))
);
ServiceLoader.load(serviceClass, loader).forEach(impl -> tryAdd(impl, "local file system"));
}
catch (Exception e) {
throw new RuntimeException(e);
}
}
}
private void tryAdd(T serviceImpl, String extensionType)
{
final String serviceImplName = serviceImpl.getClass().getName();
if (serviceImplName == null) {
log.warn(
"Implementation [%s] was ignored because it doesn't have a canonical name, "
+ "is it a local or anonymous class?",
serviceImpl.getClass().getName()
);
} else if (!implClassNamesToLoad.contains(serviceImplName)) {
log.debug(
"Adding implementation [%s] for class [%s] from %s extension",
serviceImplName,
serviceClass,
extensionType
);
implClassNamesToLoad.add(serviceImplName);
implsToLoad.add(serviceImpl);
}
}
}
}

64
processing/src/main/java/org/apache/druid/guice/ExtensionsModule.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.guice;
import com.google.inject.Binder;
import com.google.inject.Module;
import javax.inject.Inject;
/**
* Module for the extensions loader. Add to the startup injector
* for Druid servers. Not visible to the {@link StartupInjectorBuilder},
* so must be added by servers explicitly.
*/
public class ExtensionsModule implements Module
{
@Override
public void configure(Binder binder)
{
binder.bind(ExtensionsLoader.class).in(LazySingleton.class);
JsonConfigProvider.bind(binder, "druid.extensions", ExtensionsConfig.class);
JsonConfigProvider.bind(binder, "druid.modules", ModulesConfig.class);
}
/**
* Transfers the now-populated extension loader instance from the
* startup to the main injector. Not done in {@code DruidSecondaryModule}
* because extensions are loaded only in the server, but
* {@code DruidSecondaryModule} is used for tests and clients also.
*/
public static class SecondaryModule implements Module
{
private final ExtensionsLoader extnLoader;
@Inject
public SecondaryModule(final ExtensionsLoader extnLoader)
{
this.extnLoader = extnLoader;
}
@Override
public void configure(Binder binder)
{
binder.bind(ExtensionsLoader.class).toInstance(extnLoader);
}
}
}

40
processing/src/main/java/org/apache/druid/guice/GuiceInjectors.java
View File

@ -19,51 +19,27 @@
package org.apache.druid.guice;
import com.google.common.collect.ImmutableList;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import org.apache.druid.jackson.JacksonModule;
import org.apache.druid.math.expr.ExpressionProcessingModule;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Collections;
/**
* Creates the startup injector. Retained for backward compatibility.
* New code should prefer using {@link StartupInjectorBuilder}
*/
public class GuiceInjectors
{
public static Collection<Module> makeDefaultStartupModules()
{
return ImmutableList.of(
new DruidGuiceExtensions(),
new JacksonModule(),
new PropertiesModule(Arrays.asList("common.runtime.properties", "runtime.properties")),
new RuntimeInfoModule(),
new ConfigModule(),
new NullHandlingModule(),
new ExpressionProcessingModule(),
binder -> {
binder.bind(DruidSecondaryModule.class);
JsonConfigProvider.bind(binder, "druid.extensions", ExtensionsConfig.class);
JsonConfigProvider.bind(binder, "druid.modules", ModulesConfig.class);
}
);
}
public static Injector makeStartupInjector()
{
return Guice.createInjector(makeDefaultStartupModules());
return makeStartupInjectorWithModules(Collections.emptyList());
}
public static Injector makeStartupInjectorWithModules(Iterable<? extends Module> modules)
{
List<Module> theModules = new ArrayList<>(makeDefaultStartupModules());
for (Module theModule : modules) {
theModules.add(theModule);
}
return Guice.createInjector(theModules);
return new StartupInjectorBuilder()
.forServer()
.addAll(modules)
.build();
}
}

85
processing/src/main/java/org/apache/druid/guice/StartupInjectorBuilder.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.guice;
import org.apache.druid.jackson.JacksonModule;
import org.apache.druid.math.expr.ExpressionProcessingModule;
import java.util.Arrays;
import java.util.Properties;
/**
* Create the startup injector used to "prime" the modules for the
* main injector.
* <p>
* Servers call the {@link #forServer()} method to configure server-style
* properties and the server metrics. Servers must also add
* {@code org.apache.druid.initialization.ExtensionsModule} which is
* not visible here, and can't be added in the {@link #forServer()}
* method.
* <p>
* Tests and clients must provide
* properties via another mechanism.
*/
public class StartupInjectorBuilder extends BaseInjectorBuilder<StartupInjectorBuilder>
{
public StartupInjectorBuilder()
{
add(
new DruidGuiceExtensions(),
new JacksonModule(),
new ConfigModule(),
new NullHandlingModule(),
new ExpressionProcessingModule(),
binder -> {
binder.bind(DruidSecondaryModule.class);
}
);
}
public StartupInjectorBuilder withProperties(Properties properties)
{
add(binder -> {
binder.bind(Properties.class).toInstance(properties);
});
return this;
}
public StartupInjectorBuilder withEmptyProperties()
{
return withProperties(new Properties());
}
public StartupInjectorBuilder withExtensions()
{
add(new ExtensionsModule());
return this;
}
public StartupInjectorBuilder forServer()
{
withExtensions();
add(
new PropertiesModule(Arrays.asList("common.runtime.properties", "runtime.properties")),
new RuntimeInfoModule()
);
return this;
}
}

1
processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java
View File

@ -267,7 +267,6 @@ public abstract class AggregatorFactory implements Cacheable
return ColumnTypeFactory.ofValueType(finalized);
}
/**
* This method is deprecated and will be removed soon. Use {@link #getIntermediateType()} instead. Do not call this
* method, it will likely produce incorrect results, it exists for backwards compatibility.

1
processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GroupByMergingQueryRunnerV2.java
View File

@ -390,5 +390,4 @@ public class GroupByMergingQueryRunnerV2 implements QueryRunner<ResultRow>
throw new RuntimeException(e);
}
}
}

333
processing/src/test/java/org/apache/druid/guice/ExtensionsLoaderTest.java Normal file
View File

@ -0,0 +1,333 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.guice;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import com.google.inject.Injector;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.Pair;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
public class ExtensionsLoaderTest
{
@Rule
public final TemporaryFolder temporaryFolder = new TemporaryFolder();
private Injector startupInjector()
{
return new StartupInjectorBuilder()
.withEmptyProperties()
.withExtensions()
.build();
}
@Test
public void test02MakeStartupInjector()
{
Injector startupInjector = startupInjector();
Assert.assertNotNull(startupInjector);
Assert.assertNotNull(startupInjector.getInstance(ObjectMapper.class));
ExtensionsLoader extnLoader = ExtensionsLoader.instance(startupInjector);
Assert.assertNotNull(extnLoader);
Assert.assertSame(extnLoader, ExtensionsLoader.instance(startupInjector));
}
@Test
public void test04DuplicateClassLoaderExtensions() throws Exception
{
final File extensionDir = temporaryFolder.newFolder();
Injector startupInjector = startupInjector();
ExtensionsLoader extnLoader = ExtensionsLoader.instance(startupInjector);
Pair<File, Boolean> key = Pair.of(extensionDir, true);
extnLoader.getLoadersMap()
.put(key, new URLClassLoader(new URL[]{}, ExtensionsLoader.class.getClassLoader()));
Collection<DruidModule> modules = extnLoader.getFromExtensions(DruidModule.class);
Set<String> loadedModuleNames = new HashSet<>();
for (DruidModule module : modules) {
Assert.assertFalse("Duplicate extensions are loaded", loadedModuleNames.contains(module.getClass().getName()));
loadedModuleNames.add(module.getClass().getName());
}
}
@Test
public void test06GetClassLoaderForExtension() throws IOException
{
final ExtensionsLoader extnLoader = new ExtensionsLoader(new ExtensionsConfig());
final File some_extension_dir = temporaryFolder.newFolder();
final File a_jar = new File(some_extension_dir, "a.jar");
final File b_jar = new File(some_extension_dir, "b.jar");
final File c_jar = new File(some_extension_dir, "c.jar");
a_jar.createNewFile();
b_jar.createNewFile();
c_jar.createNewFile();
final URLClassLoader loader = extnLoader.getClassLoaderForExtension(some_extension_dir, false);
final URL[] expectedURLs = new URL[]{a_jar.toURI().toURL(), b_jar.toURI().toURL(), c_jar.toURI().toURL()};
final URL[] actualURLs = loader.getURLs();
Arrays.sort(actualURLs, Comparator.comparing(URL::getPath));
Assert.assertArrayEquals(expectedURLs, actualURLs);
}
@Test
public void testGetLoadedModules()
{
final ExtensionsLoader extnLoader = new ExtensionsLoader(new ExtensionsConfig());
Collection<DruidModule> modules = extnLoader.getModules();
HashSet<DruidModule> moduleSet = new HashSet<>(modules);
Collection<DruidModule> loadedModules = extnLoader.getModules();
Assert.assertEquals("Set from loaded modules #1 should be same!", modules.size(), loadedModules.size());
Assert.assertEquals("Set from loaded modules #1 should be same!", moduleSet, new HashSet<>(loadedModules));
Collection<DruidModule> loadedModules2 = extnLoader.getModules();
Assert.assertEquals("Set from loaded modules #2 should be same!", modules.size(), loadedModules2.size());
Assert.assertEquals("Set from loaded modules #2 should be same!", moduleSet, new HashSet<>(loadedModules2));
}
@Test
public void testGetExtensionFilesToLoad_non_exist_extensions_dir() throws IOException
{
final File tmpDir = temporaryFolder.newFolder();
Assert.assertTrue("could not create missing folder", !tmpDir.exists() || tmpDir.delete());
final ExtensionsLoader extnLoader = new ExtensionsLoader(new ExtensionsConfig()
{
@Override
public String getDirectory()
{
return tmpDir.getAbsolutePath();
}
});
Assert.assertArrayEquals(
"Non-exist root extensionsDir should return an empty array of File",
new File[]{},
extnLoader.getExtensionFilesToLoad()
);
}
@Test(expected = ISE.class)
public void testGetExtensionFilesToLoad_wrong_type_extensions_dir() throws IOException
{
final File extensionsDir = temporaryFolder.newFile();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getDirectory()
{
return extensionsDir.getAbsolutePath();
}
};
final ExtensionsLoader extnLoader = new ExtensionsLoader(config);
extnLoader.getExtensionFilesToLoad();
}
@Test
public void testGetExtensionFilesToLoad_empty_extensions_dir() throws IOException
{
final File extensionsDir = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getDirectory()
{
return extensionsDir.getAbsolutePath();
}
};
final ExtensionsLoader extnLoader = new ExtensionsLoader(config);
Assert.assertArrayEquals(
"Empty root extensionsDir should return an empty array of File",
new File[]{},
extnLoader.getExtensionFilesToLoad()
);
}
/**
* If druid.extension.load is not specified, Initialization.getExtensionFilesToLoad is supposed to return all the
* extension folders under root extensions directory.
*/
@Test
public void testGetExtensionFilesToLoad_null_load_list() throws IOException
{
final File extensionsDir = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getDirectory()
{
return extensionsDir.getAbsolutePath();
}
};
final ExtensionsLoader extnLoader = new ExtensionsLoader(config);
final File mysql_metadata_storage = new File(extensionsDir, "mysql-metadata-storage");
mysql_metadata_storage.mkdir();
final File[] expectedFileList = new File[]{mysql_metadata_storage};
final File[] actualFileList = extnLoader.getExtensionFilesToLoad();
Arrays.sort(actualFileList);
Assert.assertArrayEquals(expectedFileList, actualFileList);
}
/**
* druid.extension.load is specified, Initialization.getExtensionFilesToLoad is supposed to return all the extension
* folders appeared in the load list.
*/
@Test
public void testGetExtensionFilesToLoad_with_load_list() throws IOException
{
final File extensionsDir = temporaryFolder.newFolder();
final File absolutePathExtension = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public LinkedHashSet<String> getLoadList()
{
return Sets.newLinkedHashSet(Arrays.asList("mysql-metadata-storage", absolutePathExtension.getAbsolutePath()));
}
@Override
public String getDirectory()
{
return extensionsDir.getAbsolutePath();
}
};
final ExtensionsLoader extnLoader = new ExtensionsLoader(config);
final File mysql_metadata_storage = new File(extensionsDir, "mysql-metadata-storage");
final File random_extension = new File(extensionsDir, "random-extensions");
mysql_metadata_storage.mkdir();
random_extension.mkdir();
final File[] expectedFileList = new File[]{mysql_metadata_storage, absolutePathExtension};
final File[] actualFileList = extnLoader.getExtensionFilesToLoad();
Assert.assertArrayEquals(expectedFileList, actualFileList);
}
/**
* druid.extension.load is specified, but contains an extension that is not prepared under root extension directory.
* Initialization.getExtensionFilesToLoad is supposed to throw ISE.
*/
@Test(expected = ISE.class)
public void testGetExtensionFilesToLoad_with_non_exist_item_in_load_list() throws IOException
{
final File extensionsDir = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public LinkedHashSet<String> getLoadList()
{
return Sets.newLinkedHashSet(ImmutableList.of("mysql-metadata-storage"));
}
@Override
public String getDirectory()
{
return extensionsDir.getAbsolutePath();
}
};
final File random_extension = new File(extensionsDir, "random-extensions");
random_extension.mkdir();
final ExtensionsLoader extnLoader = new ExtensionsLoader(config);
extnLoader.getExtensionFilesToLoad();
}
@Test
public void testGetURLsForClasspath() throws Exception
{
File tmpDir1 = temporaryFolder.newFolder();
File tmpDir2 = temporaryFolder.newFolder();
File tmpDir3 = temporaryFolder.newFolder();
File tmpDir1a = new File(tmpDir1, "a.jar");
tmpDir1a.createNewFile();
File tmpDir1b = new File(tmpDir1, "b.jar");
tmpDir1b.createNewFile();
new File(tmpDir1, "note1.txt").createNewFile();
File tmpDir2c = new File(tmpDir2, "c.jar");
tmpDir2c.createNewFile();
File tmpDir2d = new File(tmpDir2, "d.jar");
tmpDir2d.createNewFile();
File tmpDir2e = new File(tmpDir2, "e.JAR");
tmpDir2e.createNewFile();
new File(tmpDir2, "note2.txt").createNewFile();
String cp = tmpDir1.getAbsolutePath() + File.separator + "*"
+ File.pathSeparator
+ tmpDir3.getAbsolutePath()
+ File.pathSeparator
+ tmpDir2.getAbsolutePath() + File.separator + "*";
// getURLsForClasspath uses listFiles which does NOT guarantee any ordering for the name strings.
List<URL> urLsForClasspath = ExtensionsLoader.getURLsForClasspath(cp);
Assert.assertEquals(Sets.newHashSet(tmpDir1a.toURI().toURL(), tmpDir1b.toURI().toURL()),
Sets.newHashSet(urLsForClasspath.subList(0, 2)));
Assert.assertEquals(tmpDir3.toURI().toURL(), urLsForClasspath.get(2));
Assert.assertEquals(Sets.newHashSet(tmpDir2c.toURI().toURL(), tmpDir2d.toURI().toURL(), tmpDir2e.toURI().toURL()),
Sets.newHashSet(urLsForClasspath.subList(3, 6)));
}
@Test
public void testExtensionsWithSameDirName() throws Exception
{
final String extensionName = "some_extension";
final File tmpDir1 = temporaryFolder.newFolder();
final File tmpDir2 = temporaryFolder.newFolder();
final File extension1 = new File(tmpDir1, extensionName);
final File extension2 = new File(tmpDir2, extensionName);
Assert.assertTrue(extension1.mkdir());
Assert.assertTrue(extension2.mkdir());
final File jar1 = new File(extension1, "jar1.jar");
final File jar2 = new File(extension2, "jar2.jar");
Assert.assertTrue(jar1.createNewFile());
Assert.assertTrue(jar2.createNewFile());
final ExtensionsLoader extnLoader = new ExtensionsLoader(new ExtensionsConfig());
final ClassLoader classLoader1 = extnLoader.getClassLoaderForExtension(extension1, false);
final ClassLoader classLoader2 = extnLoader.getClassLoaderForExtension(extension2, false);
Assert.assertArrayEquals(new URL[]{jar1.toURI().toURL()}, ((URLClassLoader) classLoader1).getURLs());
Assert.assertArrayEquals(new URL[]{jar2.toURI().toURL()}, ((URLClassLoader) classLoader2).getURLs());
}
}

6
server/pom.xml
View File

@ -48,7 +48,7 @@
<artifactId>druid-gcp-common</artifactId>
<version>${project.parent.version}</version>
<scope>runtime</scope>
</dependency>
</dependency>
<dependency>
<!-- This is needed to bundle the web console -->
<groupId>org.apache.druid</groupId>
@ -304,10 +304,6 @@
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.aether</groupId>
<artifactId>aether-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>

203
server/src/main/java/org/apache/druid/guice/DruidInjectorBuilder.java Normal file
View File

@ -0,0 +1,203 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.guice;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Module;
import org.apache.druid.discovery.NodeRole;
import org.apache.druid.guice.annotations.Json;
import org.apache.druid.guice.annotations.LoadScope;
import org.apache.druid.guice.annotations.Smile;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.logger.Logger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Druid-enabled injector builder which supports {@link DruidModule}s, module classes
* created from the base injector, and filtering based on properties and {@link LoadScope}
* annotations.
* <p>
* Can be used in clients and tests, in which case no module filtering is done.
* Presumably, the test or client has already selected the modules that it needs.
* <p>
* Druid injector builders can be chained with an earlier builder providing a set of
* modules which a later builder overrides. Again, this is typically used only in the
* server, not in clients or tests.
*/
public class DruidInjectorBuilder
{
private static final Logger log = new Logger(DruidInjectorBuilder.class);
private final List<Module> modules = new ArrayList<>();
protected final Injector baseInjector;
private final ObjectMapper jsonMapper;
private final ObjectMapper smileMapper;
private final Set<NodeRole> nodeRoles;
private final ModulesConfig modulesConfig;
public DruidInjectorBuilder(final Injector baseInjector)
{
this(baseInjector, Collections.emptySet());
}
public DruidInjectorBuilder(final Injector baseInjector, final Set<NodeRole> nodeRoles)
{
this.baseInjector = baseInjector;
this.nodeRoles = nodeRoles;
this.modulesConfig = baseInjector.getInstance(ModulesConfig.class);
this.jsonMapper = baseInjector.getInstance(Key.get(ObjectMapper.class, Json.class));
this.smileMapper = baseInjector.getInstance(Key.get(ObjectMapper.class, Smile.class));
}
public DruidInjectorBuilder(final DruidInjectorBuilder from)
{
this.baseInjector = from.baseInjector;
this.nodeRoles = from.nodeRoles;
this.modulesConfig = from.modulesConfig;
this.jsonMapper = from.jsonMapper;
this.smileMapper = from.smileMapper;
}
/**
* Add an arbitrary set of modules.
*
* @see #add(Object)
*/
public DruidInjectorBuilder add(Object...input)
{
for (Object o : input) {
add(o);
}
return this;
}
/**
* Add an arbitrary {@link Module}, {@link DruidModule} instance,
* or a subclass of these classes. If a class is provided, it is instantiated
* using the base injector to allow dependency injection. If a module
* instance is provided, its members are injected. Note that such
* modules have visibility <i>only</i> to objects defined in the base
* injector, but not to objects defined in the injector being built.
*/
public DruidInjectorBuilder add(Object input)
{
if (input instanceof DruidModule) {
return addDruidModule((DruidModule) input);
} else if (input instanceof Module) {
return addModule((Module) input);
} else if (input instanceof Class) {
return addClass((Class<?>) input);
} else {
throw new ISE("Unknown module type[%s]", input.getClass());
}
}
public DruidInjectorBuilder addDruidModule(DruidModule module)
{
if (!acceptModule(module.getClass())) {
return this;
}
baseInjector.injectMembers(module);
registerJacksonModules(module);
modules.add(module);
return this;
}
public DruidInjectorBuilder addModule(Module module)
{
if (!acceptModule(module.getClass())) {
return this;
}
baseInjector.injectMembers(module);
modules.add(module);
return this;
}
public DruidInjectorBuilder addClass(Class<?> input)
{
if (!acceptModule((Class<?>) input)) {
return this;
}
if (DruidModule.class.isAssignableFrom(input)) {
@SuppressWarnings("unchecked")
DruidModule module = baseInjector.getInstance((Class<? extends DruidModule>) input);
registerJacksonModules(module);
modules.add(module);
} else if (Module.class.isAssignableFrom(input)) {
@SuppressWarnings("unchecked")
Module module = baseInjector.getInstance((Class<? extends Module>) input);
modules.add(module);
} else {
throw new ISE("Class [%s] does not implement %s", input, Module.class);
}
return this;
}
/**
* Filter module classes based on the (optional) module exclude list and
* (optional) set of known node roles.
*/
private boolean acceptModule(Class<?> moduleClass)
{
// Modules config is optional: it won't be present in tests or clients.
String moduleClassName = moduleClass.getName();
if (moduleClassName != null && modulesConfig.getExcludeList().contains(moduleClassName)) {
log.info("Not loading module [%s] because it is present in excludeList", moduleClassName);
return false;
}
LoadScope loadScope = moduleClass.getAnnotation(LoadScope.class);
if (loadScope == null) {
// always load if annotation is not specified
return true;
}
Set<NodeRole> rolesPredicate = Arrays.stream(loadScope.roles())
.map(NodeRole::fromJsonName)
.collect(Collectors.toSet());
return rolesPredicate.stream().anyMatch(nodeRoles::contains);
}
private void registerJacksonModules(DruidModule module)
{
for (com.fasterxml.jackson.databind.Module jacksonModule : module.getJacksonModules()) {
jsonMapper.registerModule(jacksonModule);
smileMapper.registerModule(jacksonModule);
}
}
public List<Module> modules()
{
return modules;
}
public Injector build()
{
return Guice.createInjector(modules);
}
}

11
server/src/main/java/org/apache/druid/guice/ExpressionModule.java
View File

@ -19,11 +19,10 @@
package org.apache.druid.guice;
import com.fasterxml.jackson.databind.Module;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import com.google.inject.Module;
import com.google.inject.multibindings.Multibinder;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.query.expression.CaseInsensitiveContainsExprMacro;
import org.apache.druid.query.expression.ContainsExprMacro;
@ -46,7 +45,7 @@ import org.apache.druid.query.expression.TrimExprMacro;
import java.util.List;
public class ExpressionModule implements DruidModule
public class ExpressionModule implements Module
{
public static final List<Class<? extends ExprMacroTable.ExprMacro>> EXPR_MACROS =
ImmutableList.<Class<? extends ExprMacroTable.ExprMacro>>builder()
@ -94,12 +93,6 @@ public class ExpressionModule implements DruidModule
}
}
@Override
public List<? extends Module> getJacksonModules()
{
return ImmutableList.of();
}
public static void addExprMacro(final Binder binder, final Class<? extends ExprMacroTable.ExprMacro> clazz)
{
Multibinder.newSetBinder(binder, ExprMacroTable.ExprMacro.class)

10
server/src/main/java/org/apache/druid/guice/security/DruidAuthModule.java
View File

@ -19,23 +19,13 @@
package org.apache.druid.guice.security;
import com.fasterxml.jackson.databind.Module;
import com.google.inject.Binder;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.server.security.AuthConfig;
import java.util.Collections;
import java.util.List;
public class DruidAuthModule implements DruidModule
{
@Override
public List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
@Override
public void configure(Binder binder)
{

142
server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java Normal file
View File

@ -0,0 +1,142 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.initialization;
import com.google.inject.Injector;
import org.apache.druid.curator.CuratorModule;
import org.apache.druid.curator.discovery.DiscoveryModule;
import org.apache.druid.discovery.NodeRole;
import org.apache.druid.guice.AnnouncerModule;
import org.apache.druid.guice.CoordinatorDiscoveryModule;
import org.apache.druid.guice.DruidInjectorBuilder;
import org.apache.druid.guice.DruidProcessingConfigModule;
import org.apache.druid.guice.DruidSecondaryModule;
import org.apache.druid.guice.ExpressionModule;
import org.apache.druid.guice.ExtensionsModule;
import org.apache.druid.guice.FirehoseModule;
import org.apache.druid.guice.IndexingServiceDiscoveryModule;
import org.apache.druid.guice.JacksonConfigManagerModule;
import org.apache.druid.guice.JavaScriptModule;
import org.apache.druid.guice.LifecycleModule;
import org.apache.druid.guice.LocalDataStorageDruidModule;
import org.apache.druid.guice.MetadataConfigModule;
import org.apache.druid.guice.NestedDataModule;
import org.apache.druid.guice.ServerModule;
import org.apache.druid.guice.ServerViewModule;
import org.apache.druid.guice.StartupLoggingModule;
import org.apache.druid.guice.StorageNodeModule;
import org.apache.druid.guice.annotations.Client;
import org.apache.druid.guice.annotations.EscalatedClient;
import org.apache.druid.guice.http.HttpClientModule;
import org.apache.druid.guice.security.AuthenticatorModule;
import org.apache.druid.guice.security.AuthorizerModule;
import org.apache.druid.guice.security.DruidAuthModule;
import org.apache.druid.guice.security.EscalatorModule;
import org.apache.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule;
import org.apache.druid.rpc.guice.ServiceClientModule;
import org.apache.druid.segment.writeout.SegmentWriteOutMediumModule;
import org.apache.druid.server.emitter.EmitterModule;
import org.apache.druid.server.initialization.AuthenticatorMapperModule;
import org.apache.druid.server.initialization.AuthorizerMapperModule;
import org.apache.druid.server.initialization.ExternalStorageAccessSecurityModule;
import org.apache.druid.server.initialization.jetty.JettyServerModule;
import org.apache.druid.server.metrics.MetricsModule;
import org.apache.druid.server.security.TLSCertificateCheckerModule;
import java.util.Collections;
import java.util.Set;
/**
* Builds the core (common) set of modules used by all Druid services and
* commands. The basic injector just adds logging and the Druid lifecycle.
* Call {@link #forServer()} to add the server-specific modules.
*/
public class CoreInjectorBuilder extends DruidInjectorBuilder
{
public CoreInjectorBuilder(final Injector baseInjector)
{
this(baseInjector, Collections.emptySet());
}
public CoreInjectorBuilder(final Injector baseInjector, final Set<NodeRole> nodeRoles)
{
super(baseInjector, nodeRoles);
add(DruidSecondaryModule.class);
}
public CoreInjectorBuilder withLogging()
{
// New modules should be added after Log4jShutterDownerModule
add(new Log4jShutterDownerModule());
return this;
}
public CoreInjectorBuilder withLifecycle()
{
add(new LifecycleModule());
return this;
}
public CoreInjectorBuilder forServer()
{
withLogging();
withLifecycle();
add(
ExtensionsModule.SecondaryModule.class,
new DruidAuthModule(),
TLSCertificateCheckerModule.class,
EmitterModule.class,
HttpClientModule.global(),
HttpClientModule.escalatedGlobal(),
new HttpClientModule("druid.broker.http", Client.class),
new HttpClientModule("druid.broker.http", EscalatedClient.class),
new CuratorModule(),
new AnnouncerModule(),
new MetricsModule(),
new SegmentWriteOutMediumModule(),
new ServerModule(),
new DruidProcessingConfigModule(),
new StorageNodeModule(),
new JettyServerModule(),
new ExpressionModule(),
new NestedDataModule(),
new DiscoveryModule(),
new ServerViewModule(),
new MetadataConfigModule(),
new DerbyMetadataStorageDruidModule(),
new JacksonConfigManagerModule(),
new IndexingServiceDiscoveryModule(),
new CoordinatorDiscoveryModule(),
new LocalDataStorageDruidModule(),
new TombstoneDataStorageModule(),
new FirehoseModule(),
new JavaScriptModule(),
new AuthenticatorModule(),
new AuthenticatorMapperModule(),
new EscalatorModule(),
new AuthorizerModule(),
new AuthorizerMapperModule(),
new StartupLoggingModule(),
new ExternalStorageAccessSecurityModule(),
new ServiceClientModule()
);
return this;
}
}

51
server/src/main/java/org/apache/druid/initialization/ExtensionInjectorBuilder.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.initialization;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.util.Modules;
import org.apache.druid.guice.DruidInjectorBuilder;
import org.apache.druid.guice.ExtensionsLoader;
/**
* Injector builder which overrides service modules with extension
* modules. Used only in the server, not in clients or tests.
*/
public class ExtensionInjectorBuilder extends DruidInjectorBuilder
{
private final ServiceInjectorBuilder serviceBuilder;
public ExtensionInjectorBuilder(ServiceInjectorBuilder serviceBuilder)
{
super(serviceBuilder);
this.serviceBuilder = serviceBuilder;
ExtensionsLoader extnLoader = ExtensionsLoader.instance(baseInjector);
for (DruidModule module : extnLoader.getFromExtensions(DruidModule.class)) {
addDruidModule(module);
}
}
@Override
public Injector build()
{
return Guice.createInjector(Modules.override(serviceBuilder.merge()).with(modules()));
}
}

547
server/src/main/java/org/apache/druid/initialization/Initialization.java
View File

@ -19,536 +19,47 @@
package org.apache.druid.initialization;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.Module;
import com.google.inject.util.Modules;
import org.apache.commons.io.FileUtils;
import org.apache.druid.curator.CuratorModule;
import org.apache.druid.curator.discovery.DiscoveryModule;
import org.apache.druid.discovery.NodeRole;
import org.apache.druid.guice.AnnouncerModule;
import org.apache.druid.guice.CoordinatorDiscoveryModule;
import org.apache.druid.guice.DruidProcessingConfigModule;
import org.apache.druid.guice.DruidSecondaryModule;
import org.apache.druid.guice.ExpressionModule;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.guice.FirehoseModule;
import org.apache.druid.guice.IndexingServiceDiscoveryModule;
import org.apache.druid.guice.JacksonConfigManagerModule;
import org.apache.druid.guice.JavaScriptModule;
import org.apache.druid.guice.LifecycleModule;
import org.apache.druid.guice.LocalDataStorageDruidModule;
import org.apache.druid.guice.MetadataConfigModule;
import org.apache.druid.guice.ModulesConfig;
import org.apache.druid.guice.NestedDataModule;
import org.apache.druid.guice.ServerModule;
import org.apache.druid.guice.ServerViewModule;
import org.apache.druid.guice.StartupLoggingModule;
import org.apache.druid.guice.StorageNodeModule;
import org.apache.druid.guice.annotations.Client;
import org.apache.druid.guice.annotations.EscalatedClient;
import org.apache.druid.guice.annotations.Json;
import org.apache.druid.guice.annotations.LoadScope;
import org.apache.druid.guice.annotations.Smile;
import org.apache.druid.guice.http.HttpClientModule;
import org.apache.druid.guice.security.AuthenticatorModule;
import org.apache.druid.guice.security.AuthorizerModule;
import org.apache.druid.guice.security.DruidAuthModule;
import org.apache.druid.guice.security.EscalatorModule;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule;
import org.apache.druid.rpc.guice.ServiceClientModule;
import org.apache.druid.segment.writeout.SegmentWriteOutMediumModule;
import org.apache.druid.server.emitter.EmitterModule;
import org.apache.druid.server.initialization.AuthenticatorMapperModule;
import org.apache.druid.server.initialization.AuthorizerMapperModule;
import org.apache.druid.server.initialization.ExternalStorageAccessSecurityModule;
import org.apache.druid.server.initialization.jetty.JettyServerModule;
import org.apache.druid.server.metrics.MetricsModule;
import org.apache.druid.server.security.TLSCertificateCheckerModule;
import org.eclipse.aether.artifact.DefaultArtifact;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
*
* Initialize Guice for a server. This is a legacy version, kept for
* compatibility with existing tests. Clients and tests should use
* the individual builders to create a non-server environment.
* Clients (and tests) never load extensions, and so do not need
* (and, in fact, should not use) the
* {@link ExtensionInjectorBuilder}. Instead, simple tests can use
* {@link org.apache.druid.guice.StartupInjectorBuilder
* StartupInjectorBuilder} directly, passing in any needed modules.
* <p>
* Some tests use modules that rely on the "startup injector" to
* inject values into a module. In that case, tests should use two
* builders: the {@code StartupInjectorBuilder} followed by
* the {@link CoreInjectorBuilder} class to hold extra modules.
* Look for references to {@link CoreInjectorBuilder} to find examples
* of this pattern.
* <p>
* In both cases, the injector builders have options to add the full
* set of server modules. Tests should not load those modules. Instead,
* let the injector builders provide just the required set, and then
* explicitly list the (small subset) of modules needed by any given test.
* <p>
* The server initialization formerly done here is now done in
* {@link org.apache.druid.cli.GuiceRunnable GuiceRunnable} by way of
* the {@link ServerInjectorBuilder}.
*/
public class Initialization
{
private static final Logger log = new Logger(Initialization.class);
private static final ConcurrentHashMap<File, URLClassLoader> LOADERS_MAP = new ConcurrentHashMap<>();
private static final ConcurrentHashMap<Class<?>, Collection<?>> EXTENSIONS_MAP = new ConcurrentHashMap<>();
/**
* @param clazz service class
* @param <T> the service type
*
* @return Returns a collection of implementations loaded.
*/
public static <T> Collection<T> getLoadedImplementations(Class<T> clazz)
{
@SuppressWarnings("unchecked")
Collection<T> retVal = (Collection<T>) EXTENSIONS_MAP.get(clazz);
if (retVal == null) {
return new HashSet<>();
}
return retVal;
}
@VisibleForTesting
static void clearLoadedImplementations()
{
EXTENSIONS_MAP.clear();
}
@VisibleForTesting
static Map<File, URLClassLoader> getLoadersMap()
{
return LOADERS_MAP;
}
/**
* Look for implementations for the given class from both classpath and extensions directory, using {@link
* ServiceLoader}. A user should never put the same two extensions in classpath and extensions directory, if he/she
* does that, the one that is in the classpath will be loaded, the other will be ignored.
*
* @param config Extensions configuration
* @param serviceClass The class to look the implementations of (e.g., DruidModule)
*
* @return A collection that contains implementations (of distinct concrete classes) of the given class. The order of
* elements in the returned collection is not specified and not guaranteed to be the same for different calls to
* getFromExtensions().
*/
public static <T> Collection<T> getFromExtensions(ExtensionsConfig config, Class<T> serviceClass)
{
// It's not clear whether we should recompute modules even if they have been computed already for the serviceClass,
// but that's how it used to be an preserving the old behaviour here.
Collection<?> modules = EXTENSIONS_MAP.compute(
serviceClass,
(serviceC, ignored) -> new ServiceLoadingFromExtensions<>(config, serviceC).implsToLoad
);
//noinspection unchecked
return (Collection<T>) modules;
}
private static class ServiceLoadingFromExtensions<T>
{
private final ExtensionsConfig extensionsConfig;
private final Class<T> serviceClass;
private final List<T> implsToLoad = new ArrayList<>();
private final Set<String> implClassNamesToLoad = new HashSet<>();
private ServiceLoadingFromExtensions(ExtensionsConfig extensionsConfig, Class<T> serviceClass)
{
this.extensionsConfig = extensionsConfig;
this.serviceClass = serviceClass;
if (extensionsConfig.searchCurrentClassloader()) {
addAllFromCurrentClassLoader();
}
addAllFromFileSystem();
}
private void addAllFromCurrentClassLoader()
{
ServiceLoader
.load(serviceClass, Thread.currentThread().getContextClassLoader())
.forEach(impl -> tryAdd(impl, "classpath"));
}
private void addAllFromFileSystem()
{
for (File extension : getExtensionFilesToLoad(extensionsConfig)) {
log.debug("Loading extension [%s] for class [%s]", extension.getName(), serviceClass);
try {
final URLClassLoader loader = getClassLoaderForExtension(
extension,
extensionsConfig.isUseExtensionClassloaderFirst()
);
log.info(
"Loading extension [%s], jars: %s",
extension.getName(),
Arrays.stream(loader.getURLs())
.map(u -> new File(u.getPath()).getName())
.collect(Collectors.joining(", "))
);
ServiceLoader.load(serviceClass, loader).forEach(impl -> tryAdd(impl, "local file system"));
}
catch (Exception e) {
throw new RuntimeException(e);
}
}
}
private void tryAdd(T serviceImpl, String extensionType)
{
final String serviceImplName = serviceImpl.getClass().getName();
if (serviceImplName == null) {
log.warn(
"Implementation [%s] was ignored because it doesn't have a canonical name, "
+ "is it a local or anonymous class?",
serviceImpl.getClass().getName()
);
} else if (!implClassNamesToLoad.contains(serviceImplName)) {
log.debug(
"Adding implementation [%s] for class [%s] from %s extension",
serviceImplName,
serviceClass,
extensionType
);
implClassNamesToLoad.add(serviceImplName);
implsToLoad.add(serviceImpl);
}
}
}
/**
* Find all the extension files that should be loaded by druid.
* <p/>
* If user explicitly specifies druid.extensions.loadList, then it will look for those extensions under root
* extensions directory. If one of them is not found, druid will fail loudly.
* <p/>
* If user doesn't specify druid.extension.toLoad (or its value is empty), druid will load all the extensions
* under the root extensions directory.
*
* @param config ExtensionsConfig configured by druid.extensions.xxx
*
* @return an array of druid extension files that will be loaded by druid process
*/
public static File[] getExtensionFilesToLoad(ExtensionsConfig config)
{
final File rootExtensionsDir = new File(config.getDirectory());
if (rootExtensionsDir.exists() && !rootExtensionsDir.isDirectory()) {
throw new ISE("Root extensions directory [%s] is not a directory!?", rootExtensionsDir);
}
File[] extensionsToLoad;
final LinkedHashSet<String> toLoad = config.getLoadList();
if (toLoad == null) {
extensionsToLoad = rootExtensionsDir.listFiles();
} else {
int i = 0;
extensionsToLoad = new File[toLoad.size()];
for (final String extensionName : toLoad) {
File extensionDir = new File(extensionName);
if (!extensionDir.isAbsolute()) {
extensionDir = new File(rootExtensionsDir, extensionName);
}
if (!extensionDir.isDirectory()) {
throw new ISE(
"Extension [%s] specified in \"druid.extensions.loadList\" didn't exist!?",
extensionDir.getAbsolutePath()
);
}
extensionsToLoad[i++] = extensionDir;
}
}
return extensionsToLoad == null ? new File[]{} : extensionsToLoad;
}
/**
* Find all the hadoop dependencies that should be loaded by druid
*
* @param hadoopDependencyCoordinates e.g.["org.apache.hadoop:hadoop-client:2.3.0"]
* @param extensionsConfig ExtensionsConfig configured by druid.extensions.xxx
*
* @return an array of hadoop dependency files that will be loaded by druid process
*/
public static File[] getHadoopDependencyFilesToLoad(
List<String> hadoopDependencyCoordinates,
ExtensionsConfig extensionsConfig
)
{
final File rootHadoopDependenciesDir = new File(extensionsConfig.getHadoopDependenciesDir());
if (rootHadoopDependenciesDir.exists() && !rootHadoopDependenciesDir.isDirectory()) {
throw new ISE("Root Hadoop dependencies directory [%s] is not a directory!?", rootHadoopDependenciesDir);
}
final File[] hadoopDependenciesToLoad = new File[hadoopDependencyCoordinates.size()];
int i = 0;
for (final String coordinate : hadoopDependencyCoordinates) {
final DefaultArtifact artifact = new DefaultArtifact(coordinate);
final File hadoopDependencyDir = new File(rootHadoopDependenciesDir, artifact.getArtifactId());
final File versionDir = new File(hadoopDependencyDir, artifact.getVersion());
// find the hadoop dependency with the version specified in coordinate
if (!hadoopDependencyDir.isDirectory() || !versionDir.isDirectory()) {
throw new ISE("Hadoop dependency [%s] didn't exist!?", versionDir.getAbsolutePath());
}
hadoopDependenciesToLoad[i++] = versionDir;
}
return hadoopDependenciesToLoad;
}
/**
* @param extension The File instance of the extension we want to load
*
* @return a URLClassLoader that loads all the jars on which the extension is dependent
*/
public static URLClassLoader getClassLoaderForExtension(File extension, boolean useExtensionClassloaderFirst)
{
return LOADERS_MAP.computeIfAbsent(
extension,
theExtension -> makeClassLoaderForExtension(theExtension, useExtensionClassloaderFirst)
);
}
private static URLClassLoader makeClassLoaderForExtension(
final File extension,
final boolean useExtensionClassloaderFirst
)
{
final Collection<File> jars = FileUtils.listFiles(extension, new String[]{"jar"}, false);
final URL[] urls = new URL[jars.size()];
try {
int i = 0;
for (File jar : jars) {
final URL url = jar.toURI().toURL();
log.debug("added URL[%s] for extension[%s]", url, extension.getName());
urls[i++] = url;
}
}
catch (MalformedURLException e) {
throw new RuntimeException(e);
}
if (useExtensionClassloaderFirst) {
return new ExtensionFirstClassLoader(urls, Initialization.class.getClassLoader());
} else {
return new URLClassLoader(urls, Initialization.class.getClassLoader());
}
}
public static List<URL> getURLsForClasspath(String cp)
{
try {
String[] paths = cp.split(File.pathSeparator);
List<URL> urls = new ArrayList<>();
for (String path : paths) {
File f = new File(path);
if ("*".equals(f.getName())) {
File parentDir = f.getParentFile();
if (parentDir.isDirectory()) {
File[] jars = parentDir.listFiles(
new FilenameFilter()
{
@Override
public boolean accept(File dir, String name)
{
return name != null && (name.endsWith(".jar") || name.endsWith(".JAR"));
}
}
);
for (File jar : jars) {
urls.add(jar.toURI().toURL());
}
}
} else {
urls.add(new File(path).toURI().toURL());
}
}
return urls;
}
catch (IOException ex) {
throw new RuntimeException(ex);
}
}
// Use individual builders for testing: this method brings in
// server-only dependencies, which is generally not desired.
// See class comment for more information.
@Deprecated
public static Injector makeInjectorWithModules(
final Injector baseInjector,
final Iterable<? extends Module> modules
)
{
return makeInjectorWithModules(ImmutableSet.of(), baseInjector, modules);
}
public static Injector makeInjectorWithModules(
final Set<NodeRole> nodeRoles,
final Injector baseInjector,
final Iterable<? extends Module> modules
)
{
final ModuleList defaultModules = new ModuleList(baseInjector, nodeRoles);
defaultModules.addModules(
// New modules should be added after Log4jShutterDownerModule
new Log4jShutterDownerModule(),
new DruidAuthModule(),
new LifecycleModule(),
TLSCertificateCheckerModule.class,
EmitterModule.class,
HttpClientModule.global(),
HttpClientModule.escalatedGlobal(),
new HttpClientModule("druid.broker.http", Client.class),
new HttpClientModule("druid.broker.http", EscalatedClient.class),
new CuratorModule(),
new AnnouncerModule(),
new MetricsModule(),
new SegmentWriteOutMediumModule(),
new ServerModule(),
new DruidProcessingConfigModule(),
new StorageNodeModule(),
new JettyServerModule(),
new ExpressionModule(),
new NestedDataModule(),
new DiscoveryModule(),
new ServerViewModule(),
new MetadataConfigModule(),
new DerbyMetadataStorageDruidModule(),
new JacksonConfigManagerModule(),
new IndexingServiceDiscoveryModule(),
new CoordinatorDiscoveryModule(),
new LocalDataStorageDruidModule(),
new TombstoneDataStorageModule(),
new FirehoseModule(),
new JavaScriptModule(),
new AuthenticatorModule(),
new AuthenticatorMapperModule(),
new EscalatorModule(),
new AuthorizerModule(),
new AuthorizerMapperModule(),
new StartupLoggingModule(),
new ExternalStorageAccessSecurityModule(),
new ServiceClientModule()
);
ModuleList actualModules = new ModuleList(baseInjector, nodeRoles);
actualModules.addModule(DruidSecondaryModule.class);
for (Object module : modules) {
actualModules.addModule(module);
}
Module intermediateModules = Modules.override(defaultModules.getModules()).with(actualModules.getModules());
ModuleList extensionModules = new ModuleList(baseInjector, nodeRoles);
final ExtensionsConfig config = baseInjector.getInstance(ExtensionsConfig.class);
for (DruidModule module : Initialization.getFromExtensions(config, DruidModule.class)) {
extensionModules.addModule(module);
}
return Guice.createInjector(Modules.override(intermediateModules).with(extensionModules.getModules()));
}
public static class ModuleList
{
private final Injector baseInjector;
private final Set<NodeRole> nodeRoles;
private final ModulesConfig modulesConfig;
private final ObjectMapper jsonMapper;
private final ObjectMapper smileMapper;
private final List<Module> modules;
public ModuleList(Injector baseInjector, Set<NodeRole> nodeRoles)
{
this.baseInjector = baseInjector;
this.nodeRoles = nodeRoles;
this.modulesConfig = baseInjector.getInstance(ModulesConfig.class);
this.jsonMapper = baseInjector.getInstance(Key.get(ObjectMapper.class, Json.class));
this.smileMapper = baseInjector.getInstance(Key.get(ObjectMapper.class, Smile.class));
this.modules = new ArrayList<>();
}
public List<Module> getModules()
{
return Collections.unmodifiableList(modules);
}
public void addModule(Object input)
{
if (!shouldLoadOnCurrentNodeType(input)) {
return;
}
if (input instanceof DruidModule) {
if (!checkModuleClass(input.getClass())) {
return;
}
baseInjector.injectMembers(input);
modules.add(registerJacksonModules(((DruidModule) input)));
} else if (input instanceof Module) {
if (!checkModuleClass(input.getClass())) {
return;
}
baseInjector.injectMembers(input);
modules.add((Module) input);
} else if (input instanceof Class) {
if (!checkModuleClass((Class<?>) input)) {
return;
}
if (DruidModule.class.isAssignableFrom((Class) input)) {
modules.add(registerJacksonModules(baseInjector.getInstance((Class<? extends DruidModule>) input)));
} else if (Module.class.isAssignableFrom((Class) input)) {
modules.add(baseInjector.getInstance((Class<? extends Module>) input));
return;
} else {
throw new ISE("Class[%s] does not implement %s", input.getClass(), Module.class);
}
} else {
throw new ISE("Unknown module type[%s]", input.getClass());
}
}
private boolean shouldLoadOnCurrentNodeType(Object object)
{
LoadScope loadScope = object.getClass().getAnnotation(LoadScope.class);
if (loadScope == null) {
// always load if annotation is not specified
return true;
}
Set<NodeRole> rolesPredicate = Arrays.stream(loadScope.roles())
.map(NodeRole::fromJsonName)
.collect(Collectors.toSet());
return rolesPredicate.stream().anyMatch(nodeRoles::contains);
}
private boolean checkModuleClass(Class<?> moduleClass)
{
String moduleClassName = moduleClass.getName();
if (moduleClassName != null && modulesConfig.getExcludeList().contains(moduleClassName)) {
log.info("Not loading module [%s] because it is present in excludeList", moduleClassName);
return false;
}
return true;
}
public void addModules(Object... object)
{
for (Object o : object) {
addModule(o);
}
}
private DruidModule registerJacksonModules(DruidModule module)
{
for (com.fasterxml.jackson.databind.Module jacksonModule : module.getJacksonModules()) {
jsonMapper.registerModule(jacksonModule);
smileMapper.registerModule(jacksonModule);
}
return module;
}
return ServerInjectorBuilder.makeServerInjector(baseInjector, ImmutableSet.of(), modules);
}
}

132
server/src/main/java/org/apache/druid/initialization/ServerInjectorBuilder.java Normal file
View File

@ -0,0 +1,132 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.initialization;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.MapBinder;
import com.google.inject.multibindings.Multibinder;
import org.apache.druid.discovery.DruidService;
import org.apache.druid.discovery.NodeRole;
import org.apache.druid.guice.annotations.Self;
import java.util.Set;
/**
* Initialize Guice for a server. Clients and tests should use
* the individual builders to create a non-server environment.
* <p>
* This class is in this package for historical reasons. The proper
* place is in the same module as {@code GuiceRunnable} since this
* class should only ever be used by servers. It is here until
* tests are converted to use the builders, and @{link Initialization}
* is deleted.
*/
public class ServerInjectorBuilder
{
private Injector baseInjector;
private Set<NodeRole> nodeRoles;
private Iterable<? extends Module> modules;
/**
* Create a server injector. Located here for testing. Should only be
* used by {@code GuiceRunnable} (and tests).
*
* @param nodeRoles the roles which this server provides
* @param baseInjector the startup injector
* @param modules modules for this server
* @return the injector for the server
*/
@VisibleForTesting
public static Injector makeServerInjector(
final Injector baseInjector,
final Set<NodeRole> nodeRoles,
final Iterable<? extends Module> modules
)
{
return new ServerInjectorBuilder(baseInjector)
.nodeRoles(nodeRoles)
.serviceModules(modules)
.build();
}
public ServerInjectorBuilder(Injector baseInjector)
{
this.baseInjector = baseInjector;
}
public ServerInjectorBuilder nodeRoles(final Set<NodeRole> nodeRoles)
{
this.nodeRoles = nodeRoles;
return this;
}
public ServerInjectorBuilder serviceModules(final Iterable<? extends Module> modules)
{
this.modules = modules;
return this;
}
public Injector build()
{
Module registerNodeRoleModule = registerNodeRoleModule(nodeRoles);
// Child injector, with the registered node roles
Injector childInjector = baseInjector.createChildInjector(registerNodeRoleModule);
// Create the core set of modules shared by all services.
// Here and below, the modules are filtered by the load modules list and
// the set of roles which this server provides.
CoreInjectorBuilder coreBuilder = new CoreInjectorBuilder(childInjector, nodeRoles).forServer();
// Override with the per-service modules.
ServiceInjectorBuilder serviceBuilder = new ServiceInjectorBuilder(coreBuilder).addAll(
Iterables.concat(
// bind nodeRoles for the new injector as well
ImmutableList.of(registerNodeRoleModule),
modules
)
);
// Override again with extensions.
return new ExtensionInjectorBuilder(serviceBuilder).build();
}
public static Module registerNodeRoleModule(Set<NodeRole> nodeRoles)
{
if (nodeRoles.isEmpty()) {
return binder -> {};
}
return binder -> {
Multibinder<NodeRole> selfBinder = Multibinder.newSetBinder(binder, NodeRole.class, Self.class);
nodeRoles.forEach(nodeRole -> selfBinder.addBinding().toInstance(nodeRole));
MapBinder.newMapBinder(
binder,
new TypeLiteral<NodeRole>(){},
new TypeLiteral<Set<Class<? extends DruidService>>>(){}
);
};
}
}

63
server/src/main/java/org/apache/druid/initialization/ServiceInjectorBuilder.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.initialization;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.util.Modules;
import org.apache.druid.guice.DruidInjectorBuilder;
/**
* Injector builder for a service within a server. In the server, this builder
* is input to the {@link ExtensionInjectorBuilder}. Also used to build clients
* or tests, without extensions, where this builder itself builds the injector.
*/
public class ServiceInjectorBuilder extends DruidInjectorBuilder
{
private final CoreInjectorBuilder coreBuilder;
public ServiceInjectorBuilder(
final CoreInjectorBuilder coreBuilder
)
{
super(coreBuilder);
this.coreBuilder = coreBuilder;
}
public Module merge()
{
return Modules.override(coreBuilder.modules()).with(modules());
}
@Override
public Injector build()
{
return Guice.createInjector(merge());
}
public ServiceInjectorBuilder addAll(Iterable<? extends Module> modules)
{
for (Module module : modules) {
add(module);
}
return this;
}
}

9
server/src/main/java/org/apache/druid/rpc/guice/ServiceClientModule.java
View File

@ -19,7 +19,6 @@
package org.apache.druid.rpc.guice;
import com.fasterxml.jackson.databind.Module;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Binder;
import com.google.inject.Provides;
@ -41,8 +40,6 @@ import org.apache.druid.rpc.StandardRetryPolicy;
import org.apache.druid.rpc.indexing.OverlordClient;
import org.apache.druid.rpc.indexing.OverlordClientImpl;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
public class ServiceClientModule implements DruidModule
@ -50,12 +47,6 @@ public class ServiceClientModule implements DruidModule
private static final int CONNECT_EXEC_THREADS = 4;
private static final int OVERLORD_ATTEMPTS = 6;
@Override
public List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
@Override
public void configure(Binder binder)
{

13
server/src/main/java/org/apache/druid/server/StatusResource.java
View File

@ -24,8 +24,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Maps;
import com.sun.jersey.spi.container.ResourceFilters;
import org.apache.druid.client.DruidServerConfig;
import org.apache.druid.guice.ExtensionsLoader;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.initialization.Initialization;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.server.http.security.ConfigResourceFilter;
import org.apache.druid.server.http.security.StateResourceFilter;
@ -39,6 +39,7 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@ -52,14 +53,18 @@ import java.util.Set;
public class StatusResource
{
private final Properties properties;
private final DruidServerConfig druidServerConfig;
private final ExtensionsLoader extnLoader;
@Inject
public StatusResource(Properties properties, DruidServerConfig druidServerConfig)
public StatusResource(
final Properties properties,
final DruidServerConfig druidServerConfig,
final ExtensionsLoader extnLoader)
{
this.properties = properties;
this.druidServerConfig = druidServerConfig;
this.extnLoader = extnLoader;
}
@GET
@ -80,7 +85,7 @@ public class StatusResource
@Context final HttpServletRequest req
)
{
return new Status(Initialization.getLoadedImplementations(DruidModule.class));
return new Status(extnLoader.getLoadedModules());
}
/**

8
server/src/main/java/org/apache/druid/server/emitter/ComposingEmitterModule.java
View File

@ -19,7 +19,6 @@
package org.apache.druid.server.emitter;
import com.fasterxml.jackson.databind.Module;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.inject.Binder;
@ -35,7 +34,6 @@ import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.java.util.emitter.core.ComposingEmitter;
import org.apache.druid.java.util.emitter.core.Emitter;
import java.util.Collections;
import java.util.List;
/**
@ -50,12 +48,6 @@ public class ComposingEmitterModule implements DruidModule
JsonConfigProvider.bind(binder, "druid.emitter.composing", ComposingEmitterConfig.class);
}
@Override
public List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
@Provides
@ManageLifecycle
@Named("composing")

10
server/src/main/java/org/apache/druid/server/initialization/AuthenticatorMapperModule.java
View File

@ -19,7 +19,6 @@
package org.apache.druid.server.initialization;
import com.fasterxml.jackson.databind.Module;
import com.google.common.base.Supplier;
import com.google.common.collect.Maps;
import com.google.inject.Binder;
@ -34,13 +33,11 @@ import org.apache.druid.initialization.DruidModule;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.server.security.AllowAllAuthenticator;
import org.apache.druid.server.security.AuthConfig;
import org.apache.druid.server.security.Authenticator;
import org.apache.druid.server.security.AuthenticatorMapper;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -50,7 +47,6 @@ import java.util.Set;
public class AuthenticatorMapperModule implements DruidModule
{
private static final String AUTHENTICATOR_PROPERTIES_FORMAT_STRING = "druid.auth.authenticator.%s";
private static Logger log = new Logger(AuthenticatorMapperModule.class);
@Override
public void configure(Binder binder)
@ -62,12 +58,6 @@ public class AuthenticatorMapperModule implements DruidModule
LifecycleModule.register(binder, AuthenticatorMapper.class);
}
@Override
public List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
private static class AuthenticatorMapperProvider implements Provider<AuthenticatorMapper>
{
private AuthConfig authConfig;

8
server/src/main/java/org/apache/druid/server/initialization/AuthorizerMapperModule.java
View File

@ -19,7 +19,6 @@
package org.apache.druid.server.initialization;
import com.fasterxml.jackson.databind.Module;
import com.google.common.base.Supplier;
import com.google.inject.Binder;
import com.google.inject.Inject;
@ -39,7 +38,6 @@ import org.apache.druid.server.security.AuthValidator;
import org.apache.druid.server.security.Authorizer;
import org.apache.druid.server.security.AuthorizerMapper;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -62,12 +60,6 @@ public class AuthorizerMapperModule implements DruidModule
LifecycleModule.register(binder, AuthorizerMapper.class);
}
@Override
public List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
private static class AuthorizerMapperProvider implements Provider<AuthorizerMapper>
{
private AuthConfig authConfig;

11
server/src/main/java/org/apache/druid/server/initialization/ExternalStorageAccessSecurityModule.java
View File

@ -19,23 +19,12 @@
package org.apache.druid.server.initialization;
import com.fasterxml.jackson.databind.Module;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.initialization.DruidModule;
import java.util.List;
public class ExternalStorageAccessSecurityModule implements DruidModule
{
@Override
public List<? extends Module> getJacksonModules()
{
return ImmutableList.of();
}
@Override
public void configure(Binder binder)
{

2
server/src/test/java/org/apache/druid/client/DruidServerConfigTest.java
View File

@ -61,7 +61,6 @@ public class DruidServerConfigTest
{
testSegmentCacheDir1 = tmpFolder.newFolder("segment_cache_folder1");
testSegmentCacheDir2 = tmpFolder.newFolder("segment_cache_folder2");
}
@Test
@ -74,7 +73,6 @@ public class DruidServerConfigTest
Assert.assertNotNull(druidServerConfig);
Assert.assertEquals(DruidServerConfig.class, druidServerConfig.getClass());
}
@Test

19
server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java
View File

@ -19,11 +19,7 @@
package org.apache.druid.curator;
import com.google.common.collect.ImmutableList;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.util.Modules;
import org.apache.curator.RetryPolicy;
import org.apache.curator.ensemble.EnsembleProvider;
import org.apache.curator.ensemble.exhibitor.ExhibitorEnsembleProvider;
@ -31,8 +27,8 @@ import org.apache.curator.ensemble.fixed.FixedEnsembleProvider;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.retry.BoundedExponentialBackoffRetry;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.druid.guice.GuiceInjectors;
import org.apache.druid.guice.LifecycleModule;
import org.apache.druid.guice.StartupInjectorBuilder;
import org.apache.druid.testing.junit.LoggerCaptureRule;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.LogEvent;
@ -178,14 +174,13 @@ public final class CuratorModuleTest
private Injector newInjector(final Properties props)
{
List<Module> modules = ImmutableList.<Module>builder()
.addAll(GuiceInjectors.makeDefaultStartupModules())
.add(new LifecycleModule())
.add(new CuratorModule())
return new StartupInjectorBuilder()
.add(
new LifecycleModule(),
new CuratorModule(),
binder -> binder.bind(Properties.class).toInstance(props)
)
.build();
return Guice.createInjector(
Modules.override(modules).with(binder -> binder.bind(Properties.class).toInstance(props))
);
}
private static CuratorFramework createCuratorFramework(Injector injector, int maxRetries)

631
server/src/test/java/org/apache/druid/initialization/InitializationTest.java
View File

@ -1,631 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.initialization;
import com.fasterxml.jackson.databind.Module;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.inject.Binder;
import com.google.inject.ConfigurationException;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.name.Names;
import org.apache.druid.discovery.NodeRole;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.guice.GuiceInjectors;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.guice.annotations.LoadScope;
import org.apache.druid.guice.annotations.Self;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.server.DruidNode;
import org.junit.Assert;
import org.junit.FixMethodOrder;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runners.MethodSorters;
import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class InitializationTest
{
@Rule
public final TemporaryFolder temporaryFolder = new TemporaryFolder();
@Test
public void test01InitialModulesEmpty()
{
Initialization.clearLoadedImplementations();
Assert.assertEquals(
"Initial set of loaded modules must be empty",
0,
Initialization.getLoadedImplementations(DruidModule.class).size()
);
}
@Test
public void test02MakeStartupInjector()
{
Injector startupInjector = GuiceInjectors.makeStartupInjector();
Assert.assertNotNull(startupInjector);
Assert.assertNotNull(startupInjector.getInstance(ObjectMapper.class));
}
@Test
public void test03ClassLoaderExtensionsLoading()
{
Injector startupInjector = GuiceInjectors.makeStartupInjector();
Function<DruidModule, String> fnClassName = new Function<DruidModule, String>()
{
@Nullable
@Override
public String apply(@Nullable DruidModule input)
{
return input.getClass().getName();
}
};
Assert.assertFalse(
"modules does not contain TestDruidModule",
Collections2.transform(Initialization.getLoadedImplementations(DruidModule.class), fnClassName)
.contains("org.apache.druid.initialization.InitializationTest.TestDruidModule")
);
Collection<DruidModule> modules = Initialization.getFromExtensions(
startupInjector.getInstance(ExtensionsConfig.class),
DruidModule.class
);
Assert.assertTrue(
"modules contains TestDruidModule",
Collections2.transform(modules, fnClassName).contains(TestDruidModule.class.getName())
);
}
@Test
public void test04DuplicateClassLoaderExtensions() throws Exception
{
final File extensionDir = temporaryFolder.newFolder();
Initialization.getLoadersMap()
.put(extensionDir, new URLClassLoader(new URL[]{}, Initialization.class.getClassLoader()));
Collection<DruidModule> modules = Initialization.getFromExtensions(new ExtensionsConfig(), DruidModule.class);
Set<String> loadedModuleNames = new HashSet<>();
for (DruidModule module : modules) {
Assert.assertFalse("Duplicate extensions are loaded", loadedModuleNames.contains(module.getClass().getName()));
loadedModuleNames.add(module.getClass().getName());
}
Initialization.getLoadersMap().clear();
}
@Test
public void test05MakeInjectorWithModules()
{
Injector startupInjector = GuiceInjectors.makeStartupInjector();
Injector injector = Initialization.makeInjectorWithModules(
startupInjector,
ImmutableList.<com.google.inject.Module>of(
new com.google.inject.Module()
{
@Override
public void configure(Binder binder)
{
JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
new DruidNode("test-inject", null, false, null, null, true, false)
);
}
}
)
);
Assert.assertNotNull(injector);
}
@Test
public void test06GetClassLoaderForExtension() throws IOException
{
final File some_extension_dir = temporaryFolder.newFolder();
final File a_jar = new File(some_extension_dir, "a.jar");
final File b_jar = new File(some_extension_dir, "b.jar");
final File c_jar = new File(some_extension_dir, "c.jar");
a_jar.createNewFile();
b_jar.createNewFile();
c_jar.createNewFile();
final URLClassLoader loader = Initialization.getClassLoaderForExtension(some_extension_dir, false);
final URL[] expectedURLs = new URL[]{a_jar.toURI().toURL(), b_jar.toURI().toURL(), c_jar.toURI().toURL()};
final URL[] actualURLs = loader.getURLs();
Arrays.sort(actualURLs, Comparator.comparing(URL::getPath));
Assert.assertArrayEquals(expectedURLs, actualURLs);
}
@Test
public void testGetLoadedModules()
{
Collection<DruidModule> modules = Initialization.getLoadedImplementations(DruidModule.class);
HashSet<DruidModule> moduleSet = new HashSet<>(modules);
Collection<DruidModule> loadedModules = Initialization.getLoadedImplementations(DruidModule.class);
Assert.assertEquals("Set from loaded modules #1 should be same!", modules.size(), loadedModules.size());
Assert.assertEquals("Set from loaded modules #1 should be same!", moduleSet, new HashSet<>(loadedModules));
Collection<DruidModule> loadedModules2 = Initialization.getLoadedImplementations(DruidModule.class);
Assert.assertEquals("Set from loaded modules #2 should be same!", modules.size(), loadedModules2.size());
Assert.assertEquals("Set from loaded modules #2 should be same!", moduleSet, new HashSet<>(loadedModules2));
}
@Test
public void testGetExtensionFilesToLoad_non_exist_extensions_dir() throws IOException
{
final File tmpDir = temporaryFolder.newFolder();
Assert.assertTrue("could not create missing folder", !tmpDir.exists() || tmpDir.delete());
Assert.assertArrayEquals(
"Non-exist root extensionsDir should return an empty array of File",
new File[]{},
Initialization.getExtensionFilesToLoad(new ExtensionsConfig()
{
@Override
public String getDirectory()
{
return tmpDir.getAbsolutePath();
}
})
);
}
@Test(expected = ISE.class)
public void testGetExtensionFilesToLoad_wrong_type_extensions_dir() throws IOException
{
final File extensionsDir = temporaryFolder.newFile();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getDirectory()
{
return extensionsDir.getAbsolutePath();
}
};
Initialization.getExtensionFilesToLoad(config);
}
@Test
public void testGetExtensionFilesToLoad_empty_extensions_dir() throws IOException
{
final File extensionsDir = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getDirectory()
{
return extensionsDir.getAbsolutePath();
}
};
Assert.assertArrayEquals(
"Empty root extensionsDir should return an empty array of File",
new File[]{},
Initialization.getExtensionFilesToLoad(config)
);
}
/**
* If druid.extension.load is not specified, Initialization.getExtensionFilesToLoad is supposed to return all the
* extension folders under root extensions directory.
*/
@Test
public void testGetExtensionFilesToLoad_null_load_list() throws IOException
{
final File extensionsDir = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getDirectory()
{
return extensionsDir.getAbsolutePath();
}
};
final File mysql_metadata_storage = new File(extensionsDir, "mysql-metadata-storage");
mysql_metadata_storage.mkdir();
final File[] expectedFileList = new File[]{mysql_metadata_storage};
final File[] actualFileList = Initialization.getExtensionFilesToLoad(config);
Arrays.sort(actualFileList);
Assert.assertArrayEquals(expectedFileList, actualFileList);
}
/**
* druid.extension.load is specified, Initialization.getExtensionFilesToLoad is supposed to return all the extension
* folders appeared in the load list.
*/
@Test
public void testGetExtensionFilesToLoad_with_load_list() throws IOException
{
final File extensionsDir = temporaryFolder.newFolder();
final File absolutePathExtension = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public LinkedHashSet<String> getLoadList()
{
return Sets.newLinkedHashSet(Arrays.asList("mysql-metadata-storage", absolutePathExtension.getAbsolutePath()));
}
@Override
public String getDirectory()
{
return extensionsDir.getAbsolutePath();
}
};
final File mysql_metadata_storage = new File(extensionsDir, "mysql-metadata-storage");
final File random_extension = new File(extensionsDir, "random-extensions");
mysql_metadata_storage.mkdir();
random_extension.mkdir();
final File[] expectedFileList = new File[]{mysql_metadata_storage, absolutePathExtension};
final File[] actualFileList = Initialization.getExtensionFilesToLoad(config);
Assert.assertArrayEquals(expectedFileList, actualFileList);
}
/**
* druid.extension.load is specified, but contains an extension that is not prepared under root extension directory.
* Initialization.getExtensionFilesToLoad is supposed to throw ISE.
*/
@Test(expected = ISE.class)
public void testGetExtensionFilesToLoad_with_non_exist_item_in_load_list() throws IOException
{
final File extensionsDir = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public LinkedHashSet<String> getLoadList()
{
return Sets.newLinkedHashSet(ImmutableList.of("mysql-metadata-storage"));
}
@Override
public String getDirectory()
{
return extensionsDir.getAbsolutePath();
}
};
final File random_extension = new File(extensionsDir, "random-extensions");
random_extension.mkdir();
Initialization.getExtensionFilesToLoad(config);
}
@Test(expected = ISE.class)
public void testGetHadoopDependencyFilesToLoad_wrong_type_root_hadoop_depenencies_dir() throws IOException
{
final File rootHadoopDependenciesDir = temporaryFolder.newFile();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getHadoopDependenciesDir()
{
return rootHadoopDependenciesDir.getAbsolutePath();
}
};
Initialization.getHadoopDependencyFilesToLoad(ImmutableList.of(), config);
}
@Test(expected = ISE.class)
public void testGetHadoopDependencyFilesToLoad_non_exist_version_dir() throws IOException
{
final File rootHadoopDependenciesDir = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getHadoopDependenciesDir()
{
return rootHadoopDependenciesDir.getAbsolutePath();
}
};
final File hadoopClient = new File(rootHadoopDependenciesDir, "hadoop-client");
hadoopClient.mkdir();
Initialization.getHadoopDependencyFilesToLoad(ImmutableList.of("org.apache.hadoop:hadoop-client:2.3.0"), config);
}
@Test
public void testGetHadoopDependencyFilesToLoad_with_hadoop_coordinates() throws IOException
{
final File rootHadoopDependenciesDir = temporaryFolder.newFolder();
final ExtensionsConfig config = new ExtensionsConfig()
{
@Override
public String getHadoopDependenciesDir()
{
return rootHadoopDependenciesDir.getAbsolutePath();
}
};
final File hadoopClient = new File(rootHadoopDependenciesDir, "hadoop-client");
final File versionDir = new File(hadoopClient, "2.3.0");
hadoopClient.mkdir();
versionDir.mkdir();
final File[] expectedFileList = new File[]{versionDir};
final File[] actualFileList = Initialization.getHadoopDependencyFilesToLoad(
ImmutableList.of(
"org.apache.hadoop:hadoop-client:2.3.0"
), config
);
Assert.assertArrayEquals(expectedFileList, actualFileList);
}
@Test
public void testGetURLsForClasspath() throws Exception
{
File tmpDir1 = temporaryFolder.newFolder();
File tmpDir2 = temporaryFolder.newFolder();
File tmpDir3 = temporaryFolder.newFolder();
File tmpDir1a = new File(tmpDir1, "a.jar");
tmpDir1a.createNewFile();
File tmpDir1b = new File(tmpDir1, "b.jar");
tmpDir1b.createNewFile();
new File(tmpDir1, "note1.txt").createNewFile();
File tmpDir2c = new File(tmpDir2, "c.jar");
tmpDir2c.createNewFile();
File tmpDir2d = new File(tmpDir2, "d.jar");
tmpDir2d.createNewFile();
File tmpDir2e = new File(tmpDir2, "e.JAR");
tmpDir2e.createNewFile();
new File(tmpDir2, "note2.txt").createNewFile();
String cp = tmpDir1.getAbsolutePath() + File.separator + "*"
+ File.pathSeparator
+ tmpDir3.getAbsolutePath()
+ File.pathSeparator
+ tmpDir2.getAbsolutePath() + File.separator + "*";
// getURLsForClasspath uses listFiles which does NOT guarantee any ordering for the name strings.
List<URL> urLsForClasspath = Initialization.getURLsForClasspath(cp);
Assert.assertEquals(Sets.newHashSet(tmpDir1a.toURI().toURL(), tmpDir1b.toURI().toURL()),
Sets.newHashSet(urLsForClasspath.subList(0, 2)));
Assert.assertEquals(tmpDir3.toURI().toURL(), urLsForClasspath.get(2));
Assert.assertEquals(Sets.newHashSet(tmpDir2c.toURI().toURL(), tmpDir2d.toURI().toURL(), tmpDir2e.toURI().toURL()),
Sets.newHashSet(urLsForClasspath.subList(3, 6)));
}
@Test
public void testExtensionsWithSameDirName() throws Exception
{
final String extensionName = "some_extension";
final File tmpDir1 = temporaryFolder.newFolder();
final File tmpDir2 = temporaryFolder.newFolder();
final File extension1 = new File(tmpDir1, extensionName);
final File extension2 = new File(tmpDir2, extensionName);
Assert.assertTrue(extension1.mkdir());
Assert.assertTrue(extension2.mkdir());
final File jar1 = new File(extension1, "jar1.jar");
final File jar2 = new File(extension2, "jar2.jar");
Assert.assertTrue(jar1.createNewFile());
Assert.assertTrue(jar2.createNewFile());
final ClassLoader classLoader1 = Initialization.getClassLoaderForExtension(extension1, false);
final ClassLoader classLoader2 = Initialization.getClassLoaderForExtension(extension2, false);
Assert.assertArrayEquals(new URL[]{jar1.toURI().toURL()}, ((URLClassLoader) classLoader1).getURLs());
Assert.assertArrayEquals(new URL[]{jar2.toURI().toURL()}, ((URLClassLoader) classLoader2).getURLs());
}
public static class TestDruidModule implements DruidModule
{
@Override
public List<? extends Module> getJacksonModules()
{
return ImmutableList.of();
}
@Override
public void configure(Binder binder)
{
// Do nothing
}
}
@Test
public void testCreateInjectorWithNodeRoles()
{
final DruidNode expected = new DruidNode("test-inject", null, false, null, null, true, false);
Injector startupInjector = GuiceInjectors.makeStartupInjector();
Injector injector = Initialization.makeInjectorWithModules(
ImmutableSet.of(new NodeRole("role1"), new NodeRole("role2")),
startupInjector,
ImmutableList.of(
binder -> JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
expected
)
)
);
Assert.assertNotNull(injector);
Assert.assertEquals(expected, injector.getInstance(Key.get(DruidNode.class, Self.class)));
}
@Test
public void testCreateInjectorWithNodeRoleFilter_moduleNotLoaded()
{
final DruidNode expected = new DruidNode("test-inject", null, false, null, null, true, false);
Injector startupInjector = GuiceInjectors.makeStartupInjector();
Injector injector = Initialization.makeInjectorWithModules(
ImmutableSet.of(new NodeRole("role1"), new NodeRole("role2")),
startupInjector,
ImmutableList.of(
(com.google.inject.Module) binder -> JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
expected
),
new LoadOnAnnotationTestModule()
)
);
Assert.assertNotNull(injector);
Assert.assertEquals(expected, injector.getInstance(Key.get(DruidNode.class, Self.class)));
Assert.assertThrows(
"Guice configuration errors",
ConfigurationException.class,
() -> injector.getInstance(Key.get(String.class, Names.named("emperor")))
);
}
@Test
public void testCreateInjectorWithNodeRoleFilterUsingAnnotation_moduleLoaded()
{
final DruidNode expected = new DruidNode("test-inject", null, false, null, null, true, false);
Injector startupInjector = GuiceInjectors.makeStartupInjector();
Injector injector = Initialization.makeInjectorWithModules(
ImmutableSet.of(new NodeRole("role1"), new NodeRole("druid")),
startupInjector,
ImmutableList.of(
(com.google.inject.Module) binder -> JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
expected
),
new LoadOnAnnotationTestModule()
)
);
Assert.assertNotNull(injector);
Assert.assertEquals(expected, injector.getInstance(Key.get(DruidNode.class, Self.class)));
Assert.assertEquals("I am Druid", injector.getInstance(Key.get(String.class, Names.named("emperor"))));
}
@LoadScope(roles = {"emperor", "druid"})
private static class LoadOnAnnotationTestModule implements com.google.inject.Module
{
@Override
public void configure(Binder binder)
{
binder.bind(String.class).annotatedWith(Names.named("emperor")).toInstance("I am Druid");
}
}
@Test
public void testCreateInjectorWithNodeRoleFilterUsingInject_moduleNotLoaded()
{
final Set<NodeRole> nodeRoles = ImmutableSet.of(new NodeRole("role1"), new NodeRole("role2"));
final DruidNode expected = new DruidNode("test-inject", null, false, null, null, true, false);
Injector startupInjector = GuiceInjectors.makeStartupInjectorWithModules(
ImmutableList.of(
binder -> {
Multibinder<NodeRole> selfBinder = Multibinder.newSetBinder(binder, NodeRole.class, Self.class);
nodeRoles.forEach(nodeRole -> selfBinder.addBinding().toInstance(nodeRole));
}
)
);
Injector injector = Initialization.makeInjectorWithModules(
nodeRoles,
startupInjector,
ImmutableList.of(
(com.google.inject.Module) binder -> JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
expected
),
new NodeRolesInjectTestModule()
)
);
Assert.assertNotNull(injector);
Assert.assertEquals(expected, injector.getInstance(Key.get(DruidNode.class, Self.class)));
Assert.assertThrows(
"Guice configuration errors",
ConfigurationException.class,
() -> injector.getInstance(Key.get(String.class, Names.named("emperor")))
);
}
@Test
public void testCreateInjectorWithNodeRoleFilterUsingInject_moduleLoaded()
{
final Set<NodeRole> nodeRoles = ImmutableSet.of(new NodeRole("role1"), new NodeRole("druid"));
final DruidNode expected = new DruidNode("test-inject", null, false, null, null, true, false);
Injector startupInjector = GuiceInjectors.makeStartupInjectorWithModules(
ImmutableList.of(
binder -> {
Multibinder<NodeRole> selfBinder = Multibinder.newSetBinder(binder, NodeRole.class, Self.class);
nodeRoles.forEach(nodeRole -> selfBinder.addBinding().toInstance(nodeRole));
}
)
);
Injector injector = Initialization.makeInjectorWithModules(
nodeRoles,
startupInjector,
ImmutableList.of(
(com.google.inject.Module) binder -> JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
expected
),
new NodeRolesInjectTestModule()
)
);
Assert.assertNotNull(injector);
Assert.assertEquals(expected, injector.getInstance(Key.get(DruidNode.class, Self.class)));
Assert.assertEquals("I am Druid", injector.getInstance(Key.get(String.class, Names.named("emperor"))));
}
private static class NodeRolesInjectTestModule implements com.google.inject.Module
{
private Set<NodeRole> nodeRoles;
@Inject
public void init(@Self Set<NodeRole> nodeRoles)
{
this.nodeRoles = nodeRoles;
}
@Override
public void configure(Binder binder)
{
if (nodeRoles.contains(new NodeRole("emperor")) || nodeRoles.contains(new NodeRole("druid"))) {
binder.bind(String.class).annotatedWith(Names.named("emperor")).toInstance("I am Druid");
}
}
}
}

283
server/src/test/java/org/apache/druid/initialization/ServerInjectorBuilderTest.java Normal file
View File

@ -0,0 +1,283 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.initialization;
import com.fasterxml.jackson.databind.Module;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Binder;
import com.google.inject.ConfigurationException;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.name.Names;
import org.apache.druid.discovery.NodeRole;
import org.apache.druid.guice.ExtensionsLoader;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.guice.StartupInjectorBuilder;
import org.apache.druid.guice.annotations.LoadScope;
import org.apache.druid.guice.annotations.Self;
import org.apache.druid.server.DruidNode;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import javax.annotation.Nullable;
import java.util.Collection;
import java.util.List;
import java.util.Set;
public class ServerInjectorBuilderTest
{
@Rule
public final TemporaryFolder temporaryFolder = new TemporaryFolder();
private Injector startupInjector()
{
return new StartupInjectorBuilder()
.withEmptyProperties()
.withExtensions()
.build();
}
@Test
public void test03ClassLoaderExtensionsLoading()
{
Injector startupInjector = startupInjector();
ExtensionsLoader extnLoader = ExtensionsLoader.instance(startupInjector);
Function<DruidModule, String> fnClassName = new Function<DruidModule, String>()
{
@Nullable
@Override
public String apply(@Nullable DruidModule input)
{
return input.getClass().getName();
}
};
Collection<DruidModule> modules = extnLoader.getFromExtensions(
DruidModule.class
);
Assert.assertTrue(
"modules contains TestDruidModule",
Collections2.transform(modules, fnClassName).contains(TestDruidModule.class.getName())
);
}
@Test
public void test05MakeInjectorWithModules()
{
Injector startupInjector = startupInjector();
ExtensionsLoader extnLoader = ExtensionsLoader.instance(startupInjector);
Injector injector = ServerInjectorBuilder.makeServerInjector(
startupInjector,
ImmutableSet.of(),
ImmutableList.<com.google.inject.Module>of(
new com.google.inject.Module()
{
@Override
public void configure(Binder binder)
{
JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
new DruidNode("test-inject", null, false, null, null, true, false)
);
}
}
)
);
Assert.assertNotNull(injector);
Assert.assertNotNull(ExtensionsLoader.instance(injector));
Assert.assertSame(extnLoader, ExtensionsLoader.instance(injector));
}
// Note: this name is referenced in
// src/test/resources/META-INF/services/org.apache.druid.initialization.DruidModule
// which impacts many tests, not just those here.
// This is likely a bug, not a feature.
public static class TestDruidModule implements DruidModule
{
@Override
public List<? extends Module> getJacksonModules()
{
return ImmutableList.of();
}
@Override
public void configure(Binder binder)
{
// Do nothing
}
}
@Test
public void testCreateInjectorWithNodeRoles()
{
final DruidNode expected = new DruidNode("test-inject", null, false, null, null, true, false);
Injector startupInjector = startupInjector();
Injector injector = ServerInjectorBuilder.makeServerInjector(
startupInjector,
ImmutableSet.of(new NodeRole("role1"), new NodeRole("role2")),
ImmutableList.of(
binder -> JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
expected
)
)
);
Assert.assertNotNull(injector);
Assert.assertEquals(expected, injector.getInstance(Key.get(DruidNode.class, Self.class)));
}
@Test
public void testCreateInjectorWithNodeRoleFilter_moduleNotLoaded()
{
final DruidNode expected = new DruidNode("test-inject", null, false, null, null, true, false);
Injector startupInjector = startupInjector();
Injector injector = ServerInjectorBuilder.makeServerInjector(
startupInjector,
ImmutableSet.of(new NodeRole("role1"), new NodeRole("role2")),
ImmutableList.of(
(com.google.inject.Module) binder -> JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
expected
),
new LoadOnAnnotationTestModule()
)
);
Assert.assertNotNull(injector);
Assert.assertEquals(expected, injector.getInstance(Key.get(DruidNode.class, Self.class)));
Assert.assertThrows(
"Guice configuration errors",
ConfigurationException.class,
() -> injector.getInstance(Key.get(String.class, Names.named("emperor")))
);
}
@Test
public void testCreateInjectorWithNodeRoleFilterUsingAnnotation_moduleLoaded()
{
final DruidNode expected = new DruidNode("test-inject", null, false, null, null, true, false);
Injector startupInjector = startupInjector();
Injector injector = ServerInjectorBuilder.makeServerInjector(
startupInjector,
ImmutableSet.of(new NodeRole("role1"), new NodeRole("druid")),
ImmutableList.of(
(com.google.inject.Module) binder -> JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
expected
),
new LoadOnAnnotationTestModule()
)
);
Assert.assertNotNull(injector);
Assert.assertEquals(expected, injector.getInstance(Key.get(DruidNode.class, Self.class)));
Assert.assertEquals("I am Druid", injector.getInstance(Key.get(String.class, Names.named("emperor"))));
}
@LoadScope(roles = {"emperor", "druid"})
private static class LoadOnAnnotationTestModule implements com.google.inject.Module
{
@Override
public void configure(Binder binder)
{
binder.bind(String.class).annotatedWith(Names.named("emperor")).toInstance("I am Druid");
}
}
@Test
public void testCreateInjectorWithNodeRoleFilterUsingInject_moduleNotLoaded()
{
final Set<NodeRole> nodeRoles = ImmutableSet.of(new NodeRole("role1"), new NodeRole("role2"));
final DruidNode expected = new DruidNode("test-inject", null, false, null, null, true, false);
Injector startupInjector = startupInjector();
Injector injector = ServerInjectorBuilder.makeServerInjector(
startupInjector,
nodeRoles,
ImmutableList.of(
(com.google.inject.Module) binder -> JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
expected
),
new NodeRolesInjectTestModule()
)
);
Assert.assertNotNull(injector);
Assert.assertEquals(expected, injector.getInstance(Key.get(DruidNode.class, Self.class)));
Assert.assertThrows(
"Guice configuration errors",
ConfigurationException.class,
() -> injector.getInstance(Key.get(String.class, Names.named("emperor")))
);
}
@Test
public void testCreateInjectorWithNodeRoleFilterUsingInject_moduleLoaded()
{
final Set<NodeRole> nodeRoles = ImmutableSet.of(new NodeRole("role1"), new NodeRole("druid"));
final DruidNode expected = new DruidNode("test-inject", null, false, null, null, true, false);
Injector startupInjector = startupInjector();
Injector injector = ServerInjectorBuilder.makeServerInjector(
startupInjector,
nodeRoles,
ImmutableList.of(
(com.google.inject.Module) binder -> JsonConfigProvider.bindInstance(
binder,
Key.get(DruidNode.class, Self.class),
expected
),
new NodeRolesInjectTestModule()
)
);
Assert.assertNotNull(injector);
Assert.assertEquals(expected, injector.getInstance(Key.get(DruidNode.class, Self.class)));
Assert.assertEquals("I am Druid", injector.getInstance(Key.get(String.class, Names.named("emperor"))));
}
private static class NodeRolesInjectTestModule implements com.google.inject.Module
{
private Set<NodeRole> nodeRoles;
@Inject
public void init(@Self Set<NodeRole> nodeRoles)
{
this.nodeRoles = nodeRoles;
}
@Override
public void configure(Binder binder)
{
if (nodeRoles.contains(new NodeRole("emperor")) || nodeRoles.contains(new NodeRole("druid"))) {
binder.bind(String.class).annotatedWith(Names.named("emperor")).toInstance("I am Druid");
}
}
}
}

26
server/src/test/java/org/apache/druid/query/lookup/LookupSerdeModuleTest.java
View File

@ -19,15 +19,13 @@
package org.apache.druid.query.lookup;
import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.inject.Injector;
import com.google.inject.Key;
import org.apache.druid.guice.ExpressionModule;
import org.apache.druid.guice.GuiceInjectors;
import org.apache.druid.guice.StartupInjectorBuilder;
import org.apache.druid.guice.annotations.Json;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.initialization.CoreInjectorBuilder;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.query.dimension.ExtractionDimensionSpec;
@ -49,22 +47,14 @@ public class LookupSerdeModuleTest
@Before
public void setUp()
{
final ImmutableList<DruidModule> modules = ImmutableList.of(
new ExpressionModule(),
new LookupSerdeModule()
);
injector = new CoreInjectorBuilder(new StartupInjectorBuilder().build())
.add(
new ExpressionModule(),
new LookupSerdeModule()
)
.build();
injector = GuiceInjectors.makeStartupInjectorWithModules(modules);
objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
objectMapper.setInjectableValues(
new InjectableValues.Std()
.addValue(ExprMacroTable.class, injector.getInstance(ExprMacroTable.class))
.addValue(
LookupExtractorFactoryContainerProvider.class,
injector.getInstance(LookupExtractorFactoryContainerProvider.class)
)
);
modules.stream().flatMap(module -> module.getJacksonModules().stream()).forEach(objectMapper::registerModule);
}
@Test

4
server/src/test/java/org/apache/druid/server/StatusResourceTest.java
View File

@ -25,7 +25,7 @@ import com.google.inject.Guice;
import com.google.inject.Injector;
import org.apache.druid.guice.PropertiesModule;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.initialization.InitializationTest;
import org.apache.druid.initialization.ServerInjectorBuilderTest;
import org.junit.Assert;
import org.junit.Test;
@ -42,7 +42,7 @@ public class StatusResourceTest
public void testLoadedModules()
{
Collection<DruidModule> modules = ImmutableList.of(new InitializationTest.TestDruidModule());
Collection<DruidModule> modules = ImmutableList.of(new ServerInjectorBuilderTest.TestDruidModule());
List<StatusResource.ModuleVersion> statusResourceModuleList = new StatusResource.Status(modules).getModules();
Assert.assertEquals("Status should have all modules loaded!", modules.size(), statusResourceModuleList.size());

2
server/src/test/resources/META-INF/services/org.apache.druid.initialization.DruidModule
View File

@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.druid.initialization.InitializationTest$TestDruidModule
org.apache.druid.initialization.ServerInjectorBuilderTest$TestDruidModule

1
services/pom.xml
View File

@ -324,6 +324,7 @@
</usedDependencies>
<ignoredUsedUndeclaredDependencies>
<ignoredUsedUndeclaredDependency>jakarta.xml.bind:jakarta.xml.bind-api</ignoredUsedUndeclaredDependency>
<ignoredUsedUndeclaredDependency>jakarta.inject:jakarta.inject-api</ignoredUsedUndeclaredDependency>
</ignoredUsedUndeclaredDependencies>
</configuration>
</plugin>

17
services/src/main/java/org/apache/druid/cli/CliHadoopIndexer.java
View File

@ -25,9 +25,9 @@ import com.github.rvesse.airline.annotations.Option;
import com.github.rvesse.airline.annotations.restrictions.Required;
import com.google.common.base.Joiner;
import com.google.inject.Inject;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.guice.ExtensionsLoader;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.initialization.Initialization;
import org.apache.druid.indexing.common.task.Initialization;
import org.apache.druid.java.util.common.logger.Logger;
import java.io.File;
@ -46,7 +46,6 @@ import java.util.List;
)
public class CliHadoopIndexer implements Runnable
{
private static final List<String> DEFAULT_HADOOP_COORDINATES = TaskConfig.DEFAULT_DEFAULT_HADOOP_COORDINATES;
private static final Logger log = new Logger(CliHadoopIndexer.class);
@ -65,10 +64,9 @@ public class CliHadoopIndexer implements Runnable
public boolean noDefaultHadoop;
@Inject
private ExtensionsConfig extensionsConfig = null;
private ExtensionsLoader extnLoader;
@Override
@SuppressWarnings("unchecked")
public void run()
{
try {
@ -81,8 +79,8 @@ public class CliHadoopIndexer implements Runnable
}
final List<URL> extensionURLs = new ArrayList<>();
for (final File extension : Initialization.getExtensionFilesToLoad(extensionsConfig)) {
final URLClassLoader extensionLoader = Initialization.getClassLoaderForExtension(extension, false);
for (final File extension : extnLoader.getExtensionFilesToLoad()) {
final URLClassLoader extensionLoader = extnLoader.getClassLoaderForExtension(extension, false);
extensionURLs.addAll(Arrays.asList(extensionLoader.getURLs()));
}
@ -92,8 +90,8 @@ public class CliHadoopIndexer implements Runnable
final List<URL> driverURLs = new ArrayList<>(nonHadoopURLs);
// put hadoop dependencies last to avoid jets3t & apache.httpcore version conflicts
for (File hadoopDependency : Initialization.getHadoopDependencyFilesToLoad(allCoordinates, extensionsConfig)) {
final URLClassLoader hadoopLoader = Initialization.getClassLoaderForExtension(hadoopDependency, false);
for (File hadoopDependency : Initialization.getHadoopDependencyFilesToLoad(allCoordinates, extnLoader.config())) {
final URLClassLoader hadoopLoader = extnLoader.getClassLoaderForExtension(hadoopDependency, false);
driverURLs.addAll(Arrays.asList(hadoopLoader.getURLs()));
}
@ -122,5 +120,4 @@ public class CliHadoopIndexer implements Runnable
System.exit(1);
}
}
}

38
services/src/main/java/org/apache/druid/cli/GuiceRunnable.java
View File

@ -19,21 +19,14 @@
package org.apache.druid.cli;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.MapBinder;
import com.google.inject.multibindings.Multibinder;
import org.apache.druid.discovery.DruidService;
import org.apache.druid.discovery.NodeRole;
import org.apache.druid.guice.annotations.Self;
import org.apache.druid.initialization.Initialization;
import org.apache.druid.initialization.ServerInjectorBuilder;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.lifecycle.Lifecycle;
import org.apache.druid.java.util.common.logger.Logger;
@ -88,41 +81,14 @@ public abstract class GuiceRunnable implements Runnable
public Injector makeInjector(Set<NodeRole> nodeRoles)
{
Module registerNodeRoleModule = registerNodeRoleModule(nodeRoles);
try {
return Initialization.makeInjectorWithModules(
nodeRoles,
baseInjector.createChildInjector(registerNodeRoleModule),
Iterables.concat(
// bind nodeRoles for the new injector as well
ImmutableList.of(registerNodeRoleModule),
getModules()
)
);
return ServerInjectorBuilder.makeServerInjector(baseInjector, nodeRoles, getModules());
}
catch (Exception e) {
throw new RuntimeException(e);
}
}
static Module registerNodeRoleModule(Set<NodeRole> nodeRoles)
{
if (nodeRoles.isEmpty()) {
return binder -> {};
} else {
return binder -> {
Multibinder<NodeRole> selfBinder = Multibinder.newSetBinder(binder, NodeRole.class, Self.class);
nodeRoles.forEach(nodeRole -> selfBinder.addBinding().toInstance(nodeRole));
MapBinder.newMapBinder(
binder,
new TypeLiteral<NodeRole>(){},
new TypeLiteral<Set<Class<? extends DruidService>>>(){}
);
};
}
}
public Lifecycle initLifecycle(Injector injector)
{
return initLifecycle(injector, log);

14
services/src/main/java/org/apache/druid/cli/Main.java
View File

@ -26,9 +26,8 @@ import com.github.rvesse.airline.parser.errors.ParseException;
import com.google.inject.Injector;
import io.netty.util.SuppressForbidden;
import org.apache.druid.cli.validate.DruidJsonValidator;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.guice.GuiceInjectors;
import org.apache.druid.initialization.Initialization;
import org.apache.druid.guice.ExtensionsLoader;
import org.apache.druid.guice.StartupInjectorBuilder;
import java.util.Arrays;
import java.util.Collection;
@ -94,11 +93,10 @@ public class Main
.withDefaultCommand(Help.class)
.withCommands(CliPeon.class, CliInternalHadoopIndexer.class);
final Injector injector = GuiceInjectors.makeStartupInjector();
final ExtensionsConfig config = injector.getInstance(ExtensionsConfig.class);
final Collection<CliCommandCreator> extensionCommands = Initialization.getFromExtensions(
config,
CliCommandCreator.class
final Injector injector = new StartupInjectorBuilder().forServer().build();
ExtensionsLoader extnLoader = ExtensionsLoader.instance(injector);
final Collection<CliCommandCreator> extensionCommands = extnLoader.getFromExtensions(
CliCommandCreator.class
);
for (CliCommandCreator creator : extensionCommands) {

13
services/src/main/java/org/apache/druid/cli/Version.java
View File

@ -21,20 +21,27 @@ package org.apache.druid.cli;
import com.github.rvesse.airline.annotations.Command;
import io.netty.util.SuppressForbidden;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.initialization.Initialization;
import org.apache.druid.guice.ExtensionsLoader;
import org.apache.druid.server.StatusResource;
import javax.inject.Inject;
@Command(
name = "version",
description = "Returns Druid version information"
)
public class Version implements Runnable
{
@Inject
private ExtensionsLoader extnLoader;
@Override
@SuppressForbidden(reason = "System#out")
public void run()
{
System.out.println(new StatusResource.Status(Initialization.getLoadedImplementations(DruidModule.class)));
// Since Version is a command, we don't go through the server
// process to load modules and they are thus not already loaded.
// Explicitly load them here.
System.out.println(new StatusResource.Status(extnLoader.getModules()));
}
}

6
services/src/main/java/org/apache/druid/cli/validate/DruidJsonValidator.java
View File

@ -39,7 +39,7 @@ import org.apache.druid.cli.GuiceRunnable;
import org.apache.druid.data.input.InputRow;
import org.apache.druid.data.input.impl.StringInputRowParser;
import org.apache.druid.guice.DruidProcessingModule;
import org.apache.druid.guice.ExtensionsConfig;
import org.apache.druid.guice.ExtensionsLoader;
import org.apache.druid.guice.FirehoseModule;
import org.apache.druid.guice.IndexingServiceFirehoseModule;
import org.apache.druid.guice.IndexingServiceInputSourceModule;
@ -50,7 +50,6 @@ import org.apache.druid.indexer.HadoopDruidIndexerConfig;
import org.apache.druid.indexer.IndexingHadoopModule;
import org.apache.druid.indexing.common.task.Task;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.initialization.Initialization;
import org.apache.druid.java.util.common.UOE;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.query.Query;
@ -124,11 +123,12 @@ public class DruidJsonValidator extends GuiceRunnable
final Injector injector = makeInjector();
final ObjectMapper jsonMapper = injector.getInstance(ObjectMapper.class);
ExtensionsLoader extnLoader = injector.getInstance(ExtensionsLoader.class);
registerModules(
jsonMapper,
Iterables.concat(
Initialization.getFromExtensions(injector.getInstance(ExtensionsConfig.class), DruidModule.class),
extnLoader.getModules(),
Arrays.asList(
new FirehoseModule(),
new IndexingHadoopModule(),

7
services/src/main/java/org/apache/druid/guice/AbstractDruidServiceModule.java
View File

@ -33,8 +33,9 @@ import java.util.Set;
/**
* An abstract module for dynamic registration of {@link DruidService}.
* DruidServices are bound to a set which is mapped to a certain {@link NodeRole}.
* See {@link org.apache.druid.cli.GuiceRunnable#registerNodeRoleModule} for how the map is bound.
*
* See {@link org.apache.druid.initialization.ServerInjectorBuilder#registerNodeRoleModule}
* for how the map is bound.
* <p>
* To register a DruidService, create a class something like below:
*
* <pre>
@ -63,7 +64,7 @@ public abstract class AbstractDruidServiceModule implements Module
}
/**
* A helper method for extensions which do not implement Module directly.
* A helper method for extensions which do not implement Module directly.
*/
public static void configure(Binder binder, NodeRole role)
{

10
services/src/main/java/org/apache/druid/guice/QueryablePeonModule.java
View File

@ -19,23 +19,13 @@
package org.apache.druid.guice;
import com.fasterxml.jackson.databind.Module;
import com.google.inject.Binder;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.server.QueryResource;
import org.apache.druid.server.metrics.QueryCountStatsProvider;
import java.util.Collections;
import java.util.List;
public class QueryablePeonModule implements DruidModule
{
@Override
public List<? extends Module> getJacksonModules()
{
return Collections.emptyList();
}
@Override
public void configure(Binder binder)
{

16
services/src/test/java/org/apache/druid/cli/DiscoverySideEffectsProviderTest.java
View File

@ -32,8 +32,9 @@ import org.apache.druid.discovery.DruidNodeAnnouncer;
import org.apache.druid.discovery.DruidService;
import org.apache.druid.discovery.NodeRole;
import org.apache.druid.guice.AbstractDruidServiceModule;
import org.apache.druid.guice.GuiceInjectors;
import org.apache.druid.guice.StartupInjectorBuilder;
import org.apache.druid.guice.annotations.Self;
import org.apache.druid.initialization.ServerInjectorBuilder;
import org.apache.druid.java.util.common.lifecycle.Lifecycle;
import org.apache.druid.server.DruidNode;
import org.junit.Assert;
@ -180,18 +181,17 @@ public class DiscoverySideEffectsProviderTest
private Injector createInjector(List<Module> modules)
{
List<Module> finalModules = new ArrayList<>(modules);
finalModules.addAll(
ImmutableList.of(
GuiceRunnable.registerNodeRoleModule(ImmutableSet.of(nodeRole)),
return new StartupInjectorBuilder()
.add(
ServerInjectorBuilder.registerNodeRoleModule(ImmutableSet.of(nodeRole)),
binder -> {
binder.bind(DruidNodeAnnouncer.class).toInstance(discoverableOnlyAnnouncer);
binder.bind(DruidNode.class).annotatedWith(Self.class).toInstance(druidNode);
binder.bind(ServiceAnnouncer.class).toInstance(legacyAnnouncer);
binder.bind(Lifecycle.class).toInstance(lifecycle);
}
)
);
return GuiceInjectors.makeStartupInjectorWithModules(finalModules);
)
.addAll(modules)
.build();
}
}

106
sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestInjectorBuilder.java Normal file
View File

@ -0,0 +1,106 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.sql.calcite.util;
import com.fasterxml.jackson.databind.Module;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Binder;
import com.google.inject.Injector;
import org.apache.druid.guice.StartupInjectorBuilder;
import org.apache.druid.initialization.CoreInjectorBuilder;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.query.expression.LookupEnabledTestExprMacroTable;
import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider;
import org.apache.druid.query.lookup.LookupSerdeModule;
import org.apache.druid.sql.calcite.aggregation.SqlAggregationModule;
import org.apache.druid.sql.calcite.expression.builtin.QueryLookupOperatorConversion;
import org.apache.druid.sql.calcite.external.ExternalOperatorConversion;
import org.apache.druid.sql.guice.SqlBindings;
import org.apache.druid.timeline.DataSegment;
import java.util.List;
/**
* Create the injector used for {@link CalciteTests#INJECTOR}, but in a way
* that is extensible.
*/
public class CalciteTestInjectorBuilder extends CoreInjectorBuilder
{
public CalciteTestInjectorBuilder()
{
super(new StartupInjectorBuilder()
.withEmptyProperties()
.build());
add(
new BasicTestModule(),
new SqlAggregationModule()
);
}
@Override
public Injector build()
{
try {
return super.build();
}
catch (Exception e) {
// Catches failures when used as a static initializer.
e.printStackTrace();
System.exit(1);
throw e;
}
}
private static class BasicTestModule implements DruidModule
{
@Override
public void configure(Binder binder)
{
final LookupExtractorFactoryContainerProvider lookupProvider =
LookupEnabledTestExprMacroTable.createTestLookupProvider(
ImmutableMap.of(
"a", "xa",
"abc", "xabc",
"nosuchkey", "mysteryvalue",
"6", "x6"
)
);
binder.bind(ExprMacroTable.class).toInstance(TestExprMacroTable.INSTANCE);
binder.bind(DataSegment.PruneSpecsHolder.class).toInstance(DataSegment.PruneSpecsHolder.DEFAULT);
binder.bind(LookupExtractorFactoryContainerProvider.class).toInstance(lookupProvider);
// This Module is just to get a LookupExtractorFactoryContainerProvider with a usable "lookyloo" lookup.
binder.bind(LookupExtractorFactoryContainerProvider.class).toInstance(lookupProvider);
SqlBindings.addOperatorConversion(binder, QueryLookupOperatorConversion.class);
// Add "EXTERN" table macro, for CalciteInsertDmlTest.
SqlBindings.addOperatorConversion(binder, ExternalOperatorConversion.class);
}
@Override
public List<? extends Module> getJacksonModules()
{
return new LookupSerdeModule().getJacksonModules();
}
}
}

47
sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java
View File

@ -19,7 +19,6 @@
package org.apache.druid.sql.calcite.util;
import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Predicate;
import com.google.common.base.Suppliers;
@ -28,7 +27,6 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import org.apache.calcite.jdbc.CalciteSchema;
@ -81,14 +79,10 @@ import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
import org.apache.druid.query.aggregation.FloatSumAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory;
import org.apache.druid.query.expression.LookupEnabledTestExprMacroTable;
import org.apache.druid.query.expression.LookupExprMacro;
import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider;
import org.apache.druid.query.lookup.LookupSerdeModule;
import org.apache.druid.segment.IndexBuilder;
import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.TestHelper;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
@ -119,9 +113,6 @@ import org.apache.druid.server.security.Escalator;
import org.apache.druid.server.security.NoopEscalator;
import org.apache.druid.server.security.ResourceType;
import org.apache.druid.sql.SqlLifecycleFactory;
import org.apache.druid.sql.calcite.aggregation.SqlAggregationModule;
import org.apache.druid.sql.calcite.expression.builtin.QueryLookupOperatorConversion;
import org.apache.druid.sql.calcite.external.ExternalOperatorConversion;
import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerFactory;
@ -143,7 +134,6 @@ import org.apache.druid.sql.calcite.schema.SystemSchema;
import org.apache.druid.sql.calcite.schema.ViewSchema;
import org.apache.druid.sql.calcite.view.DruidViewMacroFactory;
import org.apache.druid.sql.calcite.view.ViewManager;
import org.apache.druid.sql.guice.SqlBindings;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.LinearShardSpec;
import org.easymock.EasyMock;
@ -152,6 +142,7 @@ import org.joda.time.Duration;
import org.joda.time.chrono.ISOChronology;
import javax.annotation.Nullable;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
@ -263,41 +254,7 @@ public class CalciteTests
private static final String TIMESTAMP_COLUMN = "t";
public static final Injector INJECTOR = Guice.createInjector(
binder -> {
final LookupExtractorFactoryContainerProvider lookupProvider =
LookupEnabledTestExprMacroTable.createTestLookupProvider(
ImmutableMap.of(
"a", "xa",
"abc", "xabc",
"nosuchkey", "mysteryvalue",
"6", "x6"
)
);
ObjectMapper mapper = TestHelper.makeJsonMapper().registerModules(
new LookupSerdeModule().getJacksonModules()
);
mapper.setInjectableValues(
new InjectableValues.Std()
.addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE)
.addValue(ObjectMapper.class.getName(), mapper)
.addValue(DataSegment.PruneSpecsHolder.class, DataSegment.PruneSpecsHolder.DEFAULT)
.addValue(LookupExtractorFactoryContainerProvider.class.getName(), lookupProvider)
);
binder.bind(Key.get(ObjectMapper.class, Json.class)).toInstance(
mapper
);
// This Module is just to get a LookupExtractorFactoryContainerProvider with a usable "lookyloo" lookup.
binder.bind(LookupExtractorFactoryContainerProvider.class).toInstance(lookupProvider);
SqlBindings.addOperatorConversion(binder, QueryLookupOperatorConversion.class);
// Add "EXTERN" table macro, for CalciteInsertDmlTest.
SqlBindings.addOperatorConversion(binder, ExternalOperatorConversion.class);
},
new SqlAggregationModule()
);
public static final Injector INJECTOR = new CalciteTestInjectorBuilder().build();
private static final InputRowParser<Map<String, Object>> PARSER = new MapInputRowParser(
new TimeAndDimsParseSpec(

15
sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java
View File

@ -73,8 +73,7 @@ import org.junit.runner.RunWith;
import javax.validation.Validation;
import javax.validation.Validator;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@ -148,7 +147,7 @@ public class SqlModuleTest
Assert.assertNotNull(viewManager);
Assert.assertTrue(viewManager instanceof NoopViewManager);
}
@Test
public void testNonDefaultViewManagerBind()
{
@ -207,12 +206,6 @@ public class SqlModuleTest
private static class TestViewManagerModule implements DruidModule
{
@Override
public List<? extends com.fasterxml.jackson.databind.Module> getJacksonModules()
{
return Collections.emptyList();
}
@Override
public void configure(Binder binder)
{
@ -225,7 +218,6 @@ public class SqlModuleTest
private static class BindTestViewManager implements ViewManager
{
@Override
public void createView(
PlannerFactory plannerFactory,
@ -233,7 +225,6 @@ public class SqlModuleTest
String viewSql
)
{
}
@Override
@ -243,13 +234,11 @@ public class SqlModuleTest
String viewSql
)
{
}
@Override
public void dropView(String viewName)
{
}
@Override