From 98d05e63bf1b3ac96cb8db67d4b5bc7f7cc5f0e4 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 10 Nov 2014 15:34:49 -0800 Subject: [PATCH 1/3] make derby instance only on coordinator --- .../MetadataStorageConnectorConfig.java | 2 +- .../DerbyMetadataStorageDruidModule.java | 9 ++- ...gModule.java => MetadataConfigModule.java} | 4 +- .../guice/SQLMetadataStorageDruidModule.java | 4 ++ .../druid/initialization/Initialization.java | 4 +- .../io/druid/metadata/DerbyConnector.java | 31 ++++----- .../druid/metadata/DerbyMetadataStorage.java | 69 +++++++++++++++++++ .../io/druid/metadata/MetadataStorage.java | 21 ++++++ .../java/io/druid/cli/CliCoordinator.java | 17 +++-- 9 files changed, 129 insertions(+), 32 deletions(-) rename server/src/main/java/io/druid/guice/{MetadataDbConfigModule.java => MetadataConfigModule.java} (96%) create mode 100644 server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java create mode 100644 server/src/main/java/io/druid/metadata/MetadataStorage.java diff --git a/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java b/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java index 07447c36a74..329f0de1d26 100644 --- a/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java +++ b/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java @@ -29,7 +29,7 @@ public class MetadataStorageConnectorConfig private boolean createTables = true; @JsonProperty - private String connectURI = null; + private String connectURI = "jdbc:derby://localhost:1527/druid;create=true"; @JsonProperty private String user = null; diff --git a/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java b/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java index 5dfed2ca4a6..c698da8f0ed 100644 --- a/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java +++ b/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java @@ -21,11 +21,11 @@ package io.druid.guice; import com.google.inject.Binder; import com.google.inject.Key; -import com.google.inject.Provides; import io.druid.metadata.DerbyConnector; +import io.druid.metadata.DerbyMetadataStorage; +import io.druid.metadata.MetadataStorage; import io.druid.metadata.MetadataStorageConnector; import io.druid.metadata.SQLMetadataConnector; -import org.skife.jdbi.v2.IDBI; public class DerbyMetadataStorageDruidModule extends SQLMetadataStorageDruidModule { @@ -42,6 +42,11 @@ public class DerbyMetadataStorageDruidModule extends SQLMetadataStorageDruidModu createBindingChoices(binder, TYPE); super.configure(binder); + PolyBind.optionBinder(binder, Key.get(MetadataStorage.class)) + .addBinding(TYPE) + .to(DerbyMetadataStorage.class) + .in(LazySingleton.class); + PolyBind.optionBinder(binder, Key.get(MetadataStorageConnector.class)) .addBinding(TYPE) .to(DerbyConnector.class) diff --git a/server/src/main/java/io/druid/guice/MetadataDbConfigModule.java b/server/src/main/java/io/druid/guice/MetadataConfigModule.java similarity index 96% rename from server/src/main/java/io/druid/guice/MetadataDbConfigModule.java rename to server/src/main/java/io/druid/guice/MetadataConfigModule.java index 71b27b9793f..be08f43fa72 100644 --- a/server/src/main/java/io/druid/guice/MetadataDbConfigModule.java +++ b/server/src/main/java/io/druid/guice/MetadataConfigModule.java @@ -21,12 +21,12 @@ package io.druid.guice; import com.google.inject.Binder; import com.google.inject.Module; -import io.druid.metadata.MetadataStorageConnectorConfig; import io.druid.metadata.MetadataRuleManagerConfig; import io.druid.metadata.MetadataSegmentManagerConfig; +import io.druid.metadata.MetadataStorageConnectorConfig; import io.druid.metadata.MetadataStorageTablesConfig; -public class MetadataDbConfigModule implements Module +public class MetadataConfigModule implements Module { @Override public void configure(Binder binder) diff --git a/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java index f69c68e0119..96468ac075e 100644 --- a/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java +++ b/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java @@ -25,6 +25,7 @@ import com.google.inject.Module; import io.druid.indexer.MetadataStorageUpdaterJobHandler; import io.druid.indexer.SQLMetadataStorageUpdaterJobHandler; import io.druid.indexing.overlord.IndexerMetadataStorageCoordinator; +import io.druid.metadata.MetadataStorage; import io.druid.metadata.MetadataStorageActionHandlerFactory; import io.druid.metadata.IndexerSQLMetadataStorageCoordinator; import io.druid.metadata.MetadataRuleManager; @@ -62,6 +63,9 @@ public class SQLMetadataStorageDruidModule implements Module PolyBind.createChoiceWithDefault( binder, PROPERTY, Key.get(MetadataStorageConnector.class), null, defaultPropertyValue ); + PolyBind.createChoiceWithDefault( + binder, PROPERTY, Key.get(MetadataStorage.class), Key.get(MetadataStorage.class), defaultPropertyValue + ); PolyBind.createChoiceWithDefault( binder, PROPERTY, Key.get(SQLMetadataConnector.class), null, defaultPropertyValue ); diff --git a/server/src/main/java/io/druid/initialization/Initialization.java b/server/src/main/java/io/druid/initialization/Initialization.java index 32c6e89e93e..7bd7fc6e1fb 100644 --- a/server/src/main/java/io/druid/initialization/Initialization.java +++ b/server/src/main/java/io/druid/initialization/Initialization.java @@ -45,7 +45,7 @@ import io.druid.guice.IndexingServiceDiscoveryModule; import io.druid.guice.JacksonConfigManagerModule; import io.druid.guice.LifecycleModule; import io.druid.guice.LocalDataStorageDruidModule; -import io.druid.guice.MetadataDbConfigModule; +import io.druid.guice.MetadataConfigModule; import io.druid.guice.ParsersModule; import io.druid.guice.QueryRunnerFactoryModule; import io.druid.guice.QueryableModule; @@ -333,7 +333,7 @@ public class Initialization new QueryRunnerFactoryModule(), new DiscoveryModule(), new ServerViewModule(), - new MetadataDbConfigModule(), + new MetadataConfigModule(), new DerbyMetadataStorageDruidModule(), new JacksonConfigManagerModule(), new IndexingServiceDiscoveryModule(), diff --git a/server/src/main/java/io/druid/metadata/DerbyConnector.java b/server/src/main/java/io/druid/metadata/DerbyConnector.java index 70ba42f7fc6..ad57c55be00 100644 --- a/server/src/main/java/io/druid/metadata/DerbyConnector.java +++ b/server/src/main/java/io/druid/metadata/DerbyConnector.java @@ -19,15 +19,11 @@ package io.druid.metadata; -import com.google.api.client.repackaged.com.google.common.base.Throwables; import com.google.common.base.Supplier; import com.google.inject.Inject; -import org.apache.derby.drda.NetworkServerControl; +import org.apache.commons.dbcp2.BasicDataSource; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; -import org.skife.jdbi.v2.tweak.ConnectionFactory; - -import java.net.InetAddress; public class DerbyConnector extends SQLMetadataConnector { @@ -37,10 +33,20 @@ public class DerbyConnector extends SQLMetadataConnector @Inject public DerbyConnector(Supplier config, Supplier dbTables) { - this(config, dbTables, new DBI(getConnectionFactory("druidDerbyDb"))); + super(config, dbTables); + + final BasicDataSource datasource = getDatasource(); + datasource.setDriverClassLoader(getClass().getClassLoader()); + datasource.setDriverClassName("org.apache.derby.jdbc.ClientDriver"); + + this.dbi = new DBI(datasource); } - public DerbyConnector(Supplier config, Supplier dbTables, DBI dbi) + public DerbyConnector( + Supplier config, + Supplier dbTables, + DBI dbi + ) { super(config, dbTables); this.dbi = dbi; @@ -63,15 +69,4 @@ public class DerbyConnector extends SQLMetadataConnector @Override public DBI getDBI() { return dbi; } - - private static ConnectionFactory getConnectionFactory(String dbName) - { - try { - NetworkServerControl server = new NetworkServerControl(InetAddress.getByName("localhost"),1527); - server.start(null); - } catch (Exception e) { - throw Throwables.propagate(e); - } - return new DerbyConnectionFactory(dbName); - } } diff --git a/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java b/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java new file mode 100644 index 00000000000..647b1ffb791 --- /dev/null +++ b/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java @@ -0,0 +1,69 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.metadata; + +import com.google.common.base.Throwables; +import com.metamx.common.lifecycle.LifecycleStart; +import com.metamx.common.lifecycle.LifecycleStop; +import org.apache.derby.drda.NetworkServerControl; + +import java.net.InetAddress; + +/** + */ +public class DerbyMetadataStorage extends MetadataStorage +{ + private final NetworkServerControl server; + + public DerbyMetadataStorage() + { + try { + this.server = new NetworkServerControl(InetAddress.getByName("localhost"), 1527); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + } + + @Override + @LifecycleStart + public void start() + { + try { + server.start(null); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + @Override + @LifecycleStop + public void stop() + { + try { + server.shutdown(); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } +} diff --git a/server/src/main/java/io/druid/metadata/MetadataStorage.java b/server/src/main/java/io/druid/metadata/MetadataStorage.java new file mode 100644 index 00000000000..2bdd3378385 --- /dev/null +++ b/server/src/main/java/io/druid/metadata/MetadataStorage.java @@ -0,0 +1,21 @@ +package io.druid.metadata; + +import com.metamx.common.lifecycle.LifecycleStart; +import com.metamx.common.lifecycle.LifecycleStop; + +/** + */ +public abstract class MetadataStorage +{ + @LifecycleStart + public void start() + { + // do nothing + } + + @LifecycleStop + public void stop() + { + // do nothing + } +} diff --git a/services/src/main/java/io/druid/cli/CliCoordinator.java b/services/src/main/java/io/druid/cli/CliCoordinator.java index b1f670350bb..9470b4c1542 100644 --- a/services/src/main/java/io/druid/cli/CliCoordinator.java +++ b/services/src/main/java/io/druid/cli/CliCoordinator.java @@ -29,18 +29,19 @@ import com.metamx.common.concurrent.ScheduledExecutorFactory; import com.metamx.common.logger.Logger; import io.airlift.command.Command; import io.druid.client.indexing.IndexingServiceClient; -import io.druid.metadata.MetadataRuleManager; -import io.druid.metadata.MetadataRuleManagerConfig; -import io.druid.metadata.MetadataRuleManagerProvider; -import io.druid.metadata.MetadataSegmentManager; -import io.druid.metadata.MetadataSegmentManagerConfig; -import io.druid.metadata.MetadataSegmentManagerProvider; import io.druid.guice.ConfigProvider; import io.druid.guice.Jerseys; import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; import io.druid.guice.LifecycleModule; import io.druid.guice.ManageLifecycle; +import io.druid.metadata.MetadataRuleManager; +import io.druid.metadata.MetadataRuleManagerConfig; +import io.druid.metadata.MetadataRuleManagerProvider; +import io.druid.metadata.MetadataSegmentManager; +import io.druid.metadata.MetadataSegmentManagerConfig; +import io.druid.metadata.MetadataSegmentManagerProvider; +import io.druid.metadata.MetadataStorage; import io.druid.server.coordinator.DruidCoordinator; import io.druid.server.coordinator.DruidCoordinatorConfig; import io.druid.server.coordinator.LoadQueueTaskMaster; @@ -49,9 +50,9 @@ import io.druid.server.http.BackwardsCompatibleInfoResource; import io.druid.server.http.CoordinatorDynamicConfigsResource; import io.druid.server.http.CoordinatorRedirectInfo; import io.druid.server.http.CoordinatorResource; -import io.druid.server.http.MetadataResource; import io.druid.server.http.DatasourcesResource; import io.druid.server.http.InfoResource; +import io.druid.server.http.MetadataResource; import io.druid.server.http.RedirectFilter; import io.druid.server.http.RedirectInfo; import io.druid.server.http.RulesResource; @@ -93,6 +94,8 @@ public class CliCoordinator extends ServerRunnable ConfigProvider.bind(binder, DruidCoordinatorConfig.class); + binder.bind(MetadataStorage.class).in(ManageLifecycle.class); + JsonConfigProvider.bind(binder, "druid.manager.segments", MetadataSegmentManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class); From 8cbd21af902697d2811479de19a52479ce72cb23 Mon Sep 17 00:00:00 2001 From: fjy Date: Mon, 10 Nov 2014 15:40:57 -0800 Subject: [PATCH 2/3] fix defaults --- .../MetadataStorageConnectorConfig.java | 22 ++++++++++++++++++- .../druid/metadata/DerbyMetadataStorage.java | 6 +++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java b/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java index 329f0de1d26..c4169b1c20d 100644 --- a/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java +++ b/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java @@ -29,7 +29,13 @@ public class MetadataStorageConnectorConfig private boolean createTables = true; @JsonProperty - private String connectURI = "jdbc:derby://localhost:1527/druid;create=true"; + private String host = "localhost"; + + @JsonProperty + private int port = 1527; + + @JsonProperty + private String connectURI; @JsonProperty private String user = null; @@ -45,8 +51,22 @@ public class MetadataStorageConnectorConfig return createTables; } + public String getHost() + { + return host; + } + + public int getPort() + { + return port; + } + public String getConnectURI() { + if (connectURI == null) { + return String.format("jdbc:derby://%s:%s/druid;create=true", host, port); + } + return connectURI; } diff --git a/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java b/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java index 647b1ffb791..c4de142d5d3 100644 --- a/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java +++ b/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java @@ -20,6 +20,7 @@ package io.druid.metadata; import com.google.common.base.Throwables; +import com.google.inject.Inject; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import org.apache.derby.drda.NetworkServerControl; @@ -32,10 +33,11 @@ public class DerbyMetadataStorage extends MetadataStorage { private final NetworkServerControl server; - public DerbyMetadataStorage() + @Inject + public DerbyMetadataStorage(MetadataStorageConnectorConfig config) { try { - this.server = new NetworkServerControl(InetAddress.getByName("localhost"), 1527); + this.server = new NetworkServerControl(InetAddress.getByName(config.getHost()), config.getPort()); } catch (Exception e) { throw Throwables.propagate(e); From 59f093882d72c72fb6a89eb29c886666baf0189b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 10 Nov 2014 16:55:53 -0800 Subject: [PATCH 3/3] fix Derby instantiation --- .../MetadataStorageConnectorConfig.java | 12 +----- .../storage/mysql/MySQLConnector.java | 2 +- .../postgresql/PostgreSQLConnector.java | 2 +- .../guice/SQLMetadataStorageDruidModule.java | 9 +++- .../druid/initialization/Initialization.java | 2 +- .../io/druid/metadata/MetadataStorage.java | 3 +- .../metadata/MetadataStorageProvider.java | 26 ++++++++++++ ....java => NoopMetadataStorageProvider.java} | 30 +++---------- .../druid/metadata/SQLMetadataConnector.java | 6 ++- .../{ => storage/derby}/DerbyConnector.java | 10 ++++- .../derby}/DerbyMetadataStorage.java | 13 ++++-- .../DerbyMetadataStorageDruidModule.java | 13 +++--- .../derby/DerbyMetadataStorageProvider.java | 42 +++++++++++++++++++ .../io/druid/metadata/TestDerbyConnector.java | 1 + .../java/io/druid/cli/CliCoordinator.java | 6 ++- 15 files changed, 121 insertions(+), 56 deletions(-) create mode 100644 server/src/main/java/io/druid/metadata/MetadataStorageProvider.java rename server/src/main/java/io/druid/metadata/{DerbyConnectionFactory.java => NoopMetadataStorageProvider.java} (50%) rename server/src/main/java/io/druid/metadata/{ => storage/derby}/DerbyConnector.java (86%) rename server/src/main/java/io/druid/metadata/{ => storage/derby}/DerbyMetadataStorage.java (83%) rename server/src/main/java/io/druid/{guice => metadata/storage/derby}/DerbyMetadataStorageDruidModule.java (83%) create mode 100644 server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageProvider.java diff --git a/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java b/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java index c4169b1c20d..048cc0e99e3 100644 --- a/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java +++ b/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java @@ -43,9 +43,6 @@ public class MetadataStorageConnectorConfig @JsonProperty private String password = null; - @JsonProperty - private String validationQuery = "SELECT 1"; - public boolean isCreateTables() { return createTables; @@ -66,7 +63,6 @@ public class MetadataStorageConnectorConfig if (connectURI == null) { return String.format("jdbc:derby://%s:%s/druid;create=true", host, port); } - return connectURI; } @@ -80,20 +76,14 @@ public class MetadataStorageConnectorConfig return password; } - public String getValidationQuery() - { - return validationQuery; - } - @Override public String toString() { return "DbConnectorConfig{" + "createTables=" + createTables + - ", connectURI='" + connectURI + '\'' + + ", connectURI='" + getConnectURI() + '\'' + ", user='" + user + '\'' + ", password=****" + - ", validationQuery='" + validationQuery + '\'' + '}'; } } diff --git a/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java b/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java index c27ace5a6cf..39b2b6ed0c8 100644 --- a/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java +++ b/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java @@ -72,7 +72,7 @@ public class MySQLConnector extends SQLMetadataConnector } @Override - protected boolean tableExists(Handle handle, String tableName) + public boolean tableExists(Handle handle, String tableName) { return !handle.createQuery("SHOW tables LIKE :tableName") .bind("tableName", tableName) diff --git a/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java b/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java index 13dbbab324a..b2114da2093 100644 --- a/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java +++ b/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java @@ -65,7 +65,7 @@ public class PostgreSQLConnector extends SQLMetadataConnector } @Override - protected boolean tableExists(final Handle handle, final String tableName) + public boolean tableExists(final Handle handle, final String tableName) { return !handle.createQuery( "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = 'public' AND tablename LIKE :tableName" diff --git a/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java index 96468ac075e..fef1fb04d95 100644 --- a/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java +++ b/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java @@ -25,7 +25,6 @@ import com.google.inject.Module; import io.druid.indexer.MetadataStorageUpdaterJobHandler; import io.druid.indexer.SQLMetadataStorageUpdaterJobHandler; import io.druid.indexing.overlord.IndexerMetadataStorageCoordinator; -import io.druid.metadata.MetadataStorage; import io.druid.metadata.MetadataStorageActionHandlerFactory; import io.druid.metadata.IndexerSQLMetadataStorageCoordinator; import io.druid.metadata.MetadataRuleManager; @@ -35,6 +34,8 @@ import io.druid.metadata.MetadataSegmentManagerProvider; import io.druid.metadata.MetadataSegmentPublisher; import io.druid.metadata.MetadataSegmentPublisherProvider; import io.druid.metadata.MetadataStorageConnector; +import io.druid.metadata.MetadataStorageProvider; +import io.druid.metadata.NoopMetadataStorageProvider; import io.druid.metadata.SQLMetadataConnector; import io.druid.metadata.SQLMetadataRuleManager; import io.druid.metadata.SQLMetadataRuleManagerProvider; @@ -64,7 +65,11 @@ public class SQLMetadataStorageDruidModule implements Module binder, PROPERTY, Key.get(MetadataStorageConnector.class), null, defaultPropertyValue ); PolyBind.createChoiceWithDefault( - binder, PROPERTY, Key.get(MetadataStorage.class), Key.get(MetadataStorage.class), defaultPropertyValue + binder, + PROPERTY, + Key.get(MetadataStorageProvider.class), + Key.get(NoopMetadataStorageProvider.class), + defaultPropertyValue ); PolyBind.createChoiceWithDefault( binder, PROPERTY, Key.get(SQLMetadataConnector.class), null, defaultPropertyValue diff --git a/server/src/main/java/io/druid/initialization/Initialization.java b/server/src/main/java/io/druid/initialization/Initialization.java index 7bd7fc6e1fb..76786946cde 100644 --- a/server/src/main/java/io/druid/initialization/Initialization.java +++ b/server/src/main/java/io/druid/initialization/Initialization.java @@ -36,7 +36,7 @@ import io.druid.curator.CuratorModule; import io.druid.curator.discovery.DiscoveryModule; import io.druid.guice.AWSModule; import io.druid.guice.AnnouncerModule; -import io.druid.guice.DerbyMetadataStorageDruidModule; +import io.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule; import io.druid.guice.DruidProcessingModule; import io.druid.guice.DruidSecondaryModule; import io.druid.guice.ExtensionsConfig; diff --git a/server/src/main/java/io/druid/metadata/MetadataStorage.java b/server/src/main/java/io/druid/metadata/MetadataStorage.java index 2bdd3378385..6a53ac251bf 100644 --- a/server/src/main/java/io/druid/metadata/MetadataStorage.java +++ b/server/src/main/java/io/druid/metadata/MetadataStorage.java @@ -2,9 +2,8 @@ package io.druid.metadata; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; +import io.druid.guice.ManageLifecycle; -/** - */ public abstract class MetadataStorage { @LifecycleStart diff --git a/server/src/main/java/io/druid/metadata/MetadataStorageProvider.java b/server/src/main/java/io/druid/metadata/MetadataStorageProvider.java new file mode 100644 index 00000000000..8c441954ffb --- /dev/null +++ b/server/src/main/java/io/druid/metadata/MetadataStorageProvider.java @@ -0,0 +1,26 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.metadata; + +import com.google.inject.Provider; + +public interface MetadataStorageProvider extends Provider +{ +} diff --git a/server/src/main/java/io/druid/metadata/DerbyConnectionFactory.java b/server/src/main/java/io/druid/metadata/NoopMetadataStorageProvider.java similarity index 50% rename from server/src/main/java/io/druid/metadata/DerbyConnectionFactory.java rename to server/src/main/java/io/druid/metadata/NoopMetadataStorageProvider.java index 75f10f6f067..278451a36c9 100644 --- a/server/src/main/java/io/druid/metadata/DerbyConnectionFactory.java +++ b/server/src/main/java/io/druid/metadata/NoopMetadataStorageProvider.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * Copyright (C) 2014 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -19,29 +19,11 @@ package io.druid.metadata; -import com.google.api.client.repackaged.com.google.common.base.Throwables; -import org.skife.jdbi.v2.tweak.ConnectionFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -public class DerbyConnectionFactory implements ConnectionFactory +public class NoopMetadataStorageProvider implements MetadataStorageProvider { - final private String dbName; - - public DerbyConnectionFactory(String dbName) { - this.dbName = dbName; - } - - public Connection openConnection() throws SQLException { - final String nsURL=String.format("jdbc:derby://localhost:1527/%s;create=true", dbName); - try { - Class.forName("org.apache.derby.jdbc.ClientDriver"); - } catch (Exception e) { - throw Throwables.propagate(e); - } - - return DriverManager.getConnection(nsURL); + @Override + public MetadataStorage get() + { + return new MetadataStorage() {}; } } diff --git a/server/src/main/java/io/druid/metadata/SQLMetadataConnector.java b/server/src/main/java/io/druid/metadata/SQLMetadataConnector.java index 561ed963b3e..af1274304d8 100644 --- a/server/src/main/java/io/druid/metadata/SQLMetadataConnector.java +++ b/server/src/main/java/io/druid/metadata/SQLMetadataConnector.java @@ -75,7 +75,9 @@ public abstract class SQLMetadataConnector implements MetadataStorageConnector */ protected abstract String getSerialType(); - protected abstract boolean tableExists(Handle handle, final String tableName); + public String getValidationQuery() { return "SELECT 1"; } + + public abstract boolean tableExists(Handle handle, final String tableName); protected boolean isTransientException(Throwable e) { return false; @@ -367,7 +369,7 @@ public abstract class SQLMetadataConnector implements MetadataStorageConnector String uri = connectorConfig.getConnectURI(); dataSource.setUrl(uri); - dataSource.setValidationQuery(connectorConfig.getValidationQuery()); + dataSource.setValidationQuery(getValidationQuery()); dataSource.setTestOnBorrow(true); return dataSource; diff --git a/server/src/main/java/io/druid/metadata/DerbyConnector.java b/server/src/main/java/io/druid/metadata/storage/derby/DerbyConnector.java similarity index 86% rename from server/src/main/java/io/druid/metadata/DerbyConnector.java rename to server/src/main/java/io/druid/metadata/storage/derby/DerbyConnector.java index ad57c55be00..e15c21472c9 100644 --- a/server/src/main/java/io/druid/metadata/DerbyConnector.java +++ b/server/src/main/java/io/druid/metadata/storage/derby/DerbyConnector.java @@ -17,10 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package io.druid.metadata; +package io.druid.metadata.storage.derby; import com.google.common.base.Supplier; import com.google.inject.Inject; +import io.druid.metadata.MetadataStorageConnectorConfig; +import io.druid.metadata.MetadataStorageTablesConfig; +import io.druid.metadata.SQLMetadataConnector; import org.apache.commons.dbcp2.BasicDataSource; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; @@ -53,7 +56,7 @@ public class DerbyConnector extends SQLMetadataConnector } @Override - protected boolean tableExists(Handle handle, String tableName) + public boolean tableExists(Handle handle, String tableName) { return !handle.createQuery("select * from SYS.SYSTABLES where tablename = :tableName") .bind("tableName", tableName.toUpperCase()) @@ -69,4 +72,7 @@ public class DerbyConnector extends SQLMetadataConnector @Override public DBI getDBI() { return dbi; } + + @Override + public String getValidationQuery() { return "VALUES 1"; } } diff --git a/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorage.java similarity index 83% rename from server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java rename to server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorage.java index c4de142d5d3..eea0757f3a3 100644 --- a/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java +++ b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorage.java @@ -17,20 +17,26 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package io.druid.metadata; +package io.druid.metadata.storage.derby; import com.google.common.base.Throwables; import com.google.inject.Inject; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; +import com.metamx.common.logger.Logger; +import io.druid.guice.ManageLifecycle; +import io.druid.metadata.MetadataStorage; +import io.druid.metadata.MetadataStorageConnectorConfig; import org.apache.derby.drda.NetworkServerControl; import java.net.InetAddress; -/** - */ + +@ManageLifecycle public class DerbyMetadataStorage extends MetadataStorage { + private static final Logger log = new Logger(DerbyMetadataStorage.class); + private final NetworkServerControl server; @Inject @@ -50,6 +56,7 @@ public class DerbyMetadataStorage extends MetadataStorage public void start() { try { + log.info("Starting Derby Metadata Storage"); server.start(null); } catch (Exception e) { diff --git a/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java similarity index 83% rename from server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java rename to server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java index c698da8f0ed..d95e822d7b3 100644 --- a/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java +++ b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java @@ -17,14 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package io.druid.guice; +package io.druid.metadata.storage.derby; import com.google.inject.Binder; import com.google.inject.Key; -import io.druid.metadata.DerbyConnector; -import io.druid.metadata.DerbyMetadataStorage; -import io.druid.metadata.MetadataStorage; +import io.druid.guice.LazySingleton; +import io.druid.guice.PolyBind; +import io.druid.guice.SQLMetadataStorageDruidModule; import io.druid.metadata.MetadataStorageConnector; +import io.druid.metadata.MetadataStorageProvider; import io.druid.metadata.SQLMetadataConnector; public class DerbyMetadataStorageDruidModule extends SQLMetadataStorageDruidModule @@ -42,9 +43,9 @@ public class DerbyMetadataStorageDruidModule extends SQLMetadataStorageDruidModu createBindingChoices(binder, TYPE); super.configure(binder); - PolyBind.optionBinder(binder, Key.get(MetadataStorage.class)) + PolyBind.optionBinder(binder, Key.get(MetadataStorageProvider.class)) .addBinding(TYPE) - .to(DerbyMetadataStorage.class) + .to(DerbyMetadataStorageProvider.class) .in(LazySingleton.class); PolyBind.optionBinder(binder, Key.get(MetadataStorageConnector.class)) diff --git a/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageProvider.java b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageProvider.java new file mode 100644 index 00000000000..11daddb6065 --- /dev/null +++ b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageProvider.java @@ -0,0 +1,42 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.metadata.storage.derby; + +import com.google.inject.Inject; +import io.druid.metadata.MetadataStorage; +import io.druid.metadata.MetadataStorageConnectorConfig; +import io.druid.metadata.MetadataStorageProvider; + +public class DerbyMetadataStorageProvider implements MetadataStorageProvider +{ + private final DerbyMetadataStorage storage; + + @Inject + public DerbyMetadataStorageProvider(MetadataStorageConnectorConfig config) + { + this.storage = new DerbyMetadataStorage(config); + } + + @Override + public MetadataStorage get() + { + return storage; + } +} diff --git a/server/src/test/java/io/druid/metadata/TestDerbyConnector.java b/server/src/test/java/io/druid/metadata/TestDerbyConnector.java index b45eb0c5070..8b301f331cb 100644 --- a/server/src/test/java/io/druid/metadata/TestDerbyConnector.java +++ b/server/src/test/java/io/druid/metadata/TestDerbyConnector.java @@ -20,6 +20,7 @@ package io.druid.metadata; import com.google.common.base.Supplier; +import io.druid.metadata.storage.derby.DerbyConnector; import org.junit.Assert; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.exceptions.UnableToObtainConnectionException; diff --git a/services/src/main/java/io/druid/cli/CliCoordinator.java b/services/src/main/java/io/druid/cli/CliCoordinator.java index 9470b4c1542..655292158ad 100644 --- a/services/src/main/java/io/druid/cli/CliCoordinator.java +++ b/services/src/main/java/io/druid/cli/CliCoordinator.java @@ -42,6 +42,7 @@ import io.druid.metadata.MetadataSegmentManager; import io.druid.metadata.MetadataSegmentManagerConfig; import io.druid.metadata.MetadataSegmentManagerProvider; import io.druid.metadata.MetadataStorage; +import io.druid.metadata.MetadataStorageProvider; import io.druid.server.coordinator.DruidCoordinator; import io.druid.server.coordinator.DruidCoordinatorConfig; import io.druid.server.coordinator.LoadQueueTaskMaster; @@ -94,7 +95,9 @@ public class CliCoordinator extends ServerRunnable ConfigProvider.bind(binder, DruidCoordinatorConfig.class); - binder.bind(MetadataStorage.class).in(ManageLifecycle.class); + binder.bind(MetadataStorage.class) + .toProvider(MetadataStorageProvider.class) + .in(ManageLifecycle.class); JsonConfigProvider.bind(binder, "druid.manager.segments", MetadataSegmentManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class); @@ -114,6 +117,7 @@ public class CliCoordinator extends ServerRunnable binder.bind(DruidCoordinator.class); + LifecycleModule.register(binder, MetadataStorage.class); LifecycleModule.register(binder, DruidCoordinator.class); binder.bind(JettyServerInitializer.class)