From 59f093882d72c72fb6a89eb29c886666baf0189b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 10 Nov 2014 16:55:53 -0800 Subject: [PATCH] fix Derby instantiation --- .../MetadataStorageConnectorConfig.java | 12 +----- .../storage/mysql/MySQLConnector.java | 2 +- .../postgresql/PostgreSQLConnector.java | 2 +- .../guice/SQLMetadataStorageDruidModule.java | 9 +++- .../druid/initialization/Initialization.java | 2 +- .../io/druid/metadata/MetadataStorage.java | 3 +- .../metadata/MetadataStorageProvider.java | 26 ++++++++++++ ....java => NoopMetadataStorageProvider.java} | 30 +++---------- .../druid/metadata/SQLMetadataConnector.java | 6 ++- .../{ => storage/derby}/DerbyConnector.java | 10 ++++- .../derby}/DerbyMetadataStorage.java | 13 ++++-- .../DerbyMetadataStorageDruidModule.java | 13 +++--- .../derby/DerbyMetadataStorageProvider.java | 42 +++++++++++++++++++ .../io/druid/metadata/TestDerbyConnector.java | 1 + .../java/io/druid/cli/CliCoordinator.java | 6 ++- 15 files changed, 121 insertions(+), 56 deletions(-) create mode 100644 server/src/main/java/io/druid/metadata/MetadataStorageProvider.java rename server/src/main/java/io/druid/metadata/{DerbyConnectionFactory.java => NoopMetadataStorageProvider.java} (50%) rename server/src/main/java/io/druid/metadata/{ => storage/derby}/DerbyConnector.java (86%) rename server/src/main/java/io/druid/metadata/{ => storage/derby}/DerbyMetadataStorage.java (83%) rename server/src/main/java/io/druid/{guice => metadata/storage/derby}/DerbyMetadataStorageDruidModule.java (83%) create mode 100644 server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageProvider.java diff --git a/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java b/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java index c4169b1c20d..048cc0e99e3 100644 --- a/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java +++ b/common/src/main/java/io/druid/metadata/MetadataStorageConnectorConfig.java @@ -43,9 +43,6 @@ public class MetadataStorageConnectorConfig @JsonProperty private String password = null; - @JsonProperty - private String validationQuery = "SELECT 1"; - public boolean isCreateTables() { return createTables; @@ -66,7 +63,6 @@ public class MetadataStorageConnectorConfig if (connectURI == null) { return String.format("jdbc:derby://%s:%s/druid;create=true", host, port); } - return connectURI; } @@ -80,20 +76,14 @@ public class MetadataStorageConnectorConfig return password; } - public String getValidationQuery() - { - return validationQuery; - } - @Override public String toString() { return "DbConnectorConfig{" + "createTables=" + createTables + - ", connectURI='" + connectURI + '\'' + + ", connectURI='" + getConnectURI() + '\'' + ", user='" + user + '\'' + ", password=****" + - ", validationQuery='" + validationQuery + '\'' + '}'; } } diff --git a/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java b/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java index c27ace5a6cf..39b2b6ed0c8 100644 --- a/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java +++ b/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java @@ -72,7 +72,7 @@ public class MySQLConnector extends SQLMetadataConnector } @Override - protected boolean tableExists(Handle handle, String tableName) + public boolean tableExists(Handle handle, String tableName) { return !handle.createQuery("SHOW tables LIKE :tableName") .bind("tableName", tableName) diff --git a/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java b/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java index 13dbbab324a..b2114da2093 100644 --- a/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java +++ b/postgresql-metadata-storage/src/main/java/io/druid/metadata/storage/postgresql/PostgreSQLConnector.java @@ -65,7 +65,7 @@ public class PostgreSQLConnector extends SQLMetadataConnector } @Override - protected boolean tableExists(final Handle handle, final String tableName) + public boolean tableExists(final Handle handle, final String tableName) { return !handle.createQuery( "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = 'public' AND tablename LIKE :tableName" diff --git a/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java index 96468ac075e..fef1fb04d95 100644 --- a/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java +++ b/server/src/main/java/io/druid/guice/SQLMetadataStorageDruidModule.java @@ -25,7 +25,6 @@ import com.google.inject.Module; import io.druid.indexer.MetadataStorageUpdaterJobHandler; import io.druid.indexer.SQLMetadataStorageUpdaterJobHandler; import io.druid.indexing.overlord.IndexerMetadataStorageCoordinator; -import io.druid.metadata.MetadataStorage; import io.druid.metadata.MetadataStorageActionHandlerFactory; import io.druid.metadata.IndexerSQLMetadataStorageCoordinator; import io.druid.metadata.MetadataRuleManager; @@ -35,6 +34,8 @@ import io.druid.metadata.MetadataSegmentManagerProvider; import io.druid.metadata.MetadataSegmentPublisher; import io.druid.metadata.MetadataSegmentPublisherProvider; import io.druid.metadata.MetadataStorageConnector; +import io.druid.metadata.MetadataStorageProvider; +import io.druid.metadata.NoopMetadataStorageProvider; import io.druid.metadata.SQLMetadataConnector; import io.druid.metadata.SQLMetadataRuleManager; import io.druid.metadata.SQLMetadataRuleManagerProvider; @@ -64,7 +65,11 @@ public class SQLMetadataStorageDruidModule implements Module binder, PROPERTY, Key.get(MetadataStorageConnector.class), null, defaultPropertyValue ); PolyBind.createChoiceWithDefault( - binder, PROPERTY, Key.get(MetadataStorage.class), Key.get(MetadataStorage.class), defaultPropertyValue + binder, + PROPERTY, + Key.get(MetadataStorageProvider.class), + Key.get(NoopMetadataStorageProvider.class), + defaultPropertyValue ); PolyBind.createChoiceWithDefault( binder, PROPERTY, Key.get(SQLMetadataConnector.class), null, defaultPropertyValue diff --git a/server/src/main/java/io/druid/initialization/Initialization.java b/server/src/main/java/io/druid/initialization/Initialization.java index 7bd7fc6e1fb..76786946cde 100644 --- a/server/src/main/java/io/druid/initialization/Initialization.java +++ b/server/src/main/java/io/druid/initialization/Initialization.java @@ -36,7 +36,7 @@ import io.druid.curator.CuratorModule; import io.druid.curator.discovery.DiscoveryModule; import io.druid.guice.AWSModule; import io.druid.guice.AnnouncerModule; -import io.druid.guice.DerbyMetadataStorageDruidModule; +import io.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule; import io.druid.guice.DruidProcessingModule; import io.druid.guice.DruidSecondaryModule; import io.druid.guice.ExtensionsConfig; diff --git a/server/src/main/java/io/druid/metadata/MetadataStorage.java b/server/src/main/java/io/druid/metadata/MetadataStorage.java index 2bdd3378385..6a53ac251bf 100644 --- a/server/src/main/java/io/druid/metadata/MetadataStorage.java +++ b/server/src/main/java/io/druid/metadata/MetadataStorage.java @@ -2,9 +2,8 @@ package io.druid.metadata; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; +import io.druid.guice.ManageLifecycle; -/** - */ public abstract class MetadataStorage { @LifecycleStart diff --git a/server/src/main/java/io/druid/metadata/MetadataStorageProvider.java b/server/src/main/java/io/druid/metadata/MetadataStorageProvider.java new file mode 100644 index 00000000000..8c441954ffb --- /dev/null +++ b/server/src/main/java/io/druid/metadata/MetadataStorageProvider.java @@ -0,0 +1,26 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.metadata; + +import com.google.inject.Provider; + +public interface MetadataStorageProvider extends Provider +{ +} diff --git a/server/src/main/java/io/druid/metadata/DerbyConnectionFactory.java b/server/src/main/java/io/druid/metadata/NoopMetadataStorageProvider.java similarity index 50% rename from server/src/main/java/io/druid/metadata/DerbyConnectionFactory.java rename to server/src/main/java/io/druid/metadata/NoopMetadataStorageProvider.java index 75f10f6f067..278451a36c9 100644 --- a/server/src/main/java/io/druid/metadata/DerbyConnectionFactory.java +++ b/server/src/main/java/io/druid/metadata/NoopMetadataStorageProvider.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * Copyright (C) 2014 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -19,29 +19,11 @@ package io.druid.metadata; -import com.google.api.client.repackaged.com.google.common.base.Throwables; -import org.skife.jdbi.v2.tweak.ConnectionFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -public class DerbyConnectionFactory implements ConnectionFactory +public class NoopMetadataStorageProvider implements MetadataStorageProvider { - final private String dbName; - - public DerbyConnectionFactory(String dbName) { - this.dbName = dbName; - } - - public Connection openConnection() throws SQLException { - final String nsURL=String.format("jdbc:derby://localhost:1527/%s;create=true", dbName); - try { - Class.forName("org.apache.derby.jdbc.ClientDriver"); - } catch (Exception e) { - throw Throwables.propagate(e); - } - - return DriverManager.getConnection(nsURL); + @Override + public MetadataStorage get() + { + return new MetadataStorage() {}; } } diff --git a/server/src/main/java/io/druid/metadata/SQLMetadataConnector.java b/server/src/main/java/io/druid/metadata/SQLMetadataConnector.java index 561ed963b3e..af1274304d8 100644 --- a/server/src/main/java/io/druid/metadata/SQLMetadataConnector.java +++ b/server/src/main/java/io/druid/metadata/SQLMetadataConnector.java @@ -75,7 +75,9 @@ public abstract class SQLMetadataConnector implements MetadataStorageConnector */ protected abstract String getSerialType(); - protected abstract boolean tableExists(Handle handle, final String tableName); + public String getValidationQuery() { return "SELECT 1"; } + + public abstract boolean tableExists(Handle handle, final String tableName); protected boolean isTransientException(Throwable e) { return false; @@ -367,7 +369,7 @@ public abstract class SQLMetadataConnector implements MetadataStorageConnector String uri = connectorConfig.getConnectURI(); dataSource.setUrl(uri); - dataSource.setValidationQuery(connectorConfig.getValidationQuery()); + dataSource.setValidationQuery(getValidationQuery()); dataSource.setTestOnBorrow(true); return dataSource; diff --git a/server/src/main/java/io/druid/metadata/DerbyConnector.java b/server/src/main/java/io/druid/metadata/storage/derby/DerbyConnector.java similarity index 86% rename from server/src/main/java/io/druid/metadata/DerbyConnector.java rename to server/src/main/java/io/druid/metadata/storage/derby/DerbyConnector.java index ad57c55be00..e15c21472c9 100644 --- a/server/src/main/java/io/druid/metadata/DerbyConnector.java +++ b/server/src/main/java/io/druid/metadata/storage/derby/DerbyConnector.java @@ -17,10 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package io.druid.metadata; +package io.druid.metadata.storage.derby; import com.google.common.base.Supplier; import com.google.inject.Inject; +import io.druid.metadata.MetadataStorageConnectorConfig; +import io.druid.metadata.MetadataStorageTablesConfig; +import io.druid.metadata.SQLMetadataConnector; import org.apache.commons.dbcp2.BasicDataSource; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; @@ -53,7 +56,7 @@ public class DerbyConnector extends SQLMetadataConnector } @Override - protected boolean tableExists(Handle handle, String tableName) + public boolean tableExists(Handle handle, String tableName) { return !handle.createQuery("select * from SYS.SYSTABLES where tablename = :tableName") .bind("tableName", tableName.toUpperCase()) @@ -69,4 +72,7 @@ public class DerbyConnector extends SQLMetadataConnector @Override public DBI getDBI() { return dbi; } + + @Override + public String getValidationQuery() { return "VALUES 1"; } } diff --git a/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorage.java similarity index 83% rename from server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java rename to server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorage.java index c4de142d5d3..eea0757f3a3 100644 --- a/server/src/main/java/io/druid/metadata/DerbyMetadataStorage.java +++ b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorage.java @@ -17,20 +17,26 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package io.druid.metadata; +package io.druid.metadata.storage.derby; import com.google.common.base.Throwables; import com.google.inject.Inject; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; +import com.metamx.common.logger.Logger; +import io.druid.guice.ManageLifecycle; +import io.druid.metadata.MetadataStorage; +import io.druid.metadata.MetadataStorageConnectorConfig; import org.apache.derby.drda.NetworkServerControl; import java.net.InetAddress; -/** - */ + +@ManageLifecycle public class DerbyMetadataStorage extends MetadataStorage { + private static final Logger log = new Logger(DerbyMetadataStorage.class); + private final NetworkServerControl server; @Inject @@ -50,6 +56,7 @@ public class DerbyMetadataStorage extends MetadataStorage public void start() { try { + log.info("Starting Derby Metadata Storage"); server.start(null); } catch (Exception e) { diff --git a/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java similarity index 83% rename from server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java rename to server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java index c698da8f0ed..d95e822d7b3 100644 --- a/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java +++ b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java @@ -17,14 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package io.druid.guice; +package io.druid.metadata.storage.derby; import com.google.inject.Binder; import com.google.inject.Key; -import io.druid.metadata.DerbyConnector; -import io.druid.metadata.DerbyMetadataStorage; -import io.druid.metadata.MetadataStorage; +import io.druid.guice.LazySingleton; +import io.druid.guice.PolyBind; +import io.druid.guice.SQLMetadataStorageDruidModule; import io.druid.metadata.MetadataStorageConnector; +import io.druid.metadata.MetadataStorageProvider; import io.druid.metadata.SQLMetadataConnector; public class DerbyMetadataStorageDruidModule extends SQLMetadataStorageDruidModule @@ -42,9 +43,9 @@ public class DerbyMetadataStorageDruidModule extends SQLMetadataStorageDruidModu createBindingChoices(binder, TYPE); super.configure(binder); - PolyBind.optionBinder(binder, Key.get(MetadataStorage.class)) + PolyBind.optionBinder(binder, Key.get(MetadataStorageProvider.class)) .addBinding(TYPE) - .to(DerbyMetadataStorage.class) + .to(DerbyMetadataStorageProvider.class) .in(LazySingleton.class); PolyBind.optionBinder(binder, Key.get(MetadataStorageConnector.class)) diff --git a/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageProvider.java b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageProvider.java new file mode 100644 index 00000000000..11daddb6065 --- /dev/null +++ b/server/src/main/java/io/druid/metadata/storage/derby/DerbyMetadataStorageProvider.java @@ -0,0 +1,42 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.metadata.storage.derby; + +import com.google.inject.Inject; +import io.druid.metadata.MetadataStorage; +import io.druid.metadata.MetadataStorageConnectorConfig; +import io.druid.metadata.MetadataStorageProvider; + +public class DerbyMetadataStorageProvider implements MetadataStorageProvider +{ + private final DerbyMetadataStorage storage; + + @Inject + public DerbyMetadataStorageProvider(MetadataStorageConnectorConfig config) + { + this.storage = new DerbyMetadataStorage(config); + } + + @Override + public MetadataStorage get() + { + return storage; + } +} diff --git a/server/src/test/java/io/druid/metadata/TestDerbyConnector.java b/server/src/test/java/io/druid/metadata/TestDerbyConnector.java index b45eb0c5070..8b301f331cb 100644 --- a/server/src/test/java/io/druid/metadata/TestDerbyConnector.java +++ b/server/src/test/java/io/druid/metadata/TestDerbyConnector.java @@ -20,6 +20,7 @@ package io.druid.metadata; import com.google.common.base.Supplier; +import io.druid.metadata.storage.derby.DerbyConnector; import org.junit.Assert; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.exceptions.UnableToObtainConnectionException; diff --git a/services/src/main/java/io/druid/cli/CliCoordinator.java b/services/src/main/java/io/druid/cli/CliCoordinator.java index 9470b4c1542..655292158ad 100644 --- a/services/src/main/java/io/druid/cli/CliCoordinator.java +++ b/services/src/main/java/io/druid/cli/CliCoordinator.java @@ -42,6 +42,7 @@ import io.druid.metadata.MetadataSegmentManager; import io.druid.metadata.MetadataSegmentManagerConfig; import io.druid.metadata.MetadataSegmentManagerProvider; import io.druid.metadata.MetadataStorage; +import io.druid.metadata.MetadataStorageProvider; import io.druid.server.coordinator.DruidCoordinator; import io.druid.server.coordinator.DruidCoordinatorConfig; import io.druid.server.coordinator.LoadQueueTaskMaster; @@ -94,7 +95,9 @@ public class CliCoordinator extends ServerRunnable ConfigProvider.bind(binder, DruidCoordinatorConfig.class); - binder.bind(MetadataStorage.class).in(ManageLifecycle.class); + binder.bind(MetadataStorage.class) + .toProvider(MetadataStorageProvider.class) + .in(ManageLifecycle.class); JsonConfigProvider.bind(binder, "druid.manager.segments", MetadataSegmentManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class); @@ -114,6 +117,7 @@ public class CliCoordinator extends ServerRunnable binder.bind(DruidCoordinator.class); + LifecycleModule.register(binder, MetadataStorage.class); LifecycleModule.register(binder, DruidCoordinator.class); binder.bind(JettyServerInitializer.class)