diff --git a/common/pom.xml b/common/pom.xml index 03e902df7c5..d54a3486cbf 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -46,9 +46,8 @@ commons-codec - commons-dbcp - commons-dbcp - 1.4 + org.apache.commons + commons-dbcp2 commons-pool diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 81260ba3295..f2b5b0ebae7 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -47,11 +47,6 @@ druid-indexing-hadoop ${project.parent.version} - - mysql - mysql-connector-java - 5.1.18 - diff --git a/mysql-storage/pom.xml b/mysql-storage/pom.xml index 3349acd3af8..d54b899a74f 100644 --- a/mysql-storage/pom.xml +++ b/mysql-storage/pom.xml @@ -49,16 +49,6 @@ druid-server ${project.parent.version} - - io.druid - druid-indexing-hadoop - ${project.parent.version} - - - io.druid - druid-indexing-service - ${project.parent.version} - mysql mysql-connector-java @@ -85,4 +75,4 @@ - \ No newline at end of file + diff --git a/mysql-storage/src/main/java/io/druid/storage/mysql/MySQLConnector.java b/mysql-storage/src/main/java/io/druid/storage/mysql/MySQLConnector.java index 2bebe0ced36..2e8c5cadb2e 100644 --- a/mysql-storage/src/main/java/io/druid/storage/mysql/MySQLConnector.java +++ b/mysql-storage/src/main/java/io/druid/storage/mysql/MySQLConnector.java @@ -22,16 +22,16 @@ package io.druid.storage.mysql; import com.google.common.base.Supplier; import com.google.inject.Inject; import com.metamx.common.logger.Logger; +import com.mysql.jdbc.exceptions.MySQLTransientException; import io.druid.db.MetadataStorageConnectorConfig; import io.druid.db.MetadataStorageTablesConfig; import io.druid.db.SQLMetadataConnector; +import org.apache.commons.dbcp2.BasicDataSource; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; -import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.tweak.HandleCallback; -import java.util.List; -import java.util.Map; +import java.sql.SQLException; public class MySQLConnector extends SQLMetadataConnector { @@ -45,7 +45,15 @@ public class MySQLConnector extends SQLMetadataConnector public MySQLConnector(Supplier config, Supplier dbTables) { super(config, dbTables); - this.dbi = new DBI(getDatasource()); + + final BasicDataSource datasource = getDatasource(); + // MySQL driver is classloader isolated as part of the extension + // so we need to help JDBC find the driver + datasource.setDriverClassLoader(getClass().getClassLoader()); + datasource.setDriverClassName("com.mysql.jdbc.Driver"); + + this.dbi = new DBI(datasource); + dbi.withHandle(new HandleCallback() { @Override @@ -78,6 +86,14 @@ public class MySQLConnector extends SQLMetadataConnector .isEmpty(); } + @Override + protected boolean isTransientException(Throwable e) + { + return e instanceof MySQLTransientException + || (e instanceof SQLException && ((SQLException) e).getErrorCode() == 1317) + ; + } + @Override public Void insertOrUpdate( final String tableName, diff --git a/mysql-storage/src/main/java/io/druid/storage/mysql/MySQLMetadataStorageModule.java b/mysql-storage/src/main/java/io/druid/storage/mysql/MySQLMetadataStorageModule.java index bc733108250..b72d5ca2cff 100644 --- a/mysql-storage/src/main/java/io/druid/storage/mysql/MySQLMetadataStorageModule.java +++ b/mysql-storage/src/main/java/io/druid/storage/mysql/MySQLMetadataStorageModule.java @@ -132,4 +132,4 @@ public class MySQLMetadataStorageModule implements DruidModule { return dbConnector.getDBI(); } -} \ No newline at end of file +} diff --git a/pom.xml b/pom.xml index e59e0bd9c90..a3e43115f81 100644 --- a/pom.xml +++ b/pom.xml @@ -389,6 +389,11 @@ commons-cli 1.2 + + org.apache.commons + commons-dbcp2 + 2.0.1 + net.jpountz.lz4 lz4 diff --git a/postgres-storage/pom.xml b/postgres-storage/pom.xml index b2f06ed99a8..6238866dcf5 100644 --- a/postgres-storage/pom.xml +++ b/postgres-storage/pom.xml @@ -50,19 +50,9 @@ ${project.parent.version} - io.druid - druid-indexing-hadoop - ${project.parent.version} - - - io.druid - druid-indexing-service - ${project.parent.version} - - - mysql - mysql-connector-java - 5.1.18 + org.postgresql + postgresql + 9.3-1102-jdbc41 org.jdbi @@ -85,4 +75,4 @@ - \ No newline at end of file + diff --git a/postgres-storage/src/main/java/io/druid/storage/postgres/PostgreSQLConnector.java b/postgres-storage/src/main/java/io/druid/storage/postgres/PostgreSQLConnector.java index dadb9791148..047833638c4 100644 --- a/postgres-storage/src/main/java/io/druid/storage/postgres/PostgreSQLConnector.java +++ b/postgres-storage/src/main/java/io/druid/storage/postgres/PostgreSQLConnector.java @@ -25,6 +25,7 @@ import com.metamx.common.logger.Logger; import io.druid.db.MetadataStorageConnectorConfig; import io.druid.db.MetadataStorageTablesConfig; import io.druid.db.SQLMetadataConnector; +import org.apache.commons.dbcp2.BasicDataSource; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; import org.skife.jdbi.v2.IDBI; @@ -43,8 +44,14 @@ public class PostgreSQLConnector extends SQLMetadataConnector public PostgreSQLConnector(Supplier config, Supplier dbTables) { super(config, dbTables); - this.dbi = new DBI(getDatasource()); + final BasicDataSource datasource = getDatasource(); + // PostgreSQL driver is classloader isolated as part of the extension + // so we need to help JDBC find the driver + datasource.setDriverClassLoader(getClass().getClassLoader()); + datasource.setDriverClassName("org.postgresql.Driver"); + + this.dbi = new DBI(datasource); } @Override diff --git a/server/pom.xml b/server/pom.xml index e264392c231..8485e146fc0 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -150,11 +150,6 @@ com.maxmind.geoip2 geoip2 - - mysql - mysql-connector-java - 5.1.18 - org.apache.derby derby diff --git a/server/src/main/java/io/druid/db/SQLMetadataConnector.java b/server/src/main/java/io/druid/db/SQLMetadataConnector.java index 23e25e2ada4..10037ca0263 100644 --- a/server/src/main/java/io/druid/db/SQLMetadataConnector.java +++ b/server/src/main/java/io/druid/db/SQLMetadataConnector.java @@ -23,7 +23,7 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.metamx.common.ISE; import com.metamx.common.logger.Logger; -import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.dbcp2.BasicDataSource; import org.skife.jdbi.v2.Batch; import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.Handle; @@ -31,7 +31,6 @@ import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.tweak.HandleCallback; import org.skife.jdbi.v2.util.ByteArrayMapper; -import javax.sql.DataSource; import java.sql.Connection; import java.util.List; import java.util.Map; @@ -77,6 +76,10 @@ public abstract class SQLMetadataConnector implements MetadataStorageConnector protected abstract boolean tableExists(Handle handle, final String tableName); + protected boolean isTransientException(Throwable e) { + return false; + } + public void createTable(final IDBI dbi, final String tableName, final Iterable sql) { try { @@ -348,7 +351,7 @@ public abstract class SQLMetadataConnector implements MetadataStorageConnector public MetadataStorageConnectorConfig getConfig() { return config.get(); } - protected DataSource getDatasource() + protected BasicDataSource getDatasource() { MetadataStorageConnectorConfig connectorConfig = getConfig(); diff --git a/server/src/main/java/io/druid/db/SQLMetadataRuleManagerProvider.java b/server/src/main/java/io/druid/db/SQLMetadataRuleManagerProvider.java index 411fa93dc03..cf810596235 100644 --- a/server/src/main/java/io/druid/db/SQLMetadataRuleManagerProvider.java +++ b/server/src/main/java/io/druid/db/SQLMetadataRuleManagerProvider.java @@ -85,4 +85,4 @@ public class SQLMetadataRuleManagerProvider implements MetadataRuleManagerProvid return new SQLMetadataRuleManager(jsonMapper, config, dbTables, dbi); } -} \ No newline at end of file +} diff --git a/server/src/main/java/io/druid/db/SQLMetadataStorageActionHandler.java b/server/src/main/java/io/druid/db/SQLMetadataStorageActionHandler.java index 625f9bc1d08..df726926f2a 100644 --- a/server/src/main/java/io/druid/db/SQLMetadataStorageActionHandler.java +++ b/server/src/main/java/io/druid/db/SQLMetadataStorageActionHandler.java @@ -23,7 +23,6 @@ import com.google.common.base.Predicate; import com.google.common.base.Throwables; import com.google.inject.Inject; import com.metamx.common.RetryUtils; -import com.mysql.jdbc.exceptions.MySQLTransientException; import io.druid.indexing.overlord.MetadataStorageActionHandler; import org.skife.jdbi.v2.Handle; import org.skife.jdbi.v2.IDBI; @@ -41,11 +40,16 @@ import java.util.concurrent.Callable; public class SQLMetadataStorageActionHandler implements MetadataStorageActionHandler { private final IDBI dbi; + private final SQLMetadataConnector connector; @Inject - public SQLMetadataStorageActionHandler(final IDBI dbi) + public SQLMetadataStorageActionHandler( + final IDBI dbi, + final SQLMetadataConnector connector + ) { this.dbi = dbi; + this.connector = connector; } /* Insert stuff. @returns number of entries inserted on success */ @@ -332,13 +336,12 @@ public class SQLMetadataStorageActionHandler implements MetadataStorageActionHan } } - private static boolean shouldRetryException(final Throwable e) + protected boolean shouldRetryException(final Throwable e) { return e != null && (e instanceof SQLTransientException - || e instanceof MySQLTransientException + || connector.isTransientException(e) || e instanceof SQLRecoverableException || e instanceof UnableToObtainConnectionException - || (e instanceof SQLException && ((SQLException) e).getErrorCode() == 1317) || (e instanceof SQLException && shouldRetryException(e.getCause())) || (e instanceof DBIException && shouldRetryException(e.getCause()))); } diff --git a/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java b/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java index dd17997b9d3..7981cf19972 100644 --- a/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java +++ b/server/src/main/java/io/druid/guice/DerbyMetadataStorageDruidModule.java @@ -23,6 +23,7 @@ import com.google.inject.Binder; import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.Provides; +import io.druid.db.SQLMetadataConnector; import io.druid.db.SQLMetadataRuleManager; import io.druid.db.SQLMetadataRuleManagerProvider; import io.druid.db.SQLMetadataSegmentManager; @@ -55,6 +56,9 @@ public class DerbyMetadataStorageDruidModule implements Module PolyBind.createChoice( binder, "druid.db.type", Key.get(MetadataStorageConnector.class), Key.get(DerbyConnector.class) ); + PolyBind.createChoice( + binder, "druid.db.type", Key.get(SQLMetadataConnector.class), Key.get(DerbyConnector.class) + ); PolyBind.createChoice( binder, "druid.db.type", Key.get(MetadataSegmentManager.class), Key.get(SQLMetadataSegmentManager.class) ); @@ -92,6 +96,11 @@ public class DerbyMetadataStorageDruidModule implements Module .to(DerbyConnector.class) .in(LazySingleton.class); + PolyBind.optionBinder(binder, Key.get(SQLMetadataConnector.class)) + .addBinding("derby") + .to(DerbyConnector.class) + .in(LazySingleton.class); + PolyBind.optionBinder(binder, Key.get(MetadataSegmentManager.class)) .addBinding("derby") .to(SQLMetadataSegmentManager.class)