include mysql-metadata-storage extension in distribution, but without… (#6497)

* include mysql-metadata-storage extension in distribution, but without the GPL-licensed connector library

* Install mysql connector package

* use symlinks to avoid versioning issues

* add documentation for fetching the mysql connector

distribution/pom.xml

@@ -164,25 +164,6 @@
                                     </arguments>
                                 </configuration>
                             </execution>
-                            <execution>
-                                <id>mysql-tarball</id>
-                                <phase>package</phase>
-                                <goals>
-                                    <goal>exec</goal>
-                                </goals>
-                                <configuration>
-                                    <executable>tar</executable>
-                                    <arguments>
-                                        <argument>-C</argument>
-                                        <argument>${project.build.directory}/extensions</argument>
-                                        <argument>-czvf</argument>
-                                        <argument>
-                                            ${project.build.directory}/mysql-metadata-storage-${project.parent.version}.tar.gz
-                                        </argument>
-                                        <argument>mysql-metadata-storage</argument>
-                                    </arguments>
-                                </configuration>
-                            </execution>
                         </executions>
                     </plugin>
                     <plugin>

distribution/src/assembly/assembly.xml

@@ -31,9 +31,6 @@
             <includes>
                 <include>*/*</include>
             </includes>
-            <excludes>
-                <exclude>mysql-metadata-storage/**</exclude>
-            </excludes>
             <outputDirectory>extensions</outputDirectory>
         </fileSet>
 

docs/content/development/extensions-core/mysql.md

@@ -6,6 +6,24 @@ layout: doc_page
 
 Make sure to [include](../../operations/including-extensions.html) `mysql-metadata-storage` as an extension.
 
+<div class="note caution">
+The MySQL extension requires the MySQL Connector/J library which is not included in the Druid distribution. 
+Refer to the following section for instructions on how to install this library.
+</div>
+
+## Installing the MySQL connector library
+
+This extension uses Oracle's MySQL JDBC driver which is not included in the Druid distribution and must be
+installed separately. There are a few ways to obtain this library:
+
+- It can be downloaded from the MySQL site at: https://dev.mysql.com/downloads/connector/j/
+- It can be fetched from Maven Central at: http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.38/mysql-connector-java-5.1.38.jar
+- It may be available through your package manager, e.g. as `libmysql-java` on APT for a Debian-based OS
+
+This should fetch a JAR file named similar to 'mysql-connector-java-x.x.xx.jar'.
+
+Copy or symlink this file to `extensions/mysql-metadata-storage` under the distribution root directory.
+
 ## Setting up MySQL
 
 1. Install MySQL

extensions-core/mysql-metadata-storage/pom.xml

@@ -51,6 +51,7 @@
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>
             <version>5.1.38</version>
+            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.jdbi</groupId>

extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java

@@ -45,6 +45,7 @@ public class MySQLConnector extends SQLMetadataConnector
   private static final String PAYLOAD_TYPE = "LONGBLOB";
   private static final String SERIAL_TYPE = "BIGINT(20) AUTO_INCREMENT";
   private static final String QUOTE_STRING = "`";
+  private static final String MYSQL_JDBC_DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver";
 
   private final DBI dbi;
 
@@ -57,11 +58,23 @@ public class MySQLConnector extends SQLMetadataConnector
   {
     super(config, dbTables);
 
+    try {
+      Class.forName(MYSQL_JDBC_DRIVER_CLASS_NAME, false, getClass().getClassLoader());
+    }
+    catch (ClassNotFoundException e) {
+      throw new ISE(e, "Could not find %s on the classpath. The MySQL Connector library is not included in the Druid "
+                   + "distribution but is required to use MySQL. Please download a compatible library (for example "
+                   + "'mysql-connector-java-5.1.38.jar') and place it under 'extensions/mysql-metadata-storage/'. See "
+                   + "https://druid.apache.org/downloads for more details.",
+                MYSQL_JDBC_DRIVER_CLASS_NAME
+      );
+    }
+
     final BasicDataSource datasource = getDatasource();
     // MySQL driver is classloader isolated as part of the extension
     // so we need to help JDBC find the driver
     datasource.setDriverClassLoader(getClass().getClassLoader());
-    datasource.setDriverClassName("com.mysql.jdbc.Driver");
+    datasource.setDriverClassName(MYSQL_JDBC_DRIVER_CLASS_NAME);
     datasource.addConnectionProperty("useSSL", String.valueOf(connectorConfig.isUseSSL()));
     if (connectorConfig.isUseSSL()) {
       log.info("SSL is enabled on this MySQL connection. ");

integration-tests/docker/Dockerfile

@@ -25,6 +25,11 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \
 # Add Druid jars
 ADD lib/* /usr/local/druid/lib/
 
+# Download the MySQL Java connector
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libmysql-java
+RUN ln -sf /usr/share/java/mysql-connector-java.jar /usr/local/druid/lib/mysql-connector-java.jar
+
 # Add sample data
 # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72.
 RUN find /var/lib/mysql -type f -exec touch {} \; && service mysql start \

integration-tests/docker/broker.conf

@@ -57,7 +57,7 @@ command=java
   -Ddruid.client.https.keyManagerPassword=druid123
   -Ddruid.client.https.keyStorePassword=druid123
   -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*
+  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
   org.apache.druid.cli.Main server broker
 redirect_stderr=true
 autorestart=false

integration-tests/docker/coordinator.conf

@@ -51,7 +51,7 @@ command=java
   -Ddruid.client.https.keyManagerPassword=druid123
   -Ddruid.client.https.keyStorePassword=druid123
   -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*
+  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
   org.apache.druid.cli.Main server coordinator
 redirect_stderr=true
 priority=100

integration-tests/docker/historical.conf

@@ -53,7 +53,7 @@ command=java
   -Ddruid.client.https.keyManagerPassword=druid123
   -Ddruid.client.https.keyStorePassword=druid123
   -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*
+  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
   org.apache.druid.cli.Main server historical
 redirect_stderr=true
 priority=100

integration-tests/docker/middlemanager.conf

@@ -58,7 +58,7 @@ command=java
   -Ddruid.client.https.keyStorePassword=druid123
   -Ddruid.startup.logging.logProperties=true
   -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*
+  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
   org.apache.druid.cli.Main server middleManager
 redirect_stderr=true
 priority=100

integration-tests/docker/overlord.conf

@@ -52,7 +52,7 @@ command=java
   -Ddruid.client.https.keyManagerPassword=druid123
   -Ddruid.client.https.keyStorePassword=druid123
   -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*
+  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
   org.apache.druid.cli.Main server overlord
 redirect_stderr=true
 priority=100

integration-tests/docker/router-no-client-auth-tls.conf

@@ -47,7 +47,7 @@ command=java
   -Ddruid.client.https.keyManagerPassword=druid123
   -Ddruid.client.https.keyStorePassword=druid123
   -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*
+  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
   org.apache.druid.cli.Main server router
 redirect_stderr=true
 priority=100

integration-tests/docker/router-permissive-tls.conf

@@ -47,7 +47,7 @@ command=java
   -Ddruid.client.https.keyManagerPassword=druid123
   -Ddruid.client.https.keyStorePassword=druid123
   -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*
+  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
   org.apache.druid.cli.Main server router
 redirect_stderr=true
 priority=100

integration-tests/docker/router.conf

@@ -46,7 +46,7 @@ command=java
   -Ddruid.client.https.keyManagerPassword=druid123
   -Ddruid.client.https.keyStorePassword=druid123
   -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml
-  -cp /shared/docker/lib/*
+  -cp /shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
   org.apache.druid.cli.Main server router
 redirect_stderr=true
 priority=100