NIFI-9585 Upgraded H2 from 1.4 to 2.1.210

- Added nifi-h2-database module shading H2 1.4.200
- Implemented version checking and migration

This closes #5724

Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Matthew Burgess 2022-01-27 18:37:12 -05:00 committed by exceptionfactory
parent 4e3871fec7
commit bcc8d03314
No known key found for this signature in database
GPG Key ID: 29B6A52D2AAE8DBA
27 changed files with 502 additions and 295 deletions

View File

@ -825,11 +825,6 @@ limitations under the License.
<artifactId>guava</artifactId>
<version>31.0.1-jre</version>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.4.199</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>

View File

@ -0,0 +1,37 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-h2</artifactId>
<version>1.16.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-h2-database-migrator</artifactId>
<dependencies>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-h2-database</artifactId>
<version>1.16.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,107 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.h2.database.migration;
import org.apache.nifi.org.h2.jdbcx.JdbcDataSource;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import static org.apache.nifi.h2.database.migration.H2DatabaseUpdater.EXPORT_FILE_POSTFIX;
import static org.apache.nifi.h2.database.migration.H2DatabaseUpdater.EXPORT_FILE_PREFIX;
public class H2DatabaseMigrator {
public static final String BACKUP_FILE_POSTFIX = ".migration_backup";
public static void exportAndBackup(final String dbUrl, final String dbPath, final String user, final String pass) {
// Attempt to connect with the latest driver
JdbcDataSource migrationDataSource = new JdbcDataSource();
migrationDataSource.setURL(dbUrl);
migrationDataSource.setUser(user);
migrationDataSource.setPassword(pass);
final File dbPathFile = Paths.get(dbPath).toFile();
String exportFile = Paths.get(dbPathFile.getParent(), EXPORT_FILE_PREFIX + dbPathFile.getName() + EXPORT_FILE_POSTFIX).toString();
Connection conn;
Statement s;
try {
conn = migrationDataSource.getConnection();
s = conn.createStatement();
} catch (SQLException sqle) {
final String message = String.format("H2 1.4 connection failed URL [%s] Path [%s] SQL State [%s]", dbUrl, dbPath, sqle.getSQLState());
throw new RuntimeException(message, sqle);
}
try {
// Perform an export of the database to SQL statements
s.execute("SCRIPT TO '" + exportFile + "'");
} catch (SQLException sqle) {
try {
s.close();
} catch (SQLException se2) {
// Ignore, the error will be handled
}
final String message = String.format("H2 1.4 export failed URL [%s] Path [%s] SQL State [%s]", dbUrl, dbPath, sqle.getSQLState());
throw new RuntimeException(message, sqle);
}
closeQuietly(s);
closeQuietly(conn);
// Verify the export file exists
if (!Files.exists(Paths.get(exportFile))) {
throw new RuntimeException(String.format("H2 1.4 export failed URL [%s] Path [%s] Export File not found [%s]", dbUrl, dbPath, exportFile));
}
// Now that the export file exists, backup (rename) the DB files so the main process with the newer H2 driver can create and import the previous database
File dbDir = new File(dbPath).getParentFile();
File[] dbFiles = dbDir.listFiles((dir, name) -> !name.endsWith(EXPORT_FILE_POSTFIX) && name.startsWith(dbPathFile.getName()));
if (dbFiles == null || dbFiles.length == 0) {
throw new RuntimeException(String.format("H2 1.4 backup failed URL [%s] Path [%s] no database files found", dbUrl, dbPath));
}
for (File dbFile : dbFiles) {
File dbBackupFile = new File(dbFile.getAbsolutePath() + BACKUP_FILE_POSTFIX);
if (!dbFile.renameTo(dbBackupFile)) {
throw new RuntimeException(String.format("H2 1.4 backup failed URL [%s] Path [%s] rename failed [%s]", dbUrl, dbPath, dbFile));
}
}
}
private static void closeQuietly(final Statement statement) {
try {
statement.close();
} catch (final SQLException e) {
// Ignore, nothing to be done
}
}
private static void closeQuietly(final Connection connection) {
try {
connection.close();
} catch (final SQLException e) {
// Ignore, nothing to be done
}
}
}

View File

@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.h2.database.migration;
import org.h2.jdbc.JdbcSQLNonTransientException;
import org.h2.jdbcx.JdbcDataSource;
import org.h2.mvstore.MVStoreException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
public class H2DatabaseUpdater {
private static final Logger logger = LoggerFactory.getLogger(H2DatabaseUpdater.class);
public static final String EXPORT_FILE_PREFIX = "export_";
public static final String EXPORT_FILE_POSTFIX = ".sql";
public static final String H2_URL_PREFIX = "jdbc:h2:";
public static void checkAndPerformMigration(final String dbPathNoExtension, final String dbUrl, final String user, final String pass) throws Exception {
final JdbcDataSource migrationDataSource = new JdbcDataSource();
// Attempt to connect with the latest driver
migrationDataSource.setURL(dbUrl);
migrationDataSource.setUser(user);
migrationDataSource.setPassword(pass);
try (Connection connection = migrationDataSource.getConnection()) {
return;
} catch (JdbcSQLNonTransientException jsqlnte) {
// Migration/version issues will be caused by an MVStoreException
final Throwable exceptionCause = jsqlnte.getCause();
if (exceptionCause instanceof MVStoreException) {
// Check for specific error message
final String errorMessage = exceptionCause.getMessage();
if (!errorMessage.contains("The write format")
&& !errorMessage.contains("is smaller than the supported format")) {
throw jsqlnte;
}
}
} catch (SQLException sqle) {
throw new RuntimeException(String.format("H2 connection failed URL [%s] File [%s]", dbUrl, dbPathNoExtension), sqle);
}
// At this point it is known that migration should be attempted
logger.info("H2 1.4 database detected [{}]: starting migration to H2 2.1", dbPathNoExtension);
H2DatabaseMigrator.exportAndBackup(dbUrl, dbPathNoExtension, user, pass);
// The export file has been created and the DB has been backed up, now create a new one with the same name and run the SQL script to import the database
try (Connection migrationConnection = migrationDataSource.getConnection();
Statement s = migrationConnection.createStatement()) {
final Path dbFilePath = Paths.get(dbPathNoExtension);
final String dbDirectory = dbFilePath.getParent().toFile().getAbsolutePath();
// use RUNSCRIPT to recreate the database
final String exportSqlLocation = dbDirectory + File.separator
+ H2DatabaseUpdater.EXPORT_FILE_PREFIX + dbFilePath.toFile().getName() + H2DatabaseUpdater.EXPORT_FILE_POSTFIX;
s.execute("RUNSCRIPT FROM '" + exportSqlLocation + "'");
} catch (SQLException sqle) {
throw new IOException(String.format("H2 import database creation failed URL [%s]", dbUrl), sqle);
}
logger.info("H2 1.4 to 2.1 migration completed [{}]", dbPathNoExtension);
}
}

View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.h2.database.migration;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestH2DatabaseUpdater {
public static final String DB_NAME = "nifi-flow-audit";
public static final String MVDB_EXTENSION = ".mv.db";
public static final String ORIGINAL_AUDIT_DB_PATH = "./src/test/resources/" + DB_NAME;
public static final String ORIGINAL_AUDIT_DB_PATH_FILE = ORIGINAL_AUDIT_DB_PATH + MVDB_EXTENSION;
@TempDir
File tmpDir;
@BeforeEach
public void copyDatabaseFile() throws IOException {
// Copy the legacy database file to a temporary directory
final Path origAuditDbPath = Paths.get(ORIGINAL_AUDIT_DB_PATH_FILE);
final Path destAuditDbPath = Paths.get(tmpDir.getAbsolutePath(), DB_NAME + MVDB_EXTENSION);
Files.copy(origAuditDbPath, destAuditDbPath, REPLACE_EXISTING);
}
@Test
public void testMigration() throws Exception {
final Path testAuditDbPath = Paths.get(tmpDir.getAbsolutePath(), DB_NAME);
final File dbFileNoExtension = testAuditDbPath.toFile();
final String migrationDbUrl = H2DatabaseUpdater.H2_URL_PREFIX + dbFileNoExtension + ";LOCK_MODE=3";
H2DatabaseUpdater.checkAndPerformMigration(dbFileNoExtension.getAbsolutePath(), migrationDbUrl, "nf", "nf");
// Verify the export, backup, and new database files were created
final File exportFile = Paths.get(dbFileNoExtension.getParent(), H2DatabaseUpdater.EXPORT_FILE_PREFIX + dbFileNoExtension.getName() + H2DatabaseUpdater.EXPORT_FILE_POSTFIX).toFile();
assertTrue(exportFile.exists());
File dbDir = dbFileNoExtension.getParentFile();
File[] backupFiles = dbDir.listFiles((dir, name) -> name.endsWith(H2DatabaseMigrator.BACKUP_FILE_POSTFIX) && name.startsWith(dbFileNoExtension.getName()));
try {
assertNotNull(backupFiles);
// The database and its trace file should exist after the initial connection is made, so they both should be backed up
assertEquals(2, backupFiles.length);
final File newDbFile = Paths.get(tmpDir.getAbsolutePath(), DB_NAME + MVDB_EXTENSION).toFile();
assertTrue(newDbFile.exists());
} finally {
// Remove the export and backup files
exportFile.delete();
for (File f : backupFiles) {
f.delete();
}
}
}
}

View File

@ -0,0 +1,70 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
license agreements. See the NOTICE file distributed with this work for additional
information regarding copyright ownership. The ASF licenses this file to
You under the Apache License, Version 2.0 (the "License"); you may not use
this file except in compliance with the License. You may obtain a copy of
the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
by applicable law or agreed to in writing, software distributed under the
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
OF ANY KIND, either express or implied. See the License for the specific
language governing permissions and limitations under the License. -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-h2</artifactId>
<version>1.16.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-h2-database</artifactId>
<version>1.16.0-SNAPSHOT</version>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.4.200</version>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.2.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
<artifact>com.h2database:h2</artifact>
<excludes>
<exclude>org/h2/fulltext/**</exclude>
<exclude>org/h2/jdbc/JdbcSQLXML.class</exclude>
<exclude>org/h2/jmx/**</exclude>
<exclude>org/h2/server/**</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>org.h2</pattern>
<shadedPattern>org.apache.nifi.org.h2</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

30
nifi-h2/pom.xml Normal file
View File

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi</artifactId>
<version>1.16.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-h2</artifactId>
<packaging>pom</packaging>
<modules>
<module>nifi-h2-database</module>
<module>nifi-h2-database-migrator</module>
</modules>
</project>

View File

@ -85,7 +85,7 @@
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.4.187</version>
<version>${h2.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

View File

@ -48,25 +48,25 @@ public class TestJdbcTypesH2 {
}
String createTable = " CREATE TABLE `users` ( "
+ " `id` int(11) NOT NULL AUTO_INCREMENT, "
+ " `id` int NOT NULL AUTO_INCREMENT, "
+ " `email` varchar(255) NOT NULL, "
+ " `password` varchar(255) DEFAULT NULL, "
+ " `activation_code` varchar(255) DEFAULT NULL, "
+ " `forgotten_password_code` varchar(255) DEFAULT NULL, "
+ " `forgotten_password_time` datetime DEFAULT NULL, "
+ " `created` datetime NOT NULL, "
+ " `active` tinyint(1) NOT NULL DEFAULT '0', "
+ " `home_module_id` int(11) DEFAULT NULL, "
+ " `active` tinyint NOT NULL DEFAULT '0', "
+ " `home_module_id` int DEFAULT NULL, "
+ " somebinary BINARY default null, "
+ " somebinary BINARY(4) default null, "
+ " somebinary2 VARBINARY default null, "
+ " somebinary3 LONGVARBINARY default null, "
+ " somearray ARRAY default null, "
+ " somearray INTEGER ARRAY default null, "
+ " someblob BLOB default null, "
+ " someclob CLOB default null, "
+ " PRIMARY KEY (`id`), "
+ " UNIQUE KEY `email` (`email`) ) " ;
+ " CONSTRAINT unique_email UNIQUE (`email`) ) " ;
// + " KEY `home_module_id` (`home_module_id`) )" ;
/* + " CONSTRAINT `users_ibfk_1` FOREIGN KEY (`home_module_id`) REFERENCES "
+ "`modules` (`id`) ON DELETE SET NULL "
@ -92,7 +92,7 @@ public class TestJdbcTypesH2 {
// + " values ('robert.gates@cold.com', '******', 'CAS', 'ounou', '2005-12-09', '2005-12-03', 1, 5)");
st.executeUpdate("insert into users (email, password, activation_code, created, active, somebinary, somebinary2, somebinary3, someblob, someclob) "
+ " values ('mari.gates@cold.com', '******', 'CAS', '2005-12-03', 3, '66FF', 'ABDF', 'EE64', 'BB22', 'CC88')");
+ " values ('mari.gates@cold.com', '******', 'CAS', '2005-12-03', 3, 0x66FF, 'ABDF', 'EE64', 'BB22', 'CC88')");
final ResultSet resultSet = st.executeQuery("select U.*, ROW_NUMBER() OVER () as rownr from users U");
// final ResultSet resultSet = st.executeQuery("select U.active from users U");

View File

@ -58,6 +58,11 @@
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-h2-database-migrator</artifactId>
<version>1.16.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>

View File

@ -17,6 +17,7 @@
package org.apache.nifi.admin;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.h2.database.migration.H2DatabaseUpdater;
import org.apache.nifi.util.NiFiProperties;
import org.h2.jdbcx.JdbcConnectionPool;
import org.slf4j.Logger;
@ -84,7 +85,7 @@ public class AuditDataSourceFactoryBean implements FactoryBean {
private static final String CREATE_CONFIGURE_DETAILS_TABLE = "CREATE TABLE CONFIGURE_DETAILS ("
+ "ACTION_ID INT NOT NULL PRIMARY KEY, "
+ "NAME VARCHAR2(1000) NOT NULL, "
+ "VALUE VARCHAR2(5000), "
+ "\"VALUE\" VARCHAR2(5000), "
+ "PREVIOUS_VALUE VARCHAR2(5000), "
+ "FOREIGN KEY (ACTION_ID) REFERENCES ACTION(ID)"
+ ")";
@ -127,15 +128,19 @@ public class AuditDataSourceFactoryBean implements FactoryBean {
File repositoryDirectory = new File(repositoryDirectoryPath);
// get a handle to the database file
File databaseFile = new File(repositoryDirectory, AUDIT_DATABASE_FILE_NAME);
File dbFileNoExtension = new File(repositoryDirectory, AUDIT_DATABASE_FILE_NAME);
// format the database url
String databaseUrl = "jdbc:h2:" + databaseFile + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3";
String databaseUrl = H2DatabaseUpdater.H2_URL_PREFIX + dbFileNoExtension + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3";
String databaseUrlAppend = properties.getProperty(NiFiProperties.H2_URL_APPEND);
if (StringUtils.isNotBlank(databaseUrlAppend)) {
databaseUrl += databaseUrlAppend;
}
// Migrate an existing database if required
final String migrationDbUrl = H2DatabaseUpdater.H2_URL_PREFIX + dbFileNoExtension + ";LOCK_MODE=3";
H2DatabaseUpdater.checkAndPerformMigration(dbFileNoExtension.getAbsolutePath(), migrationDbUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD);
// create the pool
connectionPool = JdbcConnectionPool.create(databaseUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD);
connectionPool.setMaxConnections(MAX_CONNECTIONS);

View File

@ -17,6 +17,7 @@
package org.apache.nifi.admin;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.h2.database.migration.H2DatabaseUpdater;
import org.apache.nifi.util.NiFiProperties;
import org.h2.jdbcx.JdbcConnectionPool;
import org.slf4j.Logger;
@ -84,8 +85,12 @@ public class IdpDataSourceFactoryBean implements FactoryBean<JdbcConnectionPool>
File repositoryDirectory = new File(repositoryDirectoryPath);
// create a handle to the database directory and file
File databaseFile = new File(repositoryDirectory, IDP_DATABASE_FILE_NAME);
String databaseUrl = getDatabaseUrl(databaseFile);
File dbFileNoExtension = new File(repositoryDirectory, IDP_DATABASE_FILE_NAME);
String databaseUrl = getDatabaseUrl(dbFileNoExtension);
// Migrate an existing database if required
final String migrationDbUrl = H2DatabaseUpdater.H2_URL_PREFIX + dbFileNoExtension + ";LOCK_MODE=3";
H2DatabaseUpdater.checkAndPerformMigration(dbFileNoExtension.getAbsolutePath(), migrationDbUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD);
// create the pool
connectionPool = JdbcConnectionPool.create(databaseUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD);
@ -125,7 +130,7 @@ public class IdpDataSourceFactoryBean implements FactoryBean<JdbcConnectionPool>
}
private String getDatabaseUrl(File databaseFile) {
String databaseUrl = "jdbc:h2:" + databaseFile + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3";
String databaseUrl = H2DatabaseUpdater.H2_URL_PREFIX + databaseFile + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3";
String databaseUrlAppend = properties.getProperty(NiFiProperties.H2_URL_APPEND);
if (StringUtils.isNotBlank(databaseUrlAppend)) {
databaseUrl += databaseUrlAppend;

View File

@ -307,7 +307,7 @@
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.4.199</version>
<version>${h2.version}</version>
</dependency>
<!-- open id connect - override transitive dependency version ranges -->

View File

@ -268,7 +268,7 @@
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.4.187</version>
<version>${h2.version}</version>
<scope>test</scope>
</dependency>
<dependency>

View File

@ -121,7 +121,7 @@
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.4.192</version>
<version>${h2.version}</version>
<scope>test</scope>
</dependency>
<dependency>

View File

@ -206,6 +206,11 @@
<artifactId>nifi-registry-flow-diff</artifactId>
<version>1.16.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-h2-database-migrator</artifactId>
<version>1.16.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
@ -307,6 +312,11 @@
<artifactId>flyway-core</artifactId>
<version>${flyway.version}</version>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-mysql</artifactId>
<version>${flyway.version}</version>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>

View File

@ -18,8 +18,11 @@ package org.apache.nifi.registry.db;
import org.flywaydb.core.api.FlywayException;
import org.flywaydb.core.api.configuration.FluentConfiguration;
import org.flywaydb.core.internal.jdbc.DatabaseType;
import org.flywaydb.core.internal.database.DatabaseType;
import org.flywaydb.core.internal.database.DatabaseTypeRegister;
import org.flywaydb.core.internal.database.postgresql.PostgreSQLDatabaseType;
import org.flywaydb.core.internal.jdbc.JdbcUtils;
import org.flywaydb.database.mysql.MySQLDatabaseType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.flyway.FlywayConfigurationCustomizer;
@ -52,21 +55,17 @@ public class CustomFlywayConfiguration implements FlywayConfigurationCustomizer
@Override
public void customize(final FluentConfiguration configuration) {
final DatabaseType databaseType = getDatabaseType(configuration.getDataSource());
LOGGER.info("Determined database type is {}", new Object[] {databaseType.name()});
LOGGER.info("Determined database type is {}", databaseType.getName());
switch (databaseType) {
case MYSQL:
LOGGER.info("Setting migration locations to {}", new Object[] {LOCATIONS_MYSQL});
configuration.locations(LOCATIONS_MYSQL);
break;
case POSTGRESQL:
LOGGER.info("Setting migration locations to {}", new Object[] {LOCATIONS_POSTGRES});
configuration.locations(LOCATIONS_POSTGRES);
break;
default:
LOGGER.info("Setting migration locations to {}", new Object[] {LOCATIONS_DEFAULT});
configuration.locations(LOCATIONS_DEFAULT);
break;
if (databaseType.equals(new MySQLDatabaseType())) {
LOGGER.info("Setting migration locations to {}", LOCATIONS_MYSQL);
configuration.locations(LOCATIONS_MYSQL);
} else if (databaseType.equals(new PostgreSQLDatabaseType())) {
LOGGER.info("Setting migration locations to {}", LOCATIONS_POSTGRES);
configuration.locations(LOCATIONS_POSTGRES);
} else {
LOGGER.info("Setting migration locations to {}", LOCATIONS_DEFAULT);
configuration.locations(LOCATIONS_DEFAULT);
}
// At some point Flyway changed their default table name: https://github.com/flyway/flyway/issues/1848
@ -88,7 +87,7 @@ public class CustomFlywayConfiguration implements FlywayConfigurationCustomizer
*/
private DatabaseType getDatabaseType(final DataSource dataSource) {
try (final Connection connection = dataSource.getConnection()) {
return DatabaseType.fromJdbcConnection(connection);
return DatabaseTypeRegister.getDatabaseTypeForConnection(connection);
} catch (SQLException e) {
LOGGER.error(e.getMessage(), e);
throw new FlywayException("Unable to obtain connection from Flyway DataSource", e);

View File

@ -18,6 +18,7 @@ package org.apache.nifi.registry.db;
import com.zaxxer.hikari.HikariDataSource;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.h2.database.migration.H2DatabaseUpdater;
import org.apache.nifi.registry.properties.NiFiRegistryProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -28,6 +29,7 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import javax.sql.DataSource;
import java.io.File;
/**
* Overriding Spring Boot's normal automatic creation of a DataSource in order to use the properties
@ -87,11 +89,27 @@ public class DataSourceFactory {
.build();
if (dataSource instanceof HikariDataSource) {
LOGGER.info("Setting maximum pool size on HikariDataSource to {}", new Object[]{properties.getDatabaseMaxConnections()});
((HikariDataSource)dataSource).setMaximumPoolSize(properties.getDatabaseMaxConnections());
LOGGER.info("Setting maximum pool size on HikariDataSource to {}", properties.getDatabaseMaxConnections());
((HikariDataSource) dataSource).setMaximumPoolSize(properties.getDatabaseMaxConnections());
}
// Migrate an H2 existing database if required
if (databaseUrl.startsWith(H2DatabaseUpdater.H2_URL_PREFIX)) {
final File databaseFile = getFileFromH2URL(databaseUrl);
final String migrationDbUrl = H2DatabaseUpdater.H2_URL_PREFIX + databaseFile + ";LOCK_MODE=3";
try {
H2DatabaseUpdater.checkAndPerformMigration(databaseFile.getAbsolutePath(), migrationDbUrl, databaseUsername, databasePassword);
} catch (Exception e) {
throw new RuntimeException("H2 database migration failed", e);
}
}
return dataSource;
}
private File getFileFromH2URL(final String databaseUrl) {
final String dbPath = databaseUrl.substring(H2DatabaseUpdater.H2_URL_PREFIX.length(), databaseUrl.indexOf(";"));
return new File(dbPath);
}
}

View File

@ -1,25 +0,0 @@
-- Licensed to the Apache Software Foundation (ASF) under one or more
-- contributor license agreements. See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- The ASF licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
ALTER TABLE BUCKET ALTER COLUMN NAME VARCHAR2(1000);
ALTER TABLE BUCKET ALTER COLUMN DESCRIPTION VARCHAR2(65535);
ALTER TABLE BUCKET_ITEM ALTER COLUMN NAME VARCHAR2(1000);
ALTER TABLE BUCKET_ITEM ALTER COLUMN DESCRIPTION VARCHAR2(65535);
ALTER TABLE FLOW_SNAPSHOT ALTER COLUMN CREATED_BY VARCHAR2(4096);
ALTER TABLE FLOW_SNAPSHOT ALTER COLUMN COMMENTS VARCHAR2(65535);
ALTER TABLE SIGNING_KEY ALTER COLUMN TENANT_IDENTITY VARCHAR2(4096);

View File

@ -1,27 +0,0 @@
-- Licensed to the Apache Software Foundation (ASF) under one or more
-- contributor license agreements. See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- The ASF licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
CREATE ALIAS IF NOT EXISTS EXECUTE AS $$ void executeSql(Connection conn, String sql)
throws SQLException { conn.createStatement().executeUpdate(sql); } $$;
call execute('ALTER TABLE BUCKET_ITEM DROP CONSTRAINT ' ||
(
SELECT DISTINCT CONSTRAINT_NAME
FROM INFORMATION_SCHEMA.CONSTRAINTS
WHERE TABLE_NAME = 'BUCKET_ITEM'
AND COLUMN_LIST = 'NAME'
AND CONSTRAINT_TYPE = 'UNIQUE'
)
);

View File

@ -1,54 +0,0 @@
-- Licensed to the Apache Software Foundation (ASF) under one or more
-- contributor license agreements. See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- The ASF licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
CREATE TABLE BUCKET (
ID VARCHAR2(50) NOT NULL PRIMARY KEY,
NAME VARCHAR2(200) NOT NULL UNIQUE,
DESCRIPTION VARCHAR(4096),
CREATED TIMESTAMP NOT NULL
);
CREATE TABLE BUCKET_ITEM (
ID VARCHAR2(50) NOT NULL PRIMARY KEY,
NAME VARCHAR2(200) NOT NULL UNIQUE,
DESCRIPTION VARCHAR(4096),
CREATED TIMESTAMP NOT NULL,
MODIFIED TIMESTAMP NOT NULL,
ITEM_TYPE VARCHAR(50) NOT NULL,
BUCKET_ID VARCHAR2(50) NOT NULL,
FOREIGN KEY (BUCKET_ID) REFERENCES BUCKET(ID)
);
CREATE TABLE FLOW (
ID VARCHAR2(50) NOT NULL PRIMARY KEY,
FOREIGN KEY (ID) REFERENCES BUCKET_ITEM(ID)
);
CREATE TABLE FLOW_SNAPSHOT (
FLOW_ID VARCHAR2(50) NOT NULL,
VERSION INT NOT NULL,
CREATED TIMESTAMP NOT NULL,
CREATED_BY VARCHAR2(200) NOT NULL,
COMMENTS VARCHAR(4096),
PRIMARY KEY (FLOW_ID, VERSION),
FOREIGN KEY (FLOW_ID) REFERENCES FLOW(ID)
);
CREATE TABLE SIGNING_KEY (
ID VARCHAR2(50) NOT NULL,
TENANT_IDENTITY VARCHAR2(50) NOT NULL UNIQUE,
KEY_VALUE VARCHAR2(50) NOT NULL,
PRIMARY KEY (ID)
);

View File

@ -1,143 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.db.migration;
import org.apache.nifi.registry.db.entity.BucketItemEntityType;
import org.flywaydb.core.Flyway;
import org.junit.Before;
import org.junit.Test;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.jdbc.core.JdbcTemplate;
import javax.sql.DataSource;
import java.util.Date;
import java.util.List;
import static org.junit.Assert.assertEquals;
/**
* Test running the legacy Flyway migrations against an in-memory H2 and then using the LegacyDatabaseService to
* retrieve data. Purposely not using Spring test annotations here to avoid interfering with the normal DB context/flyway.
*/
public class TestLegacyDatabaseService {
private DataSource dataSource;
private JdbcTemplate jdbcTemplate;
private Flyway flyway;
private BucketEntityV1 bucketEntityV1;
private FlowEntityV1 flowEntityV1;
private FlowSnapshotEntityV1 flowSnapshotEntityV1;
@Before
public void setup() {
dataSource = DataSourceBuilder.create()
.url("jdbc:h2:mem:legacydb")
.driverClassName("org.h2.Driver")
.build();
jdbcTemplate = new JdbcTemplate(dataSource);
flyway = Flyway.configure()
.dataSource(dataSource)
.locations("db/migration/original")
.load();
flyway.migrate();
bucketEntityV1 = new BucketEntityV1();
bucketEntityV1.setId("1");
bucketEntityV1.setName("Bucket1");
bucketEntityV1.setDescription("This is bucket 1");
bucketEntityV1.setCreated(new Date());
jdbcTemplate.update("INSERT INTO bucket (ID, NAME, DESCRIPTION, CREATED) VALUES (?, ?, ?, ?)",
bucketEntityV1.getId(),
bucketEntityV1.getName(),
bucketEntityV1.getDescription(),
bucketEntityV1.getCreated());
flowEntityV1 = new FlowEntityV1();
flowEntityV1.setId("1");
flowEntityV1.setBucketId(bucketEntityV1.getId());
flowEntityV1.setName("Flow1");
flowEntityV1.setDescription("This is flow1");
flowEntityV1.setCreated(new Date());
flowEntityV1.setModified(new Date());
jdbcTemplate.update("INSERT INTO bucket_item (ID, NAME, DESCRIPTION, CREATED, MODIFIED, ITEM_TYPE, BUCKET_ID) VALUES (?, ?, ?, ?, ?, ?, ?)",
flowEntityV1.getId(),
flowEntityV1.getName(),
flowEntityV1.getDescription(),
flowEntityV1.getCreated(),
flowEntityV1.getModified(),
BucketItemEntityType.FLOW.toString(),
flowEntityV1.getBucketId());
jdbcTemplate.update("INSERT INTO flow (ID) VALUES (?)", flowEntityV1.getId());
flowSnapshotEntityV1 = new FlowSnapshotEntityV1();
flowSnapshotEntityV1.setFlowId(flowEntityV1.getId());
flowSnapshotEntityV1.setVersion(1);
flowSnapshotEntityV1.setComments("This is v1");
flowSnapshotEntityV1.setCreated(new Date());
flowSnapshotEntityV1.setCreatedBy("user1");
jdbcTemplate.update("INSERT INTO flow_snapshot (FLOW_ID, VERSION, CREATED, CREATED_BY, COMMENTS) VALUES (?, ?, ?, ?, ?)",
flowSnapshotEntityV1.getFlowId(),
flowSnapshotEntityV1.getVersion(),
flowSnapshotEntityV1.getCreated(),
flowSnapshotEntityV1.getCreatedBy(),
flowSnapshotEntityV1.getComments());
}
@Test
public void testGetLegacyData() {
final LegacyDatabaseService service = new LegacyDatabaseService(dataSource);
final List<BucketEntityV1> buckets = service.getAllBuckets();
assertEquals(1, buckets.size());
final BucketEntityV1 b = buckets.stream().findFirst().get();
assertEquals(bucketEntityV1.getId(), b.getId());
assertEquals(bucketEntityV1.getName(), b.getName());
assertEquals(bucketEntityV1.getDescription(), b.getDescription());
assertEquals(bucketEntityV1.getCreated(), b.getCreated());
final List<FlowEntityV1> flows = service.getAllFlows();
assertEquals(1, flows.size());
final FlowEntityV1 f = flows.stream().findFirst().get();
assertEquals(flowEntityV1.getId(), f.getId());
assertEquals(flowEntityV1.getName(), f.getName());
assertEquals(flowEntityV1.getDescription(), f.getDescription());
assertEquals(flowEntityV1.getCreated(), f.getCreated());
assertEquals(flowEntityV1.getModified(), f.getModified());
assertEquals(flowEntityV1.getBucketId(), f.getBucketId());
final List<FlowSnapshotEntityV1> flowSnapshots = service.getAllFlowSnapshots();
assertEquals(1, flowSnapshots.size());
final FlowSnapshotEntityV1 fs = flowSnapshots.stream().findFirst().get();
assertEquals(flowSnapshotEntityV1.getFlowId(), fs.getFlowId());
assertEquals(flowSnapshotEntityV1.getVersion(), fs.getVersion());
assertEquals(flowSnapshotEntityV1.getComments(), fs.getComments());
assertEquals(flowSnapshotEntityV1.getCreatedBy(), fs.getCreatedBy());
assertEquals(flowSnapshotEntityV1.getCreated(), fs.getCreated());
}
}

View File

@ -56,5 +56,11 @@
<version>${flyway.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-mysql</artifactId>
<version>${flyway.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -27,7 +27,9 @@ import org.apache.nifi.registry.revision.api.RevisionUpdate;
import org.apache.nifi.registry.revision.api.UpdateRevisionTask;
import org.apache.nifi.registry.revision.standard.StandardRevisionClaim;
import org.apache.nifi.registry.revision.standard.StandardUpdateResult;
import org.flywaydb.core.internal.jdbc.DatabaseType;
import org.flywaydb.core.internal.database.DatabaseType;
import org.flywaydb.core.internal.database.DatabaseTypeRegister;
import org.flywaydb.database.mysql.MySQLDatabaseType;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@ -97,8 +99,8 @@ public class TestJdbcRevisionManager {
try (final Connection connection = dataSource.getConnection()) {
final String createTableSql;
final DatabaseType databaseType = DatabaseType.fromJdbcConnection(connection);
if (databaseType == DatabaseType.MYSQL) {
final DatabaseType databaseType = DatabaseTypeRegister.getDatabaseTypeForConnection(connection);
if (databaseType.equals(new MySQLDatabaseType())) {
createTableSql = CREATE_TABLE_SQL_MYSQL;
} else {
createTableSql = CREATE_TABLE_SQL_DEFAULT;

View File

@ -38,11 +38,10 @@
<properties>
<jax.rs.api.version>2.1</jax.rs.api.version>
<spring.boot.version>2.6.3</spring.boot.version>
<flyway.version>6.5.7</flyway.version>
<flyway.tests.version>6.4.0</flyway.tests.version>
<flyway.version>8.4.2</flyway.version>
<flyway.tests.version>7.0.0</flyway.tests.version>
<swagger.ui.version>3.12.0</swagger.ui.version>
<testcontainers.version>1.16.0</testcontainers.version>
<h2.version>1.4.199</h2.version>
<groovy.eclipse.compiler.version>3.4.0-01</groovy.eclipse.compiler.version>
<jaxb.version>2.3.2</jaxb.version>
<jgit.version>5.13.0.202109080827-r</jgit.version>

View File

@ -41,6 +41,7 @@
<module>nifi-toolkit</module>
<module>nifi-manifest</module>
<module>c2</module>
<module>nifi-h2</module>
</modules>
<url>https://nifi.apache.org</url>
<organization>
@ -123,6 +124,7 @@
<netty.4.version>4.1.73.Final</netty.4.version>
<spring.version>5.3.15</spring.version>
<spring.security.version>5.6.1</spring.security.version>
<h2.version>2.1.210</h2.version>
</properties>
<repositories>