From bcc8d03314889e7d2d0724390059d0315efe2a34 Mon Sep 17 00:00:00 2001 From: Matthew Burgess Date: Thu, 27 Jan 2022 18:37:12 -0500 Subject: [PATCH] NIFI-9585 Upgraded H2 from 1.4 to 2.1.210 - Added nifi-h2-database module shading H2 1.4.200 - Implemented version checking and migration This closes #5724 Signed-off-by: David Handermann --- minifi/pom.xml | 5 - nifi-h2/nifi-h2-database-migrator/pom.xml | 37 +++++ .../migration/H2DatabaseMigrator.java | 107 +++++++++++++ .../database/migration/H2DatabaseUpdater.java | 85 +++++++++++ .../migration/TestH2DatabaseUpdater.java | 81 ++++++++++ .../src/test/resources/nifi-flow-audit.mv.db | Bin 0 -> 20480 bytes nifi-h2/nifi-h2-database/pom.xml | 70 +++++++++ nifi-h2/pom.xml | 30 ++++ .../nifi-database-utils/pom.xml | 2 +- .../apache/nifi/util/db/TestJdbcTypesH2.java | 14 +- .../nifi-administration/pom.xml | 5 + .../admin/AuditDataSourceFactoryBean.java | 11 +- .../nifi/admin/IdpDataSourceFactoryBean.java | 11 +- .../nifi-framework-bundle/pom.xml | 2 +- nifi-nar-bundles/nifi-standard-bundle/pom.xml | 2 +- .../nifi-dbcp-service/pom.xml | 2 +- .../nifi-registry-framework/pom.xml | 10 ++ .../db/CustomFlywayConfiguration.java | 31 ++-- .../nifi/registry/db/DataSourceFactory.java | 22 ++- .../original/V1.2__IncreaseColumnSizes.sql | 25 --- .../V1.3__DropBucketItemNameUniqueness.sql | 27 ---- .../db/migration/original/V1__Initial.sql | 54 ------- .../migration/TestLegacyDatabaseService.java | 143 ------------------ .../pom.xml | 6 + .../jdbc/TestJdbcRevisionManager.java | 8 +- nifi-registry/pom.xml | 5 +- pom.xml | 2 + 27 files changed, 502 insertions(+), 295 deletions(-) create mode 100644 nifi-h2/nifi-h2-database-migrator/pom.xml create mode 100644 nifi-h2/nifi-h2-database-migrator/src/main/java/org/apache/nifi/h2/database/migration/H2DatabaseMigrator.java create mode 100644 nifi-h2/nifi-h2-database-migrator/src/main/java/org/apache/nifi/h2/database/migration/H2DatabaseUpdater.java create mode 100644 nifi-h2/nifi-h2-database-migrator/src/test/java/org/apache/nifi/h2/database/migration/TestH2DatabaseUpdater.java create mode 100644 nifi-h2/nifi-h2-database-migrator/src/test/resources/nifi-flow-audit.mv.db create mode 100644 nifi-h2/nifi-h2-database/pom.xml create mode 100644 nifi-h2/pom.xml delete mode 100644 nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1.2__IncreaseColumnSizes.sql delete mode 100644 nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1.3__DropBucketItemNameUniqueness.sql delete mode 100644 nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1__Initial.sql delete mode 100644 nifi-registry/nifi-registry-core/nifi-registry-framework/src/test/java/org/apache/nifi/registry/db/migration/TestLegacyDatabaseService.java diff --git a/minifi/pom.xml b/minifi/pom.xml index bd2583f118..71415843d8 100644 --- a/minifi/pom.xml +++ b/minifi/pom.xml @@ -825,11 +825,6 @@ limitations under the License. guava 31.0.1-jre - - com.h2database - h2 - 1.4.199 - com.fasterxml.jackson.core jackson-databind diff --git a/nifi-h2/nifi-h2-database-migrator/pom.xml b/nifi-h2/nifi-h2-database-migrator/pom.xml new file mode 100644 index 0000000000..18e34ca1af --- /dev/null +++ b/nifi-h2/nifi-h2-database-migrator/pom.xml @@ -0,0 +1,37 @@ + + + + 4.0.0 + + org.apache.nifi + nifi-h2 + 1.16.0-SNAPSHOT + + nifi-h2-database-migrator + + + + com.h2database + h2 + ${h2.version} + + + org.apache.nifi + nifi-h2-database + 1.16.0-SNAPSHOT + + + diff --git a/nifi-h2/nifi-h2-database-migrator/src/main/java/org/apache/nifi/h2/database/migration/H2DatabaseMigrator.java b/nifi-h2/nifi-h2-database-migrator/src/main/java/org/apache/nifi/h2/database/migration/H2DatabaseMigrator.java new file mode 100644 index 0000000000..e8e255ecee --- /dev/null +++ b/nifi-h2/nifi-h2-database-migrator/src/main/java/org/apache/nifi/h2/database/migration/H2DatabaseMigrator.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.h2.database.migration; + +import org.apache.nifi.org.h2.jdbcx.JdbcDataSource; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.nifi.h2.database.migration.H2DatabaseUpdater.EXPORT_FILE_POSTFIX; +import static org.apache.nifi.h2.database.migration.H2DatabaseUpdater.EXPORT_FILE_PREFIX; + +public class H2DatabaseMigrator { + public static final String BACKUP_FILE_POSTFIX = ".migration_backup"; + + public static void exportAndBackup(final String dbUrl, final String dbPath, final String user, final String pass) { + + // Attempt to connect with the latest driver + JdbcDataSource migrationDataSource = new JdbcDataSource(); + migrationDataSource.setURL(dbUrl); + migrationDataSource.setUser(user); + migrationDataSource.setPassword(pass); + + final File dbPathFile = Paths.get(dbPath).toFile(); + + String exportFile = Paths.get(dbPathFile.getParent(), EXPORT_FILE_PREFIX + dbPathFile.getName() + EXPORT_FILE_POSTFIX).toString(); + + Connection conn; + Statement s; + try { + conn = migrationDataSource.getConnection(); + s = conn.createStatement(); + } catch (SQLException sqle) { + final String message = String.format("H2 1.4 connection failed URL [%s] Path [%s] SQL State [%s]", dbUrl, dbPath, sqle.getSQLState()); + throw new RuntimeException(message, sqle); + } + + try { + // Perform an export of the database to SQL statements + s.execute("SCRIPT TO '" + exportFile + "'"); + } catch (SQLException sqle) { + try { + s.close(); + } catch (SQLException se2) { + // Ignore, the error will be handled + } + final String message = String.format("H2 1.4 export failed URL [%s] Path [%s] SQL State [%s]", dbUrl, dbPath, sqle.getSQLState()); + throw new RuntimeException(message, sqle); + } + + closeQuietly(s); + closeQuietly(conn); + + // Verify the export file exists + if (!Files.exists(Paths.get(exportFile))) { + throw new RuntimeException(String.format("H2 1.4 export failed URL [%s] Path [%s] Export File not found [%s]", dbUrl, dbPath, exportFile)); + } + + // Now that the export file exists, backup (rename) the DB files so the main process with the newer H2 driver can create and import the previous database + File dbDir = new File(dbPath).getParentFile(); + File[] dbFiles = dbDir.listFiles((dir, name) -> !name.endsWith(EXPORT_FILE_POSTFIX) && name.startsWith(dbPathFile.getName())); + if (dbFiles == null || dbFiles.length == 0) { + throw new RuntimeException(String.format("H2 1.4 backup failed URL [%s] Path [%s] no database files found", dbUrl, dbPath)); + } + + for (File dbFile : dbFiles) { + File dbBackupFile = new File(dbFile.getAbsolutePath() + BACKUP_FILE_POSTFIX); + if (!dbFile.renameTo(dbBackupFile)) { + throw new RuntimeException(String.format("H2 1.4 backup failed URL [%s] Path [%s] rename failed [%s]", dbUrl, dbPath, dbFile)); + } + } + } + + private static void closeQuietly(final Statement statement) { + try { + statement.close(); + } catch (final SQLException e) { + // Ignore, nothing to be done + } + } + + private static void closeQuietly(final Connection connection) { + try { + connection.close(); + } catch (final SQLException e) { + // Ignore, nothing to be done + } + } +} diff --git a/nifi-h2/nifi-h2-database-migrator/src/main/java/org/apache/nifi/h2/database/migration/H2DatabaseUpdater.java b/nifi-h2/nifi-h2-database-migrator/src/main/java/org/apache/nifi/h2/database/migration/H2DatabaseUpdater.java new file mode 100644 index 0000000000..8cf07fee56 --- /dev/null +++ b/nifi-h2/nifi-h2-database-migrator/src/main/java/org/apache/nifi/h2/database/migration/H2DatabaseUpdater.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.h2.database.migration; + +import org.h2.jdbc.JdbcSQLNonTransientException; +import org.h2.jdbcx.JdbcDataSource; +import org.h2.mvstore.MVStoreException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +public class H2DatabaseUpdater { + + private static final Logger logger = LoggerFactory.getLogger(H2DatabaseUpdater.class); + + public static final String EXPORT_FILE_PREFIX = "export_"; + public static final String EXPORT_FILE_POSTFIX = ".sql"; + public static final String H2_URL_PREFIX = "jdbc:h2:"; + + public static void checkAndPerformMigration(final String dbPathNoExtension, final String dbUrl, final String user, final String pass) throws Exception { + + final JdbcDataSource migrationDataSource = new JdbcDataSource(); + + // Attempt to connect with the latest driver + migrationDataSource.setURL(dbUrl); + migrationDataSource.setUser(user); + migrationDataSource.setPassword(pass); + try (Connection connection = migrationDataSource.getConnection()) { + return; + } catch (JdbcSQLNonTransientException jsqlnte) { + // Migration/version issues will be caused by an MVStoreException + final Throwable exceptionCause = jsqlnte.getCause(); + if (exceptionCause instanceof MVStoreException) { + // Check for specific error message + final String errorMessage = exceptionCause.getMessage(); + if (!errorMessage.contains("The write format") + && !errorMessage.contains("is smaller than the supported format")) { + throw jsqlnte; + } + } + } catch (SQLException sqle) { + throw new RuntimeException(String.format("H2 connection failed URL [%s] File [%s]", dbUrl, dbPathNoExtension), sqle); + } + // At this point it is known that migration should be attempted + logger.info("H2 1.4 database detected [{}]: starting migration to H2 2.1", dbPathNoExtension); + H2DatabaseMigrator.exportAndBackup(dbUrl, dbPathNoExtension, user, pass); + + // The export file has been created and the DB has been backed up, now create a new one with the same name and run the SQL script to import the database + try (Connection migrationConnection = migrationDataSource.getConnection(); + Statement s = migrationConnection.createStatement()) { + final Path dbFilePath = Paths.get(dbPathNoExtension); + final String dbDirectory = dbFilePath.getParent().toFile().getAbsolutePath(); + // use RUNSCRIPT to recreate the database + final String exportSqlLocation = dbDirectory + File.separator + + H2DatabaseUpdater.EXPORT_FILE_PREFIX + dbFilePath.toFile().getName() + H2DatabaseUpdater.EXPORT_FILE_POSTFIX; + s.execute("RUNSCRIPT FROM '" + exportSqlLocation + "'"); + + } catch (SQLException sqle) { + throw new IOException(String.format("H2 import database creation failed URL [%s]", dbUrl), sqle); + } + + logger.info("H2 1.4 to 2.1 migration completed [{}]", dbPathNoExtension); + } +} diff --git a/nifi-h2/nifi-h2-database-migrator/src/test/java/org/apache/nifi/h2/database/migration/TestH2DatabaseUpdater.java b/nifi-h2/nifi-h2-database-migrator/src/test/java/org/apache/nifi/h2/database/migration/TestH2DatabaseUpdater.java new file mode 100644 index 0000000000..9493394907 --- /dev/null +++ b/nifi-h2/nifi-h2-database-migrator/src/test/java/org/apache/nifi/h2/database/migration/TestH2DatabaseUpdater.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.h2.database.migration; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestH2DatabaseUpdater { + + public static final String DB_NAME = "nifi-flow-audit"; + public static final String MVDB_EXTENSION = ".mv.db"; + public static final String ORIGINAL_AUDIT_DB_PATH = "./src/test/resources/" + DB_NAME; + public static final String ORIGINAL_AUDIT_DB_PATH_FILE = ORIGINAL_AUDIT_DB_PATH + MVDB_EXTENSION; + + @TempDir + File tmpDir; + + @BeforeEach + public void copyDatabaseFile() throws IOException { + // Copy the legacy database file to a temporary directory + final Path origAuditDbPath = Paths.get(ORIGINAL_AUDIT_DB_PATH_FILE); + final Path destAuditDbPath = Paths.get(tmpDir.getAbsolutePath(), DB_NAME + MVDB_EXTENSION); + Files.copy(origAuditDbPath, destAuditDbPath, REPLACE_EXISTING); + } + + @Test + public void testMigration() throws Exception { + final Path testAuditDbPath = Paths.get(tmpDir.getAbsolutePath(), DB_NAME); + final File dbFileNoExtension = testAuditDbPath.toFile(); + final String migrationDbUrl = H2DatabaseUpdater.H2_URL_PREFIX + dbFileNoExtension + ";LOCK_MODE=3"; + + H2DatabaseUpdater.checkAndPerformMigration(dbFileNoExtension.getAbsolutePath(), migrationDbUrl, "nf", "nf"); + + // Verify the export, backup, and new database files were created + final File exportFile = Paths.get(dbFileNoExtension.getParent(), H2DatabaseUpdater.EXPORT_FILE_PREFIX + dbFileNoExtension.getName() + H2DatabaseUpdater.EXPORT_FILE_POSTFIX).toFile(); + assertTrue(exportFile.exists()); + + File dbDir = dbFileNoExtension.getParentFile(); + File[] backupFiles = dbDir.listFiles((dir, name) -> name.endsWith(H2DatabaseMigrator.BACKUP_FILE_POSTFIX) && name.startsWith(dbFileNoExtension.getName())); + try { + assertNotNull(backupFiles); + + // The database and its trace file should exist after the initial connection is made, so they both should be backed up + assertEquals(2, backupFiles.length); + final File newDbFile = Paths.get(tmpDir.getAbsolutePath(), DB_NAME + MVDB_EXTENSION).toFile(); + assertTrue(newDbFile.exists()); + } finally { + // Remove the export and backup files + exportFile.delete(); + for (File f : backupFiles) { + f.delete(); + } + } + } +} \ No newline at end of file diff --git a/nifi-h2/nifi-h2-database-migrator/src/test/resources/nifi-flow-audit.mv.db b/nifi-h2/nifi-h2-database-migrator/src/test/resources/nifi-flow-audit.mv.db new file mode 100644 index 0000000000000000000000000000000000000000..2a3735e0eb4dec671d3ddc3d5fb6329a9b1fe55d GIT binary patch literal 20480 zcmeHP-E!N;6&4+$lgd$QI&pq&XTvnM)UqlT|KJ@(f)dC4D*!su6$b>MM71PpWIB`S zMQ@s!-gc&afWAauq7TuV-gNpB?ZILf1i)WOoMa}mzJLi}7vI_Me0z3rcA=xHR2iL* z{Z~~b{@*$KLs-S4C|3M$-dw(_Vx!`phu&osSN!YHn}k6X>!Bw}qKQ4Jayq`g@FsBc z+wl7J*%*o$mDBTZ;(rrfS4~9^O;J*|xA;HK0B3+Rz!~5Sa0WO7oB_@NXMi)n8Q=_X z27W{aj`;ikkH|iU<_vHKI0Kvk&H!hCGr$?(3~&ZG1DpZQzz@R!UF6s4CIG2&9$rF9 z;KIAA25|hYibt>&;4=Jf0viObysyL8aQAvVo>WyIHVKHz$WW!q8k!{P(sUz$ zj_&dg_6$5ceE83|&=!_tMb&iSZQPZsrWg1cI;R#s@qo0wT1_SR=a_)X_ks*RNnc%5MM_F$FfnY ziMl-(S;sBMbx^t6EF-7ZcF}Xg^HkNuIu=zkR2AbnI;uHGa9uG)B@9FT^wc*EAD9S6 zSPFDa$BG&R%E%MNU?j z-Bzz#-rbHIP;NEKsMU2*!)n$B(4O6{b?l;-wf4Y5&7PgA=bSjM)fqb0?*>-44s^-7 zT(cU@Ay%5|P_gub;ejN}!@AaNSXgU{2bx~qBdm1??KX-IL$?Plh>_TX?g~+14Y9gz z%RMQB@pk>FW=p$@XlhdfXgQ}hu)+GM-AO^%(Xyyqx7M+;B^MTTPx>p1_WG7xBm7U% z)!8S;J{r2Mj^((uPQQ$jp9!Jcd!^zt^u^8vb@!(DM1W|6+*(Ze5Xk<`*y2Svro_q>m=nydUXg_hk60lVnSoj1^rpocBcDRhGd zs`t8%Yu6&7PK7+Z7@?$0*(DDLSvRcXRQaq~rQ#lRmdWzI(zknc%W--(sMf8u+79bB z)IqHGM4=(lH`UluB}9Jv6>xaGHiv4C!#P&bULDCA9{+1#_0NU*w9Zyui(#-K!|;@V zy`@B}h8_r+6JcF*~$CT`4uf{!;4Zh(RPknFnvx>^%H?MU~=8+l2WVR2hxS- z^$ZG|F1FN6qW@1o|6i>gKXq-k17y9ClmDm7<)5j$miQwJZl!*x2=adn_WX506wNDZ zO?zO%u^IJ00riiUi@!gx4{sPlt!`u3faJoW7r5fn8;zi5P5?QJa_;{O=>F3Jy0bOc zA{)SO<$E)7Jpx-64WPHNMc#%s2U0wlNgF*!=)D3hZ?jw(6E zWpwh}AV-rNJ#rk8qfd^(JD1)?xPLw#HN1&;pPh8PtK_#Go<1Vi$5-K{d+lAm_WTKa zD)Tz7I`Ky5;l8*-io_>a&YbD#GYMzT)XbTgex#Z{G2bQ6e{&g(+vBhI@z0_h-oAWA z^M4x4|FHz%`PsK&2hx9HCHmDTi9a!w7L*K@7L@XNLAVp;1$EVr^MaB*%?WCRiE5q+ zVq7&bju>90#1I~d8{xC@f}f=gq=$K`ST)wSWt9!7O-%QNb?+1-)Ptr2BE^Rg^R{ z^s+fwMTxkV0~vcU_c)kxkD0UC7hg~)&h{>Lwjbw!A=Mk_fU%zgjGZl>1`IHq0!j=} zrs{&)>j6s84hk6N0OPE)tg94I5~qNPoawi0RUEDajI-QRKzLp@%^fi+Dv$tXfT?fY z6xKqO^&jXjB!Rw|(iuQ02S|JfspSMPl~Y&G;ttDPC=)nLI_doGsu2%Dcb%KWC+-tf z-w$KGOuMQ?k(lq4l}I{4(wXPfY_o{p#8Zu}7I=x<Cu79t`I~q8znMltyO2(2NAlzl_lW ziUxI81;iP+TtJj#w}oQTI5U9@C|tsTS|K7jvd8rq-_;rlGxRDpPH#5S?hFJ`%PrbTf2zndX(ZS!|=pVVbOFSY0pS->G zx_tc~_NlCun&RvKsr7ii{?FI{QyUbrNnyVJzky97eEq-O=Ij4#L*;EYBk=WqzW$%v qm&@1x={|uatLwDH(uMV((E9((`}f?xqo=Fzn(g`PzB{j9Zu|eHHw<|I literal 0 HcmV?d00001 diff --git a/nifi-h2/nifi-h2-database/pom.xml b/nifi-h2/nifi-h2-database/pom.xml new file mode 100644 index 0000000000..feb24b0834 --- /dev/null +++ b/nifi-h2/nifi-h2-database/pom.xml @@ -0,0 +1,70 @@ + + + + 4.0.0 + + org.apache.nifi + nifi-h2 + 1.16.0-SNAPSHOT + + + nifi-h2-database + 1.16.0-SNAPSHOT + jar + + + + com.h2database + h2 + 1.4.200 + true + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + package + + shade + + + false + + + com.h2database:h2 + + org/h2/fulltext/** + org/h2/jdbc/JdbcSQLXML.class + org/h2/jmx/** + org/h2/server/** + + + + + + org.h2 + org.apache.nifi.org.h2 + + + + + + + + + \ No newline at end of file diff --git a/nifi-h2/pom.xml b/nifi-h2/pom.xml new file mode 100644 index 0000000000..75886270bc --- /dev/null +++ b/nifi-h2/pom.xml @@ -0,0 +1,30 @@ + + + + 4.0.0 + + org.apache.nifi + nifi + 1.16.0-SNAPSHOT + + + nifi-h2 + pom + + nifi-h2-database + nifi-h2-database-migrator + + diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-database-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/nifi-database-utils/pom.xml index b674d4c511..e037fb8583 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-database-utils/pom.xml +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-database-utils/pom.xml @@ -85,7 +85,7 @@ com.h2database h2 - 1.4.187 + ${h2.version} test diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-database-utils/src/test/java/org/apache/nifi/util/db/TestJdbcTypesH2.java b/nifi-nar-bundles/nifi-extension-utils/nifi-database-utils/src/test/java/org/apache/nifi/util/db/TestJdbcTypesH2.java index 5f594df8e6..63ca6c8e9e 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-database-utils/src/test/java/org/apache/nifi/util/db/TestJdbcTypesH2.java +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-database-utils/src/test/java/org/apache/nifi/util/db/TestJdbcTypesH2.java @@ -48,25 +48,25 @@ public class TestJdbcTypesH2 { } String createTable = " CREATE TABLE `users` ( " - + " `id` int(11) NOT NULL AUTO_INCREMENT, " + + " `id` int NOT NULL AUTO_INCREMENT, " + " `email` varchar(255) NOT NULL, " + " `password` varchar(255) DEFAULT NULL, " + " `activation_code` varchar(255) DEFAULT NULL, " + " `forgotten_password_code` varchar(255) DEFAULT NULL, " + " `forgotten_password_time` datetime DEFAULT NULL, " + " `created` datetime NOT NULL, " - + " `active` tinyint(1) NOT NULL DEFAULT '0', " - + " `home_module_id` int(11) DEFAULT NULL, " + + " `active` tinyint NOT NULL DEFAULT '0', " + + " `home_module_id` int DEFAULT NULL, " - + " somebinary BINARY default null, " + + " somebinary BINARY(4) default null, " + " somebinary2 VARBINARY default null, " + " somebinary3 LONGVARBINARY default null, " - + " somearray ARRAY default null, " + + " somearray INTEGER ARRAY default null, " + " someblob BLOB default null, " + " someclob CLOB default null, " + " PRIMARY KEY (`id`), " - + " UNIQUE KEY `email` (`email`) ) " ; + + " CONSTRAINT unique_email UNIQUE (`email`) ) " ; // + " KEY `home_module_id` (`home_module_id`) )" ; /* + " CONSTRAINT `users_ibfk_1` FOREIGN KEY (`home_module_id`) REFERENCES " + "`modules` (`id`) ON DELETE SET NULL " @@ -92,7 +92,7 @@ public class TestJdbcTypesH2 { // + " values ('robert.gates@cold.com', '******', 'CAS', 'ounou', '2005-12-09', '2005-12-03', 1, 5)"); st.executeUpdate("insert into users (email, password, activation_code, created, active, somebinary, somebinary2, somebinary3, someblob, someclob) " - + " values ('mari.gates@cold.com', '******', 'CAS', '2005-12-03', 3, '66FF', 'ABDF', 'EE64', 'BB22', 'CC88')"); + + " values ('mari.gates@cold.com', '******', 'CAS', '2005-12-03', 3, 0x66FF, 'ABDF', 'EE64', 'BB22', 'CC88')"); final ResultSet resultSet = st.executeQuery("select U.*, ROW_NUMBER() OVER () as rownr from users U"); // final ResultSet resultSet = st.executeQuery("select U.active from users U"); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml index e7e95c8ea9..1002e3185e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/pom.xml @@ -58,6 +58,11 @@ com.h2database h2 + + org.apache.nifi + nifi-h2-database-migrator + 1.16.0-SNAPSHOT + org.springframework spring-beans diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java index 18df4bc572..11d7c30e64 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/AuditDataSourceFactoryBean.java @@ -17,6 +17,7 @@ package org.apache.nifi.admin; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.h2.database.migration.H2DatabaseUpdater; import org.apache.nifi.util.NiFiProperties; import org.h2.jdbcx.JdbcConnectionPool; import org.slf4j.Logger; @@ -84,7 +85,7 @@ public class AuditDataSourceFactoryBean implements FactoryBean { private static final String CREATE_CONFIGURE_DETAILS_TABLE = "CREATE TABLE CONFIGURE_DETAILS (" + "ACTION_ID INT NOT NULL PRIMARY KEY, " + "NAME VARCHAR2(1000) NOT NULL, " - + "VALUE VARCHAR2(5000), " + + "\"VALUE\" VARCHAR2(5000), " + "PREVIOUS_VALUE VARCHAR2(5000), " + "FOREIGN KEY (ACTION_ID) REFERENCES ACTION(ID)" + ")"; @@ -127,15 +128,19 @@ public class AuditDataSourceFactoryBean implements FactoryBean { File repositoryDirectory = new File(repositoryDirectoryPath); // get a handle to the database file - File databaseFile = new File(repositoryDirectory, AUDIT_DATABASE_FILE_NAME); + File dbFileNoExtension = new File(repositoryDirectory, AUDIT_DATABASE_FILE_NAME); // format the database url - String databaseUrl = "jdbc:h2:" + databaseFile + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3"; + String databaseUrl = H2DatabaseUpdater.H2_URL_PREFIX + dbFileNoExtension + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3"; String databaseUrlAppend = properties.getProperty(NiFiProperties.H2_URL_APPEND); if (StringUtils.isNotBlank(databaseUrlAppend)) { databaseUrl += databaseUrlAppend; } + // Migrate an existing database if required + final String migrationDbUrl = H2DatabaseUpdater.H2_URL_PREFIX + dbFileNoExtension + ";LOCK_MODE=3"; + H2DatabaseUpdater.checkAndPerformMigration(dbFileNoExtension.getAbsolutePath(), migrationDbUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD); + // create the pool connectionPool = JdbcConnectionPool.create(databaseUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD); connectionPool.setMaxConnections(MAX_CONNECTIONS); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/IdpDataSourceFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/IdpDataSourceFactoryBean.java index d0078e2c46..72c1e003c6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/IdpDataSourceFactoryBean.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-administration/src/main/java/org/apache/nifi/admin/IdpDataSourceFactoryBean.java @@ -17,6 +17,7 @@ package org.apache.nifi.admin; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.h2.database.migration.H2DatabaseUpdater; import org.apache.nifi.util.NiFiProperties; import org.h2.jdbcx.JdbcConnectionPool; import org.slf4j.Logger; @@ -84,8 +85,12 @@ public class IdpDataSourceFactoryBean implements FactoryBean File repositoryDirectory = new File(repositoryDirectoryPath); // create a handle to the database directory and file - File databaseFile = new File(repositoryDirectory, IDP_DATABASE_FILE_NAME); - String databaseUrl = getDatabaseUrl(databaseFile); + File dbFileNoExtension = new File(repositoryDirectory, IDP_DATABASE_FILE_NAME); + String databaseUrl = getDatabaseUrl(dbFileNoExtension); + + // Migrate an existing database if required + final String migrationDbUrl = H2DatabaseUpdater.H2_URL_PREFIX + dbFileNoExtension + ";LOCK_MODE=3"; + H2DatabaseUpdater.checkAndPerformMigration(dbFileNoExtension.getAbsolutePath(), migrationDbUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD); // create the pool connectionPool = JdbcConnectionPool.create(databaseUrl, NF_USERNAME_PASSWORD, NF_USERNAME_PASSWORD); @@ -125,7 +130,7 @@ public class IdpDataSourceFactoryBean implements FactoryBean } private String getDatabaseUrl(File databaseFile) { - String databaseUrl = "jdbc:h2:" + databaseFile + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3"; + String databaseUrl = H2DatabaseUpdater.H2_URL_PREFIX + databaseFile + ";AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3"; String databaseUrlAppend = properties.getProperty(NiFiProperties.H2_URL_APPEND); if (StringUtils.isNotBlank(databaseUrlAppend)) { databaseUrl += databaseUrlAppend; diff --git a/nifi-nar-bundles/nifi-framework-bundle/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/pom.xml index 9c67585b25..86f483b8fa 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/pom.xml @@ -307,7 +307,7 @@ com.h2database h2 - 1.4.199 + ${h2.version} diff --git a/nifi-nar-bundles/nifi-standard-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/pom.xml index 9bc6865292..4c99b8f5e7 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/pom.xml @@ -268,7 +268,7 @@ com.h2database h2 - 1.4.187 + ${h2.version} test diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/pom.xml index fffcd56c8e..61ac0e59bd 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/pom.xml @@ -121,7 +121,7 @@ com.h2database h2 - 1.4.192 + ${h2.version} test diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/pom.xml b/nifi-registry/nifi-registry-core/nifi-registry-framework/pom.xml index d9cd76c11a..0f4fbf7793 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/pom.xml +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/pom.xml @@ -206,6 +206,11 @@ nifi-registry-flow-diff 1.16.0-SNAPSHOT + + org.apache.nifi + nifi-h2-database-migrator + 1.16.0-SNAPSHOT + javax.servlet javax.servlet-api @@ -307,6 +312,11 @@ flyway-core ${flyway.version} + + org.flywaydb + flyway-mysql + ${flyway.version} + com.h2database h2 diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/CustomFlywayConfiguration.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/CustomFlywayConfiguration.java index 7f9ef7a777..4eb3812d77 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/CustomFlywayConfiguration.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/CustomFlywayConfiguration.java @@ -18,8 +18,11 @@ package org.apache.nifi.registry.db; import org.flywaydb.core.api.FlywayException; import org.flywaydb.core.api.configuration.FluentConfiguration; -import org.flywaydb.core.internal.jdbc.DatabaseType; +import org.flywaydb.core.internal.database.DatabaseType; +import org.flywaydb.core.internal.database.DatabaseTypeRegister; +import org.flywaydb.core.internal.database.postgresql.PostgreSQLDatabaseType; import org.flywaydb.core.internal.jdbc.JdbcUtils; +import org.flywaydb.database.mysql.MySQLDatabaseType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.flyway.FlywayConfigurationCustomizer; @@ -52,21 +55,17 @@ public class CustomFlywayConfiguration implements FlywayConfigurationCustomizer @Override public void customize(final FluentConfiguration configuration) { final DatabaseType databaseType = getDatabaseType(configuration.getDataSource()); - LOGGER.info("Determined database type is {}", new Object[] {databaseType.name()}); + LOGGER.info("Determined database type is {}", databaseType.getName()); - switch (databaseType) { - case MYSQL: - LOGGER.info("Setting migration locations to {}", new Object[] {LOCATIONS_MYSQL}); - configuration.locations(LOCATIONS_MYSQL); - break; - case POSTGRESQL: - LOGGER.info("Setting migration locations to {}", new Object[] {LOCATIONS_POSTGRES}); - configuration.locations(LOCATIONS_POSTGRES); - break; - default: - LOGGER.info("Setting migration locations to {}", new Object[] {LOCATIONS_DEFAULT}); - configuration.locations(LOCATIONS_DEFAULT); - break; + if (databaseType.equals(new MySQLDatabaseType())) { + LOGGER.info("Setting migration locations to {}", LOCATIONS_MYSQL); + configuration.locations(LOCATIONS_MYSQL); + } else if (databaseType.equals(new PostgreSQLDatabaseType())) { + LOGGER.info("Setting migration locations to {}", LOCATIONS_POSTGRES); + configuration.locations(LOCATIONS_POSTGRES); + } else { + LOGGER.info("Setting migration locations to {}", LOCATIONS_DEFAULT); + configuration.locations(LOCATIONS_DEFAULT); } // At some point Flyway changed their default table name: https://github.com/flyway/flyway/issues/1848 @@ -88,7 +87,7 @@ public class CustomFlywayConfiguration implements FlywayConfigurationCustomizer */ private DatabaseType getDatabaseType(final DataSource dataSource) { try (final Connection connection = dataSource.getConnection()) { - return DatabaseType.fromJdbcConnection(connection); + return DatabaseTypeRegister.getDatabaseTypeForConnection(connection); } catch (SQLException e) { LOGGER.error(e.getMessage(), e); throw new FlywayException("Unable to obtain connection from Flyway DataSource", e); diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/DataSourceFactory.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/DataSourceFactory.java index 29c132e293..41135e7c13 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/DataSourceFactory.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/java/org/apache/nifi/registry/db/DataSourceFactory.java @@ -18,6 +18,7 @@ package org.apache.nifi.registry.db; import com.zaxxer.hikari.HikariDataSource; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.h2.database.migration.H2DatabaseUpdater; import org.apache.nifi.registry.properties.NiFiRegistryProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,6 +29,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; import javax.sql.DataSource; +import java.io.File; /** * Overriding Spring Boot's normal automatic creation of a DataSource in order to use the properties @@ -87,11 +89,27 @@ public class DataSourceFactory { .build(); if (dataSource instanceof HikariDataSource) { - LOGGER.info("Setting maximum pool size on HikariDataSource to {}", new Object[]{properties.getDatabaseMaxConnections()}); - ((HikariDataSource)dataSource).setMaximumPoolSize(properties.getDatabaseMaxConnections()); + LOGGER.info("Setting maximum pool size on HikariDataSource to {}", properties.getDatabaseMaxConnections()); + ((HikariDataSource) dataSource).setMaximumPoolSize(properties.getDatabaseMaxConnections()); + } + + // Migrate an H2 existing database if required + if (databaseUrl.startsWith(H2DatabaseUpdater.H2_URL_PREFIX)) { + final File databaseFile = getFileFromH2URL(databaseUrl); + final String migrationDbUrl = H2DatabaseUpdater.H2_URL_PREFIX + databaseFile + ";LOCK_MODE=3"; + + try { + H2DatabaseUpdater.checkAndPerformMigration(databaseFile.getAbsolutePath(), migrationDbUrl, databaseUsername, databasePassword); + } catch (Exception e) { + throw new RuntimeException("H2 database migration failed", e); + } } return dataSource; } + private File getFileFromH2URL(final String databaseUrl) { + final String dbPath = databaseUrl.substring(H2DatabaseUpdater.H2_URL_PREFIX.length(), databaseUrl.indexOf(";")); + return new File(dbPath); + } } diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1.2__IncreaseColumnSizes.sql b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1.2__IncreaseColumnSizes.sql deleted file mode 100644 index b2e92d5fe8..0000000000 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1.2__IncreaseColumnSizes.sql +++ /dev/null @@ -1,25 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -ALTER TABLE BUCKET ALTER COLUMN NAME VARCHAR2(1000); -ALTER TABLE BUCKET ALTER COLUMN DESCRIPTION VARCHAR2(65535); - -ALTER TABLE BUCKET_ITEM ALTER COLUMN NAME VARCHAR2(1000); -ALTER TABLE BUCKET_ITEM ALTER COLUMN DESCRIPTION VARCHAR2(65535); - -ALTER TABLE FLOW_SNAPSHOT ALTER COLUMN CREATED_BY VARCHAR2(4096); -ALTER TABLE FLOW_SNAPSHOT ALTER COLUMN COMMENTS VARCHAR2(65535); - -ALTER TABLE SIGNING_KEY ALTER COLUMN TENANT_IDENTITY VARCHAR2(4096); \ No newline at end of file diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1.3__DropBucketItemNameUniqueness.sql b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1.3__DropBucketItemNameUniqueness.sql deleted file mode 100644 index f29b4d0b98..0000000000 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1.3__DropBucketItemNameUniqueness.sql +++ /dev/null @@ -1,27 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -CREATE ALIAS IF NOT EXISTS EXECUTE AS $$ void executeSql(Connection conn, String sql) -throws SQLException { conn.createStatement().executeUpdate(sql); } $$; - -call execute('ALTER TABLE BUCKET_ITEM DROP CONSTRAINT ' || - ( - SELECT DISTINCT CONSTRAINT_NAME - FROM INFORMATION_SCHEMA.CONSTRAINTS - WHERE TABLE_NAME = 'BUCKET_ITEM' - AND COLUMN_LIST = 'NAME' - AND CONSTRAINT_TYPE = 'UNIQUE' - ) -); diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1__Initial.sql b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1__Initial.sql deleted file mode 100644 index a6b49602cf..0000000000 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/main/resources/db/migration/original/V1__Initial.sql +++ /dev/null @@ -1,54 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -CREATE TABLE BUCKET ( - ID VARCHAR2(50) NOT NULL PRIMARY KEY, - NAME VARCHAR2(200) NOT NULL UNIQUE, - DESCRIPTION VARCHAR(4096), - CREATED TIMESTAMP NOT NULL -); - -CREATE TABLE BUCKET_ITEM ( - ID VARCHAR2(50) NOT NULL PRIMARY KEY, - NAME VARCHAR2(200) NOT NULL UNIQUE, - DESCRIPTION VARCHAR(4096), - CREATED TIMESTAMP NOT NULL, - MODIFIED TIMESTAMP NOT NULL, - ITEM_TYPE VARCHAR(50) NOT NULL, - BUCKET_ID VARCHAR2(50) NOT NULL, - FOREIGN KEY (BUCKET_ID) REFERENCES BUCKET(ID) -); - -CREATE TABLE FLOW ( - ID VARCHAR2(50) NOT NULL PRIMARY KEY, - FOREIGN KEY (ID) REFERENCES BUCKET_ITEM(ID) -); - -CREATE TABLE FLOW_SNAPSHOT ( - FLOW_ID VARCHAR2(50) NOT NULL, - VERSION INT NOT NULL, - CREATED TIMESTAMP NOT NULL, - CREATED_BY VARCHAR2(200) NOT NULL, - COMMENTS VARCHAR(4096), - PRIMARY KEY (FLOW_ID, VERSION), - FOREIGN KEY (FLOW_ID) REFERENCES FLOW(ID) -); - -CREATE TABLE SIGNING_KEY ( - ID VARCHAR2(50) NOT NULL, - TENANT_IDENTITY VARCHAR2(50) NOT NULL UNIQUE, - KEY_VALUE VARCHAR2(50) NOT NULL, - PRIMARY KEY (ID) -); \ No newline at end of file diff --git a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/test/java/org/apache/nifi/registry/db/migration/TestLegacyDatabaseService.java b/nifi-registry/nifi-registry-core/nifi-registry-framework/src/test/java/org/apache/nifi/registry/db/migration/TestLegacyDatabaseService.java deleted file mode 100644 index 6940331913..0000000000 --- a/nifi-registry/nifi-registry-core/nifi-registry-framework/src/test/java/org/apache/nifi/registry/db/migration/TestLegacyDatabaseService.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.registry.db.migration; - -import org.apache.nifi.registry.db.entity.BucketItemEntityType; -import org.flywaydb.core.Flyway; -import org.junit.Before; -import org.junit.Test; -import org.springframework.boot.jdbc.DataSourceBuilder; -import org.springframework.jdbc.core.JdbcTemplate; - -import javax.sql.DataSource; -import java.util.Date; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * Test running the legacy Flyway migrations against an in-memory H2 and then using the LegacyDatabaseService to - * retrieve data. Purposely not using Spring test annotations here to avoid interfering with the normal DB context/flyway. - */ -public class TestLegacyDatabaseService { - - private DataSource dataSource; - private JdbcTemplate jdbcTemplate; - private Flyway flyway; - - private BucketEntityV1 bucketEntityV1; - private FlowEntityV1 flowEntityV1; - private FlowSnapshotEntityV1 flowSnapshotEntityV1; - - @Before - public void setup() { - dataSource = DataSourceBuilder.create() - .url("jdbc:h2:mem:legacydb") - .driverClassName("org.h2.Driver") - .build(); - - jdbcTemplate = new JdbcTemplate(dataSource); - - flyway = Flyway.configure() - .dataSource(dataSource) - .locations("db/migration/original") - .load(); - - flyway.migrate(); - - bucketEntityV1 = new BucketEntityV1(); - bucketEntityV1.setId("1"); - bucketEntityV1.setName("Bucket1"); - bucketEntityV1.setDescription("This is bucket 1"); - bucketEntityV1.setCreated(new Date()); - - jdbcTemplate.update("INSERT INTO bucket (ID, NAME, DESCRIPTION, CREATED) VALUES (?, ?, ?, ?)", - bucketEntityV1.getId(), - bucketEntityV1.getName(), - bucketEntityV1.getDescription(), - bucketEntityV1.getCreated()); - - flowEntityV1 = new FlowEntityV1(); - flowEntityV1.setId("1"); - flowEntityV1.setBucketId(bucketEntityV1.getId()); - flowEntityV1.setName("Flow1"); - flowEntityV1.setDescription("This is flow1"); - flowEntityV1.setCreated(new Date()); - flowEntityV1.setModified(new Date()); - - jdbcTemplate.update("INSERT INTO bucket_item (ID, NAME, DESCRIPTION, CREATED, MODIFIED, ITEM_TYPE, BUCKET_ID) VALUES (?, ?, ?, ?, ?, ?, ?)", - flowEntityV1.getId(), - flowEntityV1.getName(), - flowEntityV1.getDescription(), - flowEntityV1.getCreated(), - flowEntityV1.getModified(), - BucketItemEntityType.FLOW.toString(), - flowEntityV1.getBucketId()); - - jdbcTemplate.update("INSERT INTO flow (ID) VALUES (?)", flowEntityV1.getId()); - - flowSnapshotEntityV1 = new FlowSnapshotEntityV1(); - flowSnapshotEntityV1.setFlowId(flowEntityV1.getId()); - flowSnapshotEntityV1.setVersion(1); - flowSnapshotEntityV1.setComments("This is v1"); - flowSnapshotEntityV1.setCreated(new Date()); - flowSnapshotEntityV1.setCreatedBy("user1"); - - jdbcTemplate.update("INSERT INTO flow_snapshot (FLOW_ID, VERSION, CREATED, CREATED_BY, COMMENTS) VALUES (?, ?, ?, ?, ?)", - flowSnapshotEntityV1.getFlowId(), - flowSnapshotEntityV1.getVersion(), - flowSnapshotEntityV1.getCreated(), - flowSnapshotEntityV1.getCreatedBy(), - flowSnapshotEntityV1.getComments()); - } - - @Test - public void testGetLegacyData() { - final LegacyDatabaseService service = new LegacyDatabaseService(dataSource); - - final List buckets = service.getAllBuckets(); - assertEquals(1, buckets.size()); - - final BucketEntityV1 b = buckets.stream().findFirst().get(); - assertEquals(bucketEntityV1.getId(), b.getId()); - assertEquals(bucketEntityV1.getName(), b.getName()); - assertEquals(bucketEntityV1.getDescription(), b.getDescription()); - assertEquals(bucketEntityV1.getCreated(), b.getCreated()); - - final List flows = service.getAllFlows(); - assertEquals(1, flows.size()); - - final FlowEntityV1 f = flows.stream().findFirst().get(); - assertEquals(flowEntityV1.getId(), f.getId()); - assertEquals(flowEntityV1.getName(), f.getName()); - assertEquals(flowEntityV1.getDescription(), f.getDescription()); - assertEquals(flowEntityV1.getCreated(), f.getCreated()); - assertEquals(flowEntityV1.getModified(), f.getModified()); - assertEquals(flowEntityV1.getBucketId(), f.getBucketId()); - - final List flowSnapshots = service.getAllFlowSnapshots(); - assertEquals(1, flowSnapshots.size()); - - final FlowSnapshotEntityV1 fs = flowSnapshots.stream().findFirst().get(); - assertEquals(flowSnapshotEntityV1.getFlowId(), fs.getFlowId()); - assertEquals(flowSnapshotEntityV1.getVersion(), fs.getVersion()); - assertEquals(flowSnapshotEntityV1.getComments(), fs.getComments()); - assertEquals(flowSnapshotEntityV1.getCreatedBy(), fs.getCreatedBy()); - assertEquals(flowSnapshotEntityV1.getCreated(), fs.getCreated()); - } - -} diff --git a/nifi-registry/nifi-registry-core/nifi-registry-revision/nifi-registry-revision-spring-jdbc/pom.xml b/nifi-registry/nifi-registry-core/nifi-registry-revision/nifi-registry-revision-spring-jdbc/pom.xml index 9d26641fa8..a41ee368e7 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-revision/nifi-registry-revision-spring-jdbc/pom.xml +++ b/nifi-registry/nifi-registry-core/nifi-registry-revision/nifi-registry-revision-spring-jdbc/pom.xml @@ -56,5 +56,11 @@ ${flyway.version} test + + org.flywaydb + flyway-mysql + ${flyway.version} + test + diff --git a/nifi-registry/nifi-registry-core/nifi-registry-revision/nifi-registry-revision-spring-jdbc/src/test/java/org/apache/nifi/registry/revision/jdbc/TestJdbcRevisionManager.java b/nifi-registry/nifi-registry-core/nifi-registry-revision/nifi-registry-revision-spring-jdbc/src/test/java/org/apache/nifi/registry/revision/jdbc/TestJdbcRevisionManager.java index cbb7e463ba..e1c2b32dde 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-revision/nifi-registry-revision-spring-jdbc/src/test/java/org/apache/nifi/registry/revision/jdbc/TestJdbcRevisionManager.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-revision/nifi-registry-revision-spring-jdbc/src/test/java/org/apache/nifi/registry/revision/jdbc/TestJdbcRevisionManager.java @@ -27,7 +27,9 @@ import org.apache.nifi.registry.revision.api.RevisionUpdate; import org.apache.nifi.registry.revision.api.UpdateRevisionTask; import org.apache.nifi.registry.revision.standard.StandardRevisionClaim; import org.apache.nifi.registry.revision.standard.StandardUpdateResult; -import org.flywaydb.core.internal.jdbc.DatabaseType; +import org.flywaydb.core.internal.database.DatabaseType; +import org.flywaydb.core.internal.database.DatabaseTypeRegister; +import org.flywaydb.database.mysql.MySQLDatabaseType; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -97,8 +99,8 @@ public class TestJdbcRevisionManager { try (final Connection connection = dataSource.getConnection()) { final String createTableSql; - final DatabaseType databaseType = DatabaseType.fromJdbcConnection(connection); - if (databaseType == DatabaseType.MYSQL) { + final DatabaseType databaseType = DatabaseTypeRegister.getDatabaseTypeForConnection(connection); + if (databaseType.equals(new MySQLDatabaseType())) { createTableSql = CREATE_TABLE_SQL_MYSQL; } else { createTableSql = CREATE_TABLE_SQL_DEFAULT; diff --git a/nifi-registry/pom.xml b/nifi-registry/pom.xml index e3e01e7c00..3956a41790 100644 --- a/nifi-registry/pom.xml +++ b/nifi-registry/pom.xml @@ -38,11 +38,10 @@ 2.1 2.6.3 - 6.5.7 - 6.4.0 + 8.4.2 + 7.0.0 3.12.0 1.16.0 - 1.4.199 3.4.0-01 2.3.2 5.13.0.202109080827-r diff --git a/pom.xml b/pom.xml index 5758d54810..6e181bc682 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,7 @@ nifi-toolkit nifi-manifest c2 + nifi-h2 https://nifi.apache.org @@ -123,6 +124,7 @@ 4.1.73.Final 5.3.15 5.6.1 + 2.1.210