From a48af2f120bbc16d15eef758c325cc18eeb0dedc Mon Sep 17 00:00:00 2001 From: Gerhard Schlager Date: Tue, 19 Nov 2024 23:54:00 +0100 Subject: [PATCH] DEV: Store details log entries in converter as JSON (#29778) Plus small DB related fixes --- .../002-log_entries.sql | 2 +- migrations/lib/database.rb | 26 +++++++++++-------- migrations/lib/database/connection.rb | 9 +++---- migrations/lib/database/intermediate_db.rb | 2 -- .../lib/database/intermediate_db/log_entry.rb | 2 +- migrations/lib/database/migrator.rb | 2 ++ .../lib/database/prepared_statement_cache.rb | 2 ++ migrations/spec/lib/database_spec.rb | 18 +++++++++++++ 8 files changed, 42 insertions(+), 21 deletions(-) diff --git a/migrations/db/intermediate_db_schema/002-log_entries.sql b/migrations/db/intermediate_db_schema/002-log_entries.sql index 7ece018d6a0..a1e50dab1ea 100644 --- a/migrations/db/intermediate_db_schema/002-log_entries.sql +++ b/migrations/db/intermediate_db_schema/002-log_entries.sql @@ -4,5 +4,5 @@ CREATE TABLE log_entries type TEXT NOT NULL, message TEXT NOT NULL, exception TEXT, - details TEXT + details JSON_TEXT ); diff --git a/migrations/lib/database.rb b/migrations/lib/database.rb index aad2d9324dc..17665e33e96 100644 --- a/migrations/lib/database.rb +++ b/migrations/lib/database.rb @@ -3,23 +3,22 @@ require "date" require "extralite" require "ipaddr" +require "oj" module Migrations module Database INTERMEDIATE_DB_SCHEMA_PATH = File.join(::Migrations.root_path, "db", "intermediate_db_schema") UPLOADS_DB_SCHEMA_PATH = File.join(::Migrations.root_path, "db", "uploads_db_schema") - module_function - - def migrate(db_path, migrations_path:) + def self.migrate(db_path, migrations_path:) Migrator.new(db_path).migrate(migrations_path) end - def reset!(db_path) + def self.reset!(db_path) Migrator.new(db_path).reset! end - def connect(path) + def self.connect(path) connection = Connection.new(path:) return connection unless block_given? @@ -31,29 +30,34 @@ module Migrations nil end - def format_datetime(value) + def self.format_datetime(value) value&.utc&.iso8601 end - def format_date(value) + def self.format_date(value) value&.to_date&.iso8601 end - def format_boolean(value) + def self.format_boolean(value) return nil if value.nil? value ? 1 : 0 end - def format_ip_address(value) + def self.format_ip_address(value) return nil if value.blank? IPAddr.new(value).to_s rescue ArgumentError nil end - def to_blob(value) + def self.to_blob(value) return nil if value.blank? - Extralite::Blob.new(value) + ::Extralite::Blob.new(value) + end + + def self.to_json(value) + return nil if value.nil? + ::Oj.dump(value, mode: :compat) end end end diff --git a/migrations/lib/database/connection.rb b/migrations/lib/database/connection.rb index ca31f7120d6..189d3f26b4c 100644 --- a/migrations/lib/database/connection.rb +++ b/migrations/lib/database/connection.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "extralite" -require "lru_redux" module Migrations::Database class Connection @@ -11,7 +10,7 @@ module Migrations::Database def self.open_database(path:) FileUtils.mkdir_p(File.dirname(path)) - db = Extralite::Database.new(path) + db = ::Extralite::Database.new(path) db.pragma( busy_timeout: 60_000, # 60 seconds journal_mode: "wal", @@ -30,8 +29,6 @@ module Migrations::Database @transaction_batch_size = transaction_batch_size @db = self.class.open_database(path:) @statement_counter = 0 - - # don't cache too many prepared statements @statement_cache = PreparedStatementCache.new(PREPARED_STATEMENT_CACHE_SIZE) @fork_hooks = setup_fork_handling @@ -46,7 +43,7 @@ module Migrations::Database end def closed? - !@db || @db.closed? + @db.nil? || @db.closed? end def insert(sql, parameters = []) @@ -76,7 +73,7 @@ module Migrations::Database end def close_connection(keep_path:) - return if !@db + return if @db.nil? commit_transaction @statement_cache.clear diff --git a/migrations/lib/database/intermediate_db.rb b/migrations/lib/database/intermediate_db.rb index 9c88cefe06e..cc2642a7c56 100644 --- a/migrations/lib/database/intermediate_db.rb +++ b/migrations/lib/database/intermediate_db.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require "singleton" - module Migrations::Database module IntermediateDB def self.setup(db_connection) diff --git a/migrations/lib/database/intermediate_db/log_entry.rb b/migrations/lib/database/intermediate_db/log_entry.rb index 733272745c3..00f568d8e8e 100644 --- a/migrations/lib/database/intermediate_db/log_entry.rb +++ b/migrations/lib/database/intermediate_db/log_entry.rb @@ -18,7 +18,7 @@ module Migrations::Database::IntermediateDB type, message, exception&.full_message(highlight: false), - details, + ::Migrations::Database.to_json(details), ) end end diff --git a/migrations/lib/database/migrator.rb b/migrations/lib/database/migrator.rb index abb22057685..4ede9e5be53 100644 --- a/migrations/lib/database/migrator.rb +++ b/migrations/lib/database/migrator.rb @@ -21,12 +21,14 @@ module Migrations::Database migrate_from_path(@migrations_path, performed_migrations) @db.close + nil end def reset! [@db_path, "#{@db_path}-wal", "#{@db_path}-shm"].each do |path| FileUtils.remove_file(path, force: true) if File.exist?(path) end + nil end private diff --git a/migrations/lib/database/prepared_statement_cache.rb b/migrations/lib/database/prepared_statement_cache.rb index 1de4a4e6d37..38d59469803 100644 --- a/migrations/lib/database/prepared_statement_cache.rb +++ b/migrations/lib/database/prepared_statement_cache.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "lru_redux" + module Migrations::Database class PreparedStatementCache < LruRedux::Cache class PreparedStatementHash < Hash diff --git a/migrations/spec/lib/database_spec.rb b/migrations/spec/lib/database_spec.rb index e3fec57d456..63954ea6fe9 100644 --- a/migrations/spec/lib/database_spec.rb +++ b/migrations/spec/lib/database_spec.rb @@ -140,4 +140,22 @@ RSpec.describe ::Migrations::Database do expect(described_class.to_blob(nil)).to be_nil end end + + describe ".to_json" do + it "returns a JSON string for objects" do + expect(described_class.to_json(123)).to eq("123") + expect(described_class.to_json("hello world")).to eq(%q|"hello world"|) + expect( + described_class.to_json( + text: "foo", + number: 123, + date: DateTime.new(2023, 10, 5, 17, 30, 0), + ), + ).to eq(%q|{"text":"foo","number":123,"date":"2023-10-05T17:30:00.000+00:00"}|) + end + + it "returns nil for nil input" do + expect(described_class.to_json(nil)).to be_nil + end + end end