diff --git a/config/site_settings.yml b/config/site_settings.yml index 7e31adf9ce3..6b9a16cecc7 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -1481,6 +1481,11 @@ developer: enable_safe_mode: default: true client: true + refresh_maxmind_db_during_precompile_days: + min: 0 + max: 120 + default: 2 + hidden: true embedding: feed_polling_enabled: diff --git a/lib/discourse_ip_info.rb b/lib/discourse_ip_info.rb index 343312c6968..70819ad23cd 100644 --- a/lib/discourse_ip_info.rb +++ b/lib/discourse_ip_info.rb @@ -16,6 +16,36 @@ class DiscourseIpInfo @cache = LruRedux::ThreadSafeCache.new(2000) end + def self.mmdb_path(name) + File.join(Rails.root, 'vendor', 'data', "#{name}.mmdb") + end + + def self.mmdb_download(name) + require 'rubygems/package' + require 'zlib' + + uri = URI("https://geolite.maxmind.com/download/geoip/database/#{name}.tar.gz") + + tar_gz_file = Tempfile.new + begin + tar_gz_file.binmode + tar_gz_file.write(Net::HTTP.get(uri)) + tar_gz_file.close + + extractor = Gem::Package::TarReader.new(Zlib::GzipReader.open(tar_gz_file.path)) + extractor.rewind + + extractor.each do |entry| + next unless entry.full_name.ends_with?(".mmdb") + File.open(mmdb_path(name), "wb") { |f| f.write(entry.read) } + end + ensure + tar_gz_file.close + tar_gz_file.unlink + extractor.close + end + end + def mmdb_load(filepath) begin MaxMindDB.new(filepath, MaxMindDB::LOW_MEMORY_FILE_READER) diff --git a/lib/tasks/assets.rake b/lib/tasks/assets.rake index 2815f4440c5..09511f4056f 100644 --- a/lib/tasks/assets.rake +++ b/lib/tasks/assets.rake @@ -164,6 +164,18 @@ def concurrent? end task 'assets:precompile' => 'assets:precompile:before' do + path = DiscourseIpInfo.mmdb_path('GeoLite2-City') + mtime = File.exist?(path) && File.mtime(path) + + if refresh_days = SiteSetting.refresh_maxmind_db_during_precompile_days + if !mtime || mtime < refresh_days.days.ago + puts "Downloading MaxMindDB..." + mmdb_thread = Thread.new do + DiscourseIpInfo.mmdb_download('GeoLite2-City') + DiscourseIpInfo.mmdb_download('GeoLite2-ASN') + end + end + end if $bypass_sprockets_uglify puts "Compressing Javascript and Generating Source Maps" @@ -218,6 +230,7 @@ task 'assets:precompile' => 'assets:precompile:before' do end end + mmdb_thread.join if mmdb_thread end Rake::Task["assets:precompile"].enhance do diff --git a/lib/tasks/maxminddb.rake b/lib/tasks/maxminddb.rake index 9ce9ea995be..90961c52c06 100644 --- a/lib/tasks/maxminddb.rake +++ b/lib/tasks/maxminddb.rake @@ -1,30 +1,10 @@ -require 'rubygems/package' -require 'zlib' +require_dependency 'discourse_ip_info' desc "downloads MaxMind's GeoLite2-City database" task "maxminddb:get" do + puts "Downloading MaxMindDb's GeoLite2-City..." + DiscourseIpInfo.mmdb_download('GeoLite2-City') - def download_mmdb(name) - puts "Downloading MaxMindDb #{name}" - uri = URI("http://geolite.maxmind.com/download/geoip/database/#{name}.tar.gz") - tar_gz_archive = Net::HTTP.get(uri) - - extractor = Gem::Package::TarReader.new(Zlib::GzipReader.new(StringIO.new(tar_gz_archive))) - extractor.rewind - - extractor.each do |entry| - next unless entry.full_name.ends_with?(".mmdb") - - filename = File.join(Rails.root, 'vendor', 'data', "#{name}.mmdb") - puts "Writing #{filename}..." - File.open(filename, "wb") do |f| - f.write(entry.read) - end - end - - extractor.close - end - - download_mmdb('GeoLite2-City') - download_mmdb('GeoLite2-ASN') + puts "Downloading MaxMindDb's GeoLite2-ASN..." + DiscourseIpInfo.mmdb_download('GeoLite2-ASN') end