2019-05-02 18:17:27 -04:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2014-04-05 14:47:25 -04:00
|
|
|
require 'open-uri'
|
|
|
|
require 'nokogiri'
|
|
|
|
require 'excon'
|
2017-07-21 15:29:04 -04:00
|
|
|
require_dependency 'retrieve_title'
|
2017-10-05 23:39:00 -04:00
|
|
|
require_dependency 'topic_link'
|
2014-04-05 14:47:25 -04:00
|
|
|
|
|
|
|
module Jobs
|
|
|
|
class CrawlTopicLink < Jobs::Base
|
|
|
|
|
2019-05-21 20:18:49 -04:00
|
|
|
sidekiq_options queue: 'low'
|
|
|
|
|
2014-04-05 14:47:25 -04:00
|
|
|
def execute(args)
|
|
|
|
raise Discourse::InvalidParameters.new(:topic_link_id) unless args[:topic_link_id].present?
|
2014-04-07 14:38:18 -04:00
|
|
|
|
2014-05-06 09:41:59 -04:00
|
|
|
topic_link = TopicLink.find_by(id: args[:topic_link_id], internal: false, crawled_at: nil)
|
2014-04-17 14:00:22 -04:00
|
|
|
return if topic_link.blank?
|
|
|
|
|
|
|
|
# Look for a topic embed for the URL. If it exists, use its title and don't crawl
|
|
|
|
topic_embed = TopicEmbed.where(embed_url: topic_link.url).includes(:topic).references(:topic).first
|
2015-05-06 02:53:28 -04:00
|
|
|
# topic could be deleted, so skip
|
|
|
|
if topic_embed && topic_embed.topic
|
2014-04-17 14:00:22 -04:00
|
|
|
TopicLink.where(id: topic_link.id).update_all(['title = ?, crawled_at = CURRENT_TIMESTAMP', topic_embed.topic.title[0..255]])
|
|
|
|
return
|
|
|
|
end
|
2014-04-07 16:03:47 -04:00
|
|
|
|
2014-04-17 14:00:22 -04:00
|
|
|
begin
|
2014-04-07 16:03:47 -04:00
|
|
|
crawled = false
|
|
|
|
|
2014-04-10 13:45:13 -04:00
|
|
|
# Special case: Images
|
|
|
|
# If the link is to an image, put the filename as the title
|
2018-09-09 22:22:45 -04:00
|
|
|
if FileHelper.is_supported_image?(topic_link.url)
|
2014-04-10 13:45:13 -04:00
|
|
|
uri = URI(topic_link.url)
|
|
|
|
filename = File.basename(uri.path)
|
|
|
|
crawled = (TopicLink.where(id: topic_link.id).update_all(["title = ?, crawled_at = CURRENT_TIMESTAMP", filename]) == 1)
|
|
|
|
end
|
|
|
|
|
|
|
|
unless crawled
|
|
|
|
# Fetch the beginning of the document to find the title
|
2017-07-21 15:29:04 -04:00
|
|
|
title = RetrieveTitle.crawl(topic_link.url)
|
|
|
|
if title.present?
|
|
|
|
crawled = (TopicLink.where(id: topic_link.id).update_all(['title = ?, crawled_at = CURRENT_TIMESTAMP', title[0..254]]) == 1)
|
2014-04-05 14:47:25 -04:00
|
|
|
end
|
|
|
|
end
|
2014-04-07 14:38:18 -04:00
|
|
|
rescue Exception
|
|
|
|
# If there was a connection error, do nothing
|
|
|
|
ensure
|
2014-04-10 13:19:38 -04:00
|
|
|
TopicLink.where(id: topic_link.id).update_all('crawled_at = CURRENT_TIMESTAMP') if !crawled && topic_link.present?
|
2014-04-07 14:38:18 -04:00
|
|
|
end
|
2014-04-05 14:47:25 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
end
|
|
|
|
end
|