115 lines
3.1 KiB
Ruby
115 lines
3.1 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module RetrieveTitle
|
|
CRAWL_TIMEOUT = 1
|
|
UNRECOVERABLE_ERRORS = [
|
|
Net::ReadTimeout,
|
|
FinalDestination::SSRFError,
|
|
FinalDestination::UrlEncodingError,
|
|
]
|
|
|
|
def self.crawl(url, max_redirects: nil, initial_https_redirect_ignore_limit: false)
|
|
fetch_title(
|
|
url,
|
|
max_redirects: max_redirects,
|
|
initial_https_redirect_ignore_limit: initial_https_redirect_ignore_limit,
|
|
)
|
|
rescue *UNRECOVERABLE_ERRORS
|
|
# ¯\_(ツ)_/¯
|
|
end
|
|
|
|
def self.extract_title(html, encoding = nil)
|
|
title = nil
|
|
return nil if html =~ /<title>/ && html !~ %r{</title>}
|
|
|
|
doc = nil
|
|
begin
|
|
doc = Nokogiri.HTML5(html, nil, encoding)
|
|
rescue ArgumentError
|
|
# invalid HTML (Eg: too many attributes, status tree too deep) - ignore
|
|
# Error in nokogumbo is not specialized, uses generic ArgumentError
|
|
# see: https://www.rubydoc.info/gems/nokogiri/Nokogiri/HTML5#label-Error+reporting
|
|
end
|
|
|
|
if doc
|
|
title = doc.at("title")&.inner_text
|
|
|
|
# A horrible hack - YouTube uses `document.title` to populate the title
|
|
# for some reason. For any other site than YouTube this wouldn't be worth it.
|
|
if title == "YouTube" && html =~ /document\.title *= *"(.*)";/
|
|
title = Regexp.last_match[1].sub(/ - YouTube\z/, "")
|
|
end
|
|
|
|
if !title && node = doc.at('meta[property="og:title"]')
|
|
title = node["content"]
|
|
end
|
|
end
|
|
|
|
if title.present?
|
|
title.gsub!(/\n/, " ")
|
|
title.gsub!(/ +/, " ")
|
|
title.strip!
|
|
return title
|
|
end
|
|
nil
|
|
end
|
|
|
|
private
|
|
|
|
def self.max_chunk_size(uri)
|
|
# Exception for sites that leave the title until very late.
|
|
if uri.host =~
|
|
/(^|\.)amazon\.(com|ca|co\.uk|es|fr|de|it|com\.au|com\.br|cn|in|co\.jp|com\.mx)\z/
|
|
return 500
|
|
end
|
|
return 300 if uri.host =~ /(^|\.)youtube\.com\z/ || uri.host =~ /(^|\.)youtu\.be\z/
|
|
return 50 if uri.host =~ /(^|\.)github\.com\z/
|
|
|
|
# default is 20k
|
|
20
|
|
end
|
|
|
|
# Fetch the beginning of a HTML document at a url
|
|
def self.fetch_title(url, max_redirects: nil, initial_https_redirect_ignore_limit: false)
|
|
fd =
|
|
FinalDestination.new(
|
|
url,
|
|
timeout: CRAWL_TIMEOUT,
|
|
stop_at_blocked_pages: true,
|
|
max_redirects: max_redirects,
|
|
initial_https_redirect_ignore_limit: initial_https_redirect_ignore_limit,
|
|
headers: {
|
|
Accept: "text/html,*/*",
|
|
},
|
|
)
|
|
|
|
current = nil
|
|
title = nil
|
|
encoding = nil
|
|
|
|
fd.get do |_response, chunk, uri|
|
|
unless Net::HTTPRedirection === _response
|
|
throw :done if uri.blank?
|
|
|
|
if current
|
|
current << chunk
|
|
else
|
|
current = chunk
|
|
end
|
|
|
|
if !encoding && content_type = _response["content-type"]&.strip&.downcase
|
|
if content_type =~ /charset="?([a-z0-9_-]+)"?/
|
|
encoding = Regexp.last_match(1)
|
|
encoding = nil if !Encoding.list.map(&:name).map(&:downcase).include?(encoding)
|
|
end
|
|
end
|
|
|
|
max_size = max_chunk_size(uri) * 1024
|
|
title = extract_title(current, encoding)
|
|
throw :done if title || max_size < current.length
|
|
end
|
|
end
|
|
title
|
|
end
|
|
end
|