Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
discourse/spec/components/retrieve_title_spec.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

152 lines
5.2 KiB
Ruby
Raw Normal View History

DEV: use #frozen_string_literal: true on all spec This change both speeds up specs (less strings to allocate) and helps catch cases where methods in Discourse are mutating inputs. Overall we will be migrating everything to use #frozen_string_literal: true it will take a while, but this is the first and safest move in this direction
2019-04-29 20:27:42 -04:00
# frozen_string_literal: true
FEATURE: Whitelists for inline oneboxing
2017-07-21 15:29:04 -04:00
require 'rails_helper'
describe RetrieveTitle do
context "extract_title" do
it "will extract the value from the title tag" do
title = RetrieveTitle.extract_title(
"<html><title>My Cool Title</title></html>"
)
expect(title).to eq("My Cool Title")
end
it "will strip whitespace" do
title = RetrieveTitle.extract_title(
"<html><title> Another Title\n\n </title></html>"
)
expect(title).to eq("Another Title")
end
FEATURE: option to enable inline oneboxes for all domains Also, change to prefer title over open graph which is often way too sparse
2017-08-02 14:27:21 -04:00
it "will pick og:title if title is missing" do
title = RetrieveTitle.extract_title(<<~HTML
<html>
<meta property="og:title" content="Good Title"
</html>
HTML
)
expect(title).to eq("Good Title")
end
FEATURE: add title tag for group detail page (#13702)
2021-07-12 10:35:57 -04:00
it "will prefer the title over the opengraph tag" do
FEATURE: Whitelists for inline oneboxing
2017-07-21 15:29:04 -04:00
title = RetrieveTitle.extract_title(<<~HTML
<html>
FEATURE: option to enable inline oneboxes for all domains Also, change to prefer title over open graph which is often way too sparse
2017-08-02 14:27:21 -04:00
<title>Good Title</title>
<meta property="og:title" content="Bad Title"
FEATURE: Whitelists for inline oneboxing
2017-07-21 15:29:04 -04:00
</html>
HTML
)
expect(title).to eq("Good Title")
end
FIX: Hack our title retriever so that it parses YouTube URLs
2017-09-28 09:29:50 -04:00
it "will parse a YouTube url from javascript" do
title = RetrieveTitle.extract_title(<<~HTML
<html>
<title>YouTube</title>
<script>document.title = "Video Title";</script>
</html>
HTML
)
expect(title).to eq("Video Title")
end
PERF: ability to crawl for titles without extra HEAD req Also, introduces a much more aggressive timeout for title crawling and introduces gzip to body that is crawled
2018-01-28 23:36:52 -05:00
end
context "crawl" do
it "can properly extract a title from a url" do
stub_request(:get, "https://brelksdjflaskfj.com/amazing")
.to_return(status: 200, body: "<html><title>very amazing</title>")
FIX: Hack our title retriever so that it parses YouTube URLs
2017-09-28 09:29:50 -04:00
PERF: ability to crawl for titles without extra HEAD req Also, introduces a much more aggressive timeout for title crawling and introduces gzip to body that is crawled
2018-01-28 23:36:52 -05:00
# we still resolve the IP address for every host
IPSocket.stubs(:getaddress).returns('100.2.3.4')
expect(RetrieveTitle.crawl("https://brelksdjflaskfj.com/amazing")).to eq("very amazing")
end
FEATURE: Whitelists for inline oneboxing
2017-07-21 15:29:04 -04:00
FIX: Inline Onebox should use encoding from Content-Type header when present (#11625) * FIX: Inline onebox should use encoding from Content-Type header when present * Use Regexp.last_match(1) Signed-off-by: OsamaSayegh <asooomaasoooma90@gmail.com>
2021-01-04 14:32:08 -05:00
it "detects and uses encoding from Content-Type header" do
stub_request(:get, "https://brelksdjflaskfj.com/amazing")
.to_return(
status: 200,
body: "<html><title>fancy apostrophes ’’’</title>".dup.force_encoding('ASCII-8BIT'),
headers: { 'Content-Type' => 'text/html; charset="utf-8"' }
)
IPSocket.stubs(:getaddress).returns('100.2.3.4')
expect(RetrieveTitle.crawl("https://brelksdjflaskfj.com/amazing")).to eq("fancy apostrophes ’’’")
stub_request(:get, "https://brelksdjflaskfj.com/amazing")
.to_return(
status: 200,
body: "<html><title>japanese こんにちは website</title>".encode('EUC-JP').force_encoding('ASCII-8BIT'),
headers: { 'Content-Type' => 'text/html;charset=euc-jp' }
)
IPSocket.stubs(:getaddress).returns('100.2.3.4')
expect(RetrieveTitle.crawl("https://brelksdjflaskfj.com/amazing")).to eq("japanese こんにちは website")
end
FIX: follow redirects for inline/mini onebox (#13512)
2021-06-24 10:23:39 -04:00
it "can follow redirect" do
stub_request(:get, "http://foobar.com/amazing").
to_return(status: 301, body: "", headers: { "location" => "https://wikipedia.com/amazing" })
stub_request(:get, "https://wikipedia.com/amazing").
to_return(status: 200, body: "<html><title>very amazing</title>", headers: {})
FIX: Respect blocked domains list when redirecting (#15656) Our previous implementation used a simple `blocked_domain_array.include?(hostname)` so some values were not matching. Additionally, in some configurations like ours, we'd used "cat.*.dog.com" with the assumption we'd support globbing. This change implicitly allows globbing by blocking "http://a.b.com" if "b.com" is a blocked domain but does not actively do anything for "*". An upcoming change might include frontend validation for values that can be inserted.
2022-01-20 01:12:34 -05:00
IPSocket.stubs(:getaddress).returns('100.2.3.4')
expect(RetrieveTitle.crawl("http://foobar.com/amazing")).to eq("very amazing")
end
it "returns empty title if redirect uri is in blacklist" do
SiteSetting.blocked_onebox_domains = "wikipedia.com"
stub_request(:get, "http://foobar.com/amazing")
.to_return(status: 301, body: "", headers: { "location" => "https://wikipedia.com/amazing" })
stub_request(:get, "https://wikipedia.com/amazing")
.to_return(status: 200, body: "<html><title>very amazing</title>", headers: {})
expect(RetrieveTitle.crawl("http://foobar.com/amazing")).to eq(nil)
end
it "returns title if 'midway redirect' is blocked but final redirect uri is not blocked" do
SiteSetting.blocked_onebox_domains = "wikipedia.com"
stub_request(:get, "http://foobar.com/amazing")
.to_return(status: 301, body: "", headers: { "location" => "https://wikipedia.com/amazing" })
stub_request(:get, "https://wikipedia.com/amazing")
.to_return(status: 301, body: "", headers: { "location" => "https://cat.com/meow" })
stub_request(:get, "https://cat.com/meow")
.to_return(status: 200, body: "<html><title>very amazing</title>", headers: {})
FIX: follow redirects for inline/mini onebox (#13512)
2021-06-24 10:23:39 -04:00
expect(RetrieveTitle.crawl("http://foobar.com/amazing")).to eq("very amazing")
end
FIX: Inline Onebox should use encoding from Content-Type header when present (#11625) * FIX: Inline onebox should use encoding from Content-Type header when present * Use Regexp.last_match(1) Signed-off-by: OsamaSayegh <asooomaasoooma90@gmail.com>
2021-01-04 14:32:08 -05:00
end
FIX: increase chunk size to fetch title tag correctly (#14144)
2021-09-03 03:45:58 -04:00
context 'fetch_title' do
it "does not parse broken title tag" do
# webmock does not do chunks
stub_request(:get, "https://en.wikipedia.org/wiki/Internet").
to_return(status: 200, body: "<html><head><title>Internet - Wikipedia</ti" , headers: {})
title = RetrieveTitle.fetch_title("https://en.wikipedia.org/wiki/Internet")
expect(title).to eq(nil)
end
it "can parse correct title tag" do
# webmock does not do chunks
stub_request(:get, "https://en.wikipedia.org/wiki/Internet").
to_return(status: 200, body: "<html><head><title>Internet - Wikipedia</title>" , headers: {})
title = RetrieveTitle.fetch_title("https://en.wikipedia.org/wiki/Internet")
expect(title).to eq("Internet - Wikipedia")
end
end
FEATURE: Whitelists for inline oneboxing
2017-07-21 15:29:04 -04:00
end