Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
discourse/spec/lib/retrieve_title_spec.rb

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

167 lines
5.7 KiB
Ruby
Raw Normal View History

DEV: use #frozen_string_literal: true on all spec This change both speeds up specs (less strings to allocate) and helps catch cases where methods in Discourse are mutating inputs. Overall we will be migrating everything to use #frozen_string_literal: true it will take a while, but this is the first and safest move in this direction
2019-04-29 20:27:42 -04:00
# frozen_string_literal: true
FEATURE: Whitelists for inline oneboxing
2017-07-21 15:29:04 -04:00
describe RetrieveTitle do
context "extract_title" do
it "will extract the value from the title tag" do
title = RetrieveTitle.extract_title(
"<html><title>My Cool Title</title></html>"
)
expect(title).to eq("My Cool Title")
end
it "will strip whitespace" do
title = RetrieveTitle.extract_title(
"<html><title> Another Title\n\n </title></html>"
)
expect(title).to eq("Another Title")
end
FEATURE: option to enable inline oneboxes for all domains Also, change to prefer title over open graph which is often way too sparse
2017-08-02 14:27:21 -04:00
it "will pick og:title if title is missing" do
title = RetrieveTitle.extract_title(<<~HTML
<html>
<meta property="og:title" content="Good Title"
</html>
HTML
)
expect(title).to eq("Good Title")
end
FEATURE: add title tag for group detail page (#13702)
2021-07-12 10:35:57 -04:00
it "will prefer the title over the opengraph tag" do
FEATURE: Whitelists for inline oneboxing
2017-07-21 15:29:04 -04:00
title = RetrieveTitle.extract_title(<<~HTML
<html>
FEATURE: option to enable inline oneboxes for all domains Also, change to prefer title over open graph which is often way too sparse
2017-08-02 14:27:21 -04:00
<title>Good Title</title>
<meta property="og:title" content="Bad Title"
FEATURE: Whitelists for inline oneboxing
2017-07-21 15:29:04 -04:00
</html>
HTML
)
expect(title).to eq("Good Title")
end
FIX: Hack our title retriever so that it parses YouTube URLs
2017-09-28 09:29:50 -04:00
it "will parse a YouTube url from javascript" do
title = RetrieveTitle.extract_title(<<~HTML
<html>
<title>YouTube</title>
<script>document.title = "Video Title";</script>
</html>
HTML
)
expect(title).to eq("Video Title")
end
PERF: ability to crawl for titles without extra HEAD req Also, introduces a much more aggressive timeout for title crawling and introduces gzip to body that is crawled
2018-01-28 23:36:52 -05:00
end
context "crawl" do
it "can properly extract a title from a url" do
stub_request(:get, "https://brelksdjflaskfj.com/amazing")
.to_return(status: 200, body: "<html><title>very amazing</title>")
FIX: Hack our title retriever so that it parses YouTube URLs
2017-09-28 09:29:50 -04:00
PERF: ability to crawl for titles without extra HEAD req Also, introduces a much more aggressive timeout for title crawling and introduces gzip to body that is crawled
2018-01-28 23:36:52 -05:00
# we still resolve the IP address for every host
IPSocket.stubs(:getaddress).returns('100.2.3.4')
expect(RetrieveTitle.crawl("https://brelksdjflaskfj.com/amazing")).to eq("very amazing")
end
FEATURE: Whitelists for inline oneboxing
2017-07-21 15:29:04 -04:00
FIX: Inline Onebox should use encoding from Content-Type header when present (#11625) * FIX: Inline onebox should use encoding from Content-Type header when present * Use Regexp.last_match(1) Signed-off-by: OsamaSayegh <asooomaasoooma90@gmail.com>
2021-01-04 14:32:08 -05:00
it "detects and uses encoding from Content-Type header" do
stub_request(:get, "https://brelksdjflaskfj.com/amazing")
.to_return(
status: 200,
body: "<html><title>fancy apostrophes ’’’</title>".dup.force_encoding('ASCII-8BIT'),
headers: { 'Content-Type' => 'text/html; charset="utf-8"' }
)
IPSocket.stubs(:getaddress).returns('100.2.3.4')
expect(RetrieveTitle.crawl("https://brelksdjflaskfj.com/amazing")).to eq("fancy apostrophes ’’’")
stub_request(:get, "https://brelksdjflaskfj.com/amazing")
.to_return(
status: 200,
body: "<html><title>japanese こんにちは website</title>".encode('EUC-JP').force_encoding('ASCII-8BIT'),
headers: { 'Content-Type' => 'text/html;charset=euc-jp' }
)
IPSocket.stubs(:getaddress).returns('100.2.3.4')
expect(RetrieveTitle.crawl("https://brelksdjflaskfj.com/amazing")).to eq("japanese こんにちは website")
end
FIX: follow redirects for inline/mini onebox (#13512)
2021-06-24 10:23:39 -04:00
it "can follow redirect" do
stub_request(:get, "http://foobar.com/amazing").
to_return(status: 301, body: "", headers: { "location" => "https://wikipedia.com/amazing" })
stub_request(:get, "https://wikipedia.com/amazing").
to_return(status: 200, body: "<html><title>very amazing</title>", headers: {})
FIX: Respect blocked domains list when redirecting (#15656) Our previous implementation used a simple `blocked_domain_array.include?(hostname)` so some values were not matching. Additionally, in some configurations like ours, we'd used "cat.*.dog.com" with the assumption we'd support globbing. This change implicitly allows globbing by blocking "http://a.b.com" if "b.com" is a blocked domain but does not actively do anything for "*". An upcoming change might include frontend validation for values that can be inserted.
2022-01-20 01:12:34 -05:00
IPSocket.stubs(:getaddress).returns('100.2.3.4')
expect(RetrieveTitle.crawl("http://foobar.com/amazing")).to eq("very amazing")
end
it "returns empty title if redirect uri is in blacklist" do
SiteSetting.blocked_onebox_domains = "wikipedia.com"
stub_request(:get, "http://foobar.com/amazing")
.to_return(status: 301, body: "", headers: { "location" => "https://wikipedia.com/amazing" })
stub_request(:get, "https://wikipedia.com/amazing")
.to_return(status: 200, body: "<html><title>very amazing</title>", headers: {})
expect(RetrieveTitle.crawl("http://foobar.com/amazing")).to eq(nil)
end
FIX: Apply onebox blocked domain checks on every redirect (#16150) The `blocked onebox domains` setting lets site owners change what sites are allowed to be oneboxed. When a link is entered into a post, Discourse checks the domain of the link against that setting and blocks the onebox if the domain is blocked. But if there's a chain of redirects, then only the final destination website is checked against the site setting. This commit amends that behavior so that every website in the redirect chain is checked against the site setting, and if anything is blocked the original link doesn't onebox at all in the post. The `Discourse-No-Onebox` header is also checked in every response and the onebox is blocked if the header is set to "1". Additionally, Discourse will now include the `Discourse-No-Onebox` header with every response if the site requires login to access content. This is done to signal to a Discourse instance that it shouldn't attempt to onebox other Discourse instances if they're login-only. Non-Discourse websites can also use include that header if they don't wish to have Discourse onebox their content. Internal ticket: t59305.
2022-03-11 01:18:12 -05:00
it "doesn't return title if a blocked domain is encountered anywhere in the redirect chain" do
FIX: Respect blocked domains list when redirecting (#15656) Our previous implementation used a simple `blocked_domain_array.include?(hostname)` so some values were not matching. Additionally, in some configurations like ours, we'd used "cat.*.dog.com" with the assumption we'd support globbing. This change implicitly allows globbing by blocking "http://a.b.com" if "b.com" is a blocked domain but does not actively do anything for "*". An upcoming change might include frontend validation for values that can be inserted.
2022-01-20 01:12:34 -05:00
SiteSetting.blocked_onebox_domains = "wikipedia.com"
stub_request(:get, "http://foobar.com/amazing")
.to_return(status: 301, body: "", headers: { "location" => "https://wikipedia.com/amazing" })
stub_request(:get, "https://wikipedia.com/amazing")
.to_return(status: 301, body: "", headers: { "location" => "https://cat.com/meow" })
stub_request(:get, "https://cat.com/meow")
.to_return(status: 200, body: "<html><title>very amazing</title>", headers: {})
FIX: follow redirects for inline/mini onebox (#13512)
2021-06-24 10:23:39 -04:00
FIX: Apply onebox blocked domain checks on every redirect (#16150) The `blocked onebox domains` setting lets site owners change what sites are allowed to be oneboxed. When a link is entered into a post, Discourse checks the domain of the link against that setting and blocks the onebox if the domain is blocked. But if there's a chain of redirects, then only the final destination website is checked against the site setting. This commit amends that behavior so that every website in the redirect chain is checked against the site setting, and if anything is blocked the original link doesn't onebox at all in the post. The `Discourse-No-Onebox` header is also checked in every response and the onebox is blocked if the header is set to "1". Additionally, Discourse will now include the `Discourse-No-Onebox` header with every response if the site requires login to access content. This is done to signal to a Discourse instance that it shouldn't attempt to onebox other Discourse instances if they're login-only. Non-Discourse websites can also use include that header if they don't wish to have Discourse onebox their content. Internal ticket: t59305.
2022-03-11 01:18:12 -05:00
expect(RetrieveTitle.crawl("http://foobar.com/amazing")).to be_blank
end
it "doesn't return title if the Discourse-No-Onebox header == 1" do
stub_request(:get, "https://cat.com/meow/no-onebox")
.to_return(
status: 200,
body: "<html><title>discourse stay away</title>",
headers: { "Discourse-No-Onebox" => "1" }
)
expect(RetrieveTitle.crawl("https://cat.com/meow/no-onebox")).to be_blank
FIX: follow redirects for inline/mini onebox (#13512)
2021-06-24 10:23:39 -04:00
end
FIX: Do not raise if title cannot be crawled (#16247) If the crawled page returned an error, `FinalDestination#safe_get` yielded `nil` for `uri` and `chunk` arguments. Another problem is that `get` did not handle the case when `safe_get` failed and did not return the `location` and `set_cookie` headers.
2022-03-22 14:13:27 -04:00
it "doesn't return a title if response is unsuccessful" do
stub_request(:get, "https://example.com").to_return(status: 404, body: "")
expect(RetrieveTitle.crawl("https://example.com")).to eq(nil)
end
FIX: Inline Onebox should use encoding from Content-Type header when present (#11625) * FIX: Inline onebox should use encoding from Content-Type header when present * Use Regexp.last_match(1) Signed-off-by: OsamaSayegh <asooomaasoooma90@gmail.com>
2021-01-04 14:32:08 -05:00
end
FIX: increase chunk size to fetch title tag correctly (#14144)
2021-09-03 03:45:58 -04:00
context 'fetch_title' do
it "does not parse broken title tag" do
# webmock does not do chunks
stub_request(:get, "https://en.wikipedia.org/wiki/Internet").
to_return(status: 200, body: "<html><head><title>Internet - Wikipedia</ti" , headers: {})
title = RetrieveTitle.fetch_title("https://en.wikipedia.org/wiki/Internet")
expect(title).to eq(nil)
end
it "can parse correct title tag" do
# webmock does not do chunks
stub_request(:get, "https://en.wikipedia.org/wiki/Internet").
to_return(status: 200, body: "<html><head><title>Internet - Wikipedia</title>" , headers: {})
title = RetrieveTitle.fetch_title("https://en.wikipedia.org/wiki/Internet")
expect(title).to eq("Internet - Wikipedia")
end
end
FEATURE: Whitelists for inline oneboxing
2017-07-21 15:29:04 -04:00
end