discourse/spec/lib/final_destination_spec.rb

# frozen_string_literal: true

require 'final_destination'

RSpec.describe FinalDestination do
  let(:opts) do
    {
      ignore_redirects: ['https://ignore-me.com'],

      force_get_hosts: ['https://force.get.com', 'https://*.ihaveawildcard.com/'],

      preserve_fragment_url_hosts: ['https://eviltrout.com'],
    }
  end

  let(:doc_response) do
    {
      status: 200,
      headers: { "Content-Type" => "text/html" }
    }
  end

  let(:image_response) do
    {
      status: 200,
      headers: { "Content-Type" => "image/jpeg" }
    }
  end

  let(:body_response) do
    {
      status: 200,
      body: "<body>test</body>"
    }
  end

  def fd_stub_request(method, url)
    uri = URI.parse(url)

    host = uri.hostname
    ip = "1.2.3.4"

    # In Excon we pass the IP in the URL, so we need to stub
    # that version as well
    uri.hostname = "HOSTNAME_PLACEHOLDER"
    matcher = Regexp.escape(uri.to_s).sub(
      "HOSTNAME_PLACEHOLDER",
      "(#{Regexp.escape(host)}|#{Regexp.escape(ip)})"
    )

    stub_request(method, /\A#{matcher}\z/).with(headers: { "Host" => host })
  end

  def canonical_follow(from, dest)
    fd_stub_request(:get, from).to_return(
      status: 200,
      body: "<head><link rel=\"canonical\" href=\"#{dest}\"></head>"
    )
  end

  def redirect_response(from, dest)
    fd_stub_request(:head, from).to_return(
      status: 302,
      headers: { "Location" => dest }
    )
  end

  def fd(url)
    FinalDestination.new(url, opts)
  end

  it 'correctly parses ignored hostnames' do
    fd = FinalDestination.new('https://meta.discourse.org',
      ignore_redirects: ['http://google.com', 'youtube.com', 'https://meta.discourse.org', '://bing.com']
    )

    expect(fd.ignored).to eq(['test.localhost', 'google.com', 'meta.discourse.org'])
  end

  describe '.resolve' do
    it "has a ready status code before anything happens" do
      expect(fd('https://eviltrout.com').status).to eq(:ready)
    end

    it "returns nil for an invalid url" do
      expect(fd(nil).resolve).to be_nil
      expect(fd('asdf').resolve).to be_nil
    end

    it "returns nil for unresolvable url" do
      FinalDestination::SSRFDetector.stubs(:lookup_ips).raises(SocketError)
      expect(fd("https://example.com").resolve).to eq(nil)
    end

    it "returns nil for url timeout" do
      FinalDestination::SSRFDetector.stubs(:lookup_ips).raises(Timeout::Error)
      expect(fd("https://example.com").resolve).to eq(nil)
    end

    it "returns nil when read timeouts" do
      Excon.expects(:public_send).raises(Excon::Errors::Timeout)

      expect(fd('https://discourse.org').resolve).to eq(nil)
    end

    context "without redirects" do
      before do
        fd_stub_request(:head, "https://eviltrout.com/").to_return(doc_response)
      end

      it "returns the final url" do
        final = FinalDestination.new('https://eviltrout.com', opts)
        expect(final.resolve.to_s).to eq('https://eviltrout.com')
        expect(final.redirected?).to eq(false)
        expect(final.status).to eq(:resolved)
      end
    end

    it "ignores redirects" do
      final = FinalDestination.new('https://ignore-me.com/some-url', opts)
      expect(final.resolve.to_s).to eq('https://ignore-me.com/some-url')
      expect(final.redirected?).to eq(false)
      expect(final.status).to eq(:resolved)
    end

    context "with underscores in URLs" do
      before do
        fd_stub_request(:head, 'https://some_thing.example.com').to_return(doc_response)
      end

      it "doesn't raise errors with underscores in urls" do
        final = FinalDestination.new('https://some_thing.example.com', opts)
        expect(final.resolve.to_s).to eq('https://some_thing.example.com')
        expect(final.redirected?).to eq(false)
        expect(final.status).to eq(:resolved)
      end
    end

    context "with a couple of redirects" do
      before do
        redirect_response("https://eviltrout.com", "https://codinghorror.com/blog")
        redirect_response("https://codinghorror.com/blog", "https://discourse.org")
        fd_stub_request(:head, "https://discourse.org").to_return(doc_response)
      end

      it "returns the final url" do
        final = FinalDestination.new('https://eviltrout.com', opts)
        expect(final.resolve.to_s).to eq('https://discourse.org')
        expect(final.redirected?).to eq(true)
        expect(final.status).to eq(:resolved)
      end
    end

    context "with too many redirects" do
      before do
        redirect_response("https://eviltrout.com", "https://codinghorror.com/blog")
        redirect_response("https://codinghorror.com/blog", "https://discourse.org")
        fd_stub_request(:head, "https://discourse.org").to_return(doc_response)
      end

      it "returns the final url" do
        final = FinalDestination.new('https://eviltrout.com', opts.merge(max_redirects: 1))
        expect(final.resolve).to be_nil
        expect(final.redirected?).to eq(true)
        expect(final.status).to eq(:too_many_redirects)
      end
    end

    context "with a redirect to an internal IP" do
      before do
        redirect_response("https://eviltrout.com", "https://private-host.com")
        FinalDestination::SSRFDetector.stubs(:lookup_and_filter_ips).with("eviltrout.com").returns(["1.2.3.4"])
        FinalDestination::SSRFDetector.stubs(:lookup_and_filter_ips).with("private-host.com").raises(FinalDestination::SSRFDetector::DisallowedIpError)
      end

      it "returns the final url" do
        final = FinalDestination.new('https://eviltrout.com', opts)
        expect(final.resolve).to be_nil
        expect(final.redirected?).to eq(true)
        expect(final.status).to eq(:invalid_address)
      end
    end

    context "with a redirect to login path" do
      before do
        redirect_response("https://eviltrout.com/t/xyz/1", "https://eviltrout.com/login")
      end

      it "does not follow redirect" do
        final = FinalDestination.new('https://eviltrout.com/t/xyz/1', opts)
        expect(final.resolve.to_s).to eq('https://eviltrout.com/t/xyz/1')
        expect(final.redirected?).to eq(false)
        expect(final.status).to eq(:resolved)
      end
    end

    it 'raises error when response is too big' do
      stub_const(described_class, "MAX_REQUEST_SIZE_BYTES", 1) do
        fd_stub_request(:get, "https://codinghorror.com/blog").to_return(body_response)
        final = FinalDestination.new('https://codinghorror.com/blog', opts.merge(follow_canonical: true))
        expect { final.resolve }.to raise_error(Excon::Errors::ExpectationFailed, "response size too big: https://codinghorror.com/blog")
      end
    end

    it 'raises error when response is too slow' do
      fd_stub_request(:get, "https://codinghorror.com/blog").to_return(lambda { |request| freeze_time(11.seconds.from_now) ; body_response })
      final = FinalDestination.new('https://codinghorror.com/blog', opts.merge(follow_canonical: true))
      expect { final.resolve }.to raise_error(Excon::Errors::ExpectationFailed, "connect timeout reached: https://codinghorror.com/blog")
    end

    context 'when following canonical links' do
      it 'resolves the canonical link as the final destination' do
        canonical_follow("https://eviltrout.com", "https://codinghorror.com/blog")
        fd_stub_request(:head, "https://codinghorror.com/blog").to_return(doc_response)

        final = FinalDestination.new('https://eviltrout.com', opts.merge(follow_canonical: true))

        expect(final.resolve.to_s).to eq("https://codinghorror.com/blog")
        expect(final.redirected?).to eq(false)
        expect(final.status).to eq(:resolved)
      end

      it 'resolves the canonical link when the URL is relative' do
        host = "https://codinghorror.com"

        canonical_follow("#{host}/blog", "/blog/canonical")
        fd_stub_request(:head, "#{host}/blog/canonical").to_return(doc_response)

        final = FinalDestination.new("#{host}/blog", opts.merge(follow_canonical: true))

        expect(final.resolve.to_s).to eq("#{host}/blog/canonical")
        expect(final.redirected?).to eq(false)
        expect(final.status).to eq(:resolved)
      end

      it 'resolves the canonical link when the URL is relative and does not start with the / symbol' do
        host = "https://codinghorror.com"
        canonical_follow("#{host}/blog", "blog/canonical")
        fd_stub_request(:head, "#{host}/blog/canonical").to_return(doc_response)

        final = FinalDestination.new("#{host}/blog", opts.merge(follow_canonical: true))

        expect(final.resolve.to_s).to eq("#{host}/blog/canonical")
        expect(final.redirected?).to eq(false)
        expect(final.status).to eq(:resolved)
      end

      it "does not follow the canonical link if it's the same as the current URL" do
        canonical_follow("https://eviltrout.com", "https://eviltrout.com")

        final = FinalDestination.new('https://eviltrout.com', opts.merge(follow_canonical: true))

        expect(final.resolve.to_s).to eq("https://eviltrout.com")
        expect(final.redirected?).to eq(false)
        expect(final.status).to eq(:resolved)
      end

      it "does not follow the canonical link if it's invalid" do
        canonical_follow("https://eviltrout.com", "")

        final = FinalDestination.new('https://eviltrout.com', opts.merge(follow_canonical: true))

        expect(final.resolve.to_s).to eq("https://eviltrout.com")
        expect(final.redirected?).to eq(false)
        expect(final.status).to eq(:resolved)
      end
    end

    context "when forcing GET" do
      it "will do a GET when forced" do
        url = 'https://force.get.com/posts?page=4'
        get_stub = fd_stub_request(:get, url)
        head_stub = fd_stub_request(:head, url)

        final = FinalDestination.new(url, opts)
        expect(final.resolve.to_s).to eq(url)
        expect(final.status).to eq(:resolved)
        expect(get_stub).to have_been_requested
        expect(head_stub).to_not have_been_requested
      end

      it "will do a HEAD if not forced" do
        url = 'https://eviltrout.com/posts?page=2'
        get_stub = fd_stub_request(:get, url)
        head_stub = fd_stub_request(:head, url)

        final = FinalDestination.new(url, opts)
        expect(final.resolve.to_s).to eq(url)
        expect(final.status).to eq(:resolved)
        expect(get_stub).to_not have_been_requested
        expect(head_stub).to have_been_requested
      end

      it "will do a GET when forced on a wildcard subdomain" do
        url = 'https://any-subdomain.ihaveawildcard.com/some/other/content'
        get_stub = fd_stub_request(:get, url)
        head_stub = fd_stub_request(:head, url)

        final = FinalDestination.new(url, opts)
        expect(final.resolve.to_s).to eq(url)
        expect(final.status).to eq(:resolved)
        expect(get_stub).to have_been_requested
        expect(head_stub).to_not have_been_requested
      end

      it "will do a HEAD if on a subdomain of a forced get domain without a wildcard" do
        url = 'https://particularly.eviltrout.com/has/a/secret/plan'
        get_stub = fd_stub_request(:get, url)
        head_stub = fd_stub_request(:head, url)

        final = FinalDestination.new(url, opts)
        expect(final.resolve.to_s).to eq(url)
        expect(final.status).to eq(:resolved)
        expect(get_stub).to_not have_been_requested
        expect(head_stub).to have_been_requested
      end

    end

    context "when HEAD not supported" do
      before do
        fd_stub_request(:get, 'https://eviltrout.com').to_return(
          status: 301,
          headers: {
            "Location" => 'https://discourse.org',
            'Set-Cookie' => 'evil=trout'
          }
        )
        fd_stub_request(:head, 'https://discourse.org')
      end

      context "when the status code is 405" do
        before do
          fd_stub_request(:head, 'https://eviltrout.com').to_return(status: 405)
        end

        it "will try a GET" do
          final = FinalDestination.new('https://eviltrout.com', opts)
          expect(final.resolve.to_s).to eq('https://discourse.org')
          expect(final.status).to eq(:resolved)
          expect(final.cookie).to eq('evil=trout')
        end
      end

      context "when the status code is 501" do
        before do
          fd_stub_request(:head, 'https://eviltrout.com').to_return(status: 501)
        end

        it "will try a GET" do
          final = FinalDestination.new('https://eviltrout.com', opts)
          expect(final.resolve.to_s).to eq('https://discourse.org')
          expect(final.status).to eq(:resolved)
          expect(final.cookie).to eq('evil=trout')
        end
      end

      it "correctly extracts cookies during GET" do
        fd_stub_request(:head, "https://eviltrout.com").to_return(status: 405)

        fd_stub_request(:get, "https://eviltrout.com")
          .to_return(status: 302, body: "" , headers: {
            "Location" => "https://eviltrout.com",
            "Set-Cookie" => ["foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com",
                             "bar=1",
                             "baz=2; expires=Tue, 19-Feb-2019 10:14:24 GMT; path=/; domain=eviltrout.com"]
          })

        fd_stub_request(:head, "https://eviltrout.com")
          .with(headers: { "Cookie" => "bar=1; baz=2; foo=219ffwef9w0f" })

        final = FinalDestination.new("https://eviltrout.com", opts)
        expect(final.resolve.to_s).to eq("https://eviltrout.com")
        expect(final.status).to eq(:resolved)
        expect(final.cookie).to eq("bar=1; baz=2; foo=219ffwef9w0f")
      end
    end

    it "should use the correct format for cookies when there is only one cookie" do
      fd_stub_request(:head, "https://eviltrout.com")
        .to_return(status: 302, headers: {
          "Location" => "https://eviltrout.com",
          "Set-Cookie" => "foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com"
        })

      fd_stub_request(:head, "https://eviltrout.com")
        .with(headers: { "Cookie" => "foo=219ffwef9w0f" })

      final = FinalDestination.new("https://eviltrout.com", opts)
      expect(final.resolve.to_s).to eq("https://eviltrout.com")
      expect(final.status).to eq(:resolved)
      expect(final.cookie).to eq("foo=219ffwef9w0f")
    end

    it "should use the correct format for cookies when there are multiple cookies" do
      fd_stub_request(:head, "https://eviltrout.com")
        .to_return(status: 302, headers: {
          "Location" => "https://eviltrout.com",
          "Set-Cookie" => ["foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com",
                           "bar=1",
                           "baz=2; expires=Tue, 19-Feb-2019 10:14:24 GMT; path=/; domain=eviltrout.com"]
        })

      fd_stub_request(:head, "https://eviltrout.com")
        .with(headers: { "Cookie" => "bar=1; baz=2; foo=219ffwef9w0f" })

      final = FinalDestination.new("https://eviltrout.com", opts)
      expect(final.resolve.to_s).to eq("https://eviltrout.com")
      expect(final.status).to eq(:resolved)
      expect(final.cookie).to eq("bar=1; baz=2; foo=219ffwef9w0f")
    end

    it "persists fragment url" do
      origin_url = "https://eviltrout.com/origin/lib/code/foobar.rb"
      upstream_url = "https://eviltrout.com/upstream/lib/code/foobar.rb"

      redirect_response(origin_url, upstream_url)
      fd_stub_request(:head, upstream_url).to_return(doc_response)

      final = FinalDestination.new("#{origin_url}#L154-L205", opts)
      expect(final.resolve.to_s).to eq("#{upstream_url}#L154-L205")
      expect(final.status).to eq(:resolved)
    end

    context "with content_type" do
      before do
        fd_stub_request(:head, "https://eviltrout.com/this/is/an/image").to_return(image_response)
      end

      it "returns a content_type" do
        final = FinalDestination.new("https://eviltrout.com/this/is/an/image", opts)
        expect(final.resolve.to_s).to eq("https://eviltrout.com/this/is/an/image")
        expect(final.content_type).to eq("image/jpeg")
        expect(final.status).to eq(:resolved)
      end
    end
  end

  describe '#get' do
    let(:fd) { FinalDestination.new("http://wikipedia.com", opts.merge(verbose: true)) }

    before do
      described_class.clear_https_cache!("wikipedia.com")
    end

    context "when there is a redirect" do
      before do
        stub_request(:get, "http://wikipedia.com/").
          to_return(status: 302, body: "" , headers: { "location" => "https://wikipedia.com/" })
        # webmock does not do chunks
        stub_request(:get, "https://wikipedia.com/").
          to_return(status: 200, body: "<html><head>" , headers: {})
      end

      after do
        WebMock.reset!
      end

      it "correctly streams" do
        chunk = nil
        result = fd.get do |resp, c|
          chunk = c
          throw :done
        end

        expect(result).to eq("https://wikipedia.com/")
        expect(chunk).to eq("<html><head>")
      end
    end

    context "when there is a timeout" do
      subject(:get) { fd.get {} }

      before do
        fd.stubs(:safe_session).raises(Timeout::Error)
      end

      it "logs the exception" do
        Rails.logger.expects(:warn).with(regexp_matches(/FinalDestination could not resolve URL \(timeout\)/))
        get
      end

      it "returns nothing" do
        expect(get).to be_blank
      end
    end

    context "when there is an SSL error" do
      subject(:get) { fd.get {} }

      before do
        fd.stubs(:safe_session).raises(OpenSSL::SSL::SSLError)
      end

      it "logs the exception" do
        Rails.logger.expects(:warn).with(regexp_matches(/an error with ssl occurred/i))
        get
      end

      it "returns nothing" do
        expect(get).to be_blank
      end
    end
  end

  describe ".validate_url_format" do
    it "supports http urls" do
      expect(fd('http://eviltrout.com').validate_uri_format).to eq(true)
    end

    it "supports https urls" do
      expect(fd('https://eviltrout.com').validate_uri_format).to eq(true)
    end

    it "doesn't support ftp urls" do
      expect(fd('ftp://eviltrout.com').validate_uri_format).to eq(false)
    end

    it "doesn't support IP urls" do
      expect(fd('http://104.25.152.10').validate_uri_format).to eq(false)
      expect(fd('https://[2001:abc:de:01:0:3f0:6a65:c2bf]').validate_uri_format).to eq(false)
    end

    it "returns false for schemeless URL" do
      expect(fd('eviltrout.com').validate_uri_format).to eq(false)
    end

    it "returns false for nil URL" do
      expect(fd(nil).validate_uri_format).to eq(false)
    end

    it "returns false for invalid ports" do
      expect(fd('http://eviltrout.com:21').validate_uri_format).to eq(false)
      expect(fd('https://eviltrout.com:8000').validate_uri_format).to eq(false)
    end

    it "returns true for valid ports" do
      expect(fd('http://eviltrout.com:80').validate_uri_format).to eq(true)
      expect(fd('https://eviltrout.com:443').validate_uri_format).to eq(true)
    end
  end

  describe "https cache" do
    it 'will cache https lookups' do

      FinalDestination.clear_https_cache!("wikipedia.com")

      fd_stub_request(:head, "http://wikipedia.com/image.png")
        .to_return(status: 302, body: "", headers: { location: 'https://wikipedia.com/image.png' })

      fd_stub_request(:head, "https://wikipedia.com/image.png")

      fd('http://wikipedia.com/image.png').resolve

      fd_stub_request(:head, "https://wikipedia.com/image2.png")

      fd('http://wikipedia.com/image2.png').resolve
    end
  end

  describe "#normalized_url" do
    it "correctly normalizes url" do
      fragment_url = "https://eviltrout.com/2016/02/25/fixing-android-performance.html#discourse-comments"

      expect(fd(fragment_url).normalized_url.to_s).to eq(fragment_url)

      expect(fd("https://eviltrout.com?s=180&#038;d=mm&#038;r=g").normalized_url.to_s)
        .to eq("https://eviltrout.com?s=180&#038;d=mm&%23038;r=g")

      expect(fd("http://example.com/?a=\11\15").normalized_url.to_s).to eq("http://example.com/?a=%09%0D")

      expect(fd("https://ru.wikipedia.org/wiki/%D0%A1%D0%B2%D0%BE%D0%B1%D0%BE").normalized_url.to_s)
        .to eq('https://ru.wikipedia.org/wiki/%D0%A1%D0%B2%D0%BE%D0%B1%D0%BE')

      expect(fd('https://ru.wikipedia.org/wiki/Свобо').normalized_url.to_s)
        .to eq('https://ru.wikipedia.org/wiki/%D0%A1%D0%B2%D0%BE%D0%B1%D0%BE')
    end
  end
end
-												DEV: use #frozen_string_literal: true on all spec

This change both speeds up specs (less strings to allocate) and helps catch
cases where methods in Discourse are mutating inputs.

Overall we will be migrating everything to use #frozen_string_literal: true
it will take a while, but this is the first and safest move in this direction

											
										
										
											2019-04-29 20:27:42 -04:00
+								# frozen_string_literal: true
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								require 'final_destination'
-												Add RSpec 4 compatibility (#17652)

* Remove outdated option

https://github.com/rspec/rspec-core/commit/04078317ba6577699d06cf4dccf014254dcde7a6

* Use the non-globally exposed RSpec syntax

https://github.com/rspec/rspec-core/pull/2803

* Use the non-globally exposed RSpec syntax, cont

https://github.com/rspec/rspec-core/pull/2803

* Comply to strict predicate matchers

See:
 - https://github.com/rspec/rspec-expectations/pull/1195
 - https://github.com/rspec/rspec-expectations/pull/1196
 - https://github.com/rspec/rspec-expectations/pull/1277

											
										
										
											2022-07-27 22:27:38 -04:00
+								RSpec.describe FinalDestination do
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								  let(:opts) do
-												FIX: Support for skipping redirects on certain domains (like steam)

											
										
										
											2017-06-26 15:38:23 -04:00
+								    {
 								      ignore_redirects: ['https://ignore-me.com'],
-												DEV: Oneboxer wildcard subdomains (#13015)

* DEV: Allow wildcards in Oneboxer optional domain Site Settings

Allows a wildcard to be used as a subdomain on Oneboxer-related SiteSettings, e.g.:

- `force_get_hosts`
- `cache_onebox_response_body_domains`
- `force_custom_user_agent_hosts`

* DEV: fix typos

* FIX: Try doing a GET after receiving a 500 error from a HEAD

By default we try to do a `HEAD` requests. If this results in a 500 error response, we should try to do a `GET`

* DEV: `force_get_hosts` should be a hidden setting

* DEV: Oneboxer Strategies

Have an alternative oneboxing ‘strategy’ (i.e., set of options) to use when an attempt to generate a Onebox fails. Keep track of any non-default strategies that were used on a particular host, and use that strategy for that host in the future.

Initially, the alternate strategy (`force_get_and_ua`) forces the FinalDestination step of Oneboxing to do a `GET` rather than `HEAD`, and forces a custom user agent.

* DEV: change stubbed return code

The stubbed status code needs to be a value not recognized by FinalDestination
											
										
										
											2021-05-13 15:48:35 -04:00
+								      force_get_hosts: ['https://force.get.com', 'https://*.ihaveawildcard.com/'],
-												FIX: allows onebox to force GET hosts returning wrong headers on HEAD


											
										
										
											2017-08-08 05:44:27 -04:00
-												FIX: preserve github fragment URL

											
										
										
											2018-12-19 01:27:07 -05:00
+								      preserve_fragment_url_hosts: ['https://eviltrout.com'],
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    }
 								  end
 								  let(:doc_response) do
-												Use webmock to stub external web requests.

											
										
										
											2017-05-26 03:19:09 -04:00
+								    {
 								      status: 200,
 								      headers: { "Content-Type" => "text/html" }
 								    }
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								  end
-												FEATURE: Onebox can match engines based on the content_type (#13876)

* FEATURE: Onebox can match engines based on the content_type

`FinalDestination` now returns the `content_type` of a resolved URL.

`Oneboxer` passes this value to `Onebox` itself. Onebox engines can now specify a `matches_content_type` regex of content_types that the engine can handle, regardless of the URL.

`ImageOnebox` will match URLs with a content type of `image/png`, `jpg`, `gif`, `bmp`, `tif`, etc.

This will allow images that exist at a URL without a file type extension to be correctly rendered, assuming a valid `content_type` is returned.
											
										
										
											2021-07-30 13:36:30 -04:00
+								  let(:image_response) do
 								    {
 								      status: 200,
 								      headers: { "Content-Type" => "image/jpeg" }
 								    }
 								  end
-												SECURITY: Onebox response timeout and size limit (#15927)

Validation to ensure that Onebox request is no longer than 10 seconds and response size is not bigger than 1 MB
											
										
										
											2022-02-13 20:11:09 -05:00
+								  let(:body_response) do
 								    {
 								      status: 200,
 								      body: "<body>test</body>"
 								    }
 								  end
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								  def fd_stub_request(method, url)
 								    uri = URI.parse(url)
 								    host = uri.hostname
 								    ip = "1.2.3.4"
 								    # In Excon we pass the IP in the URL, so we need to stub
 								    # that version as well
 								    uri.hostname = "HOSTNAME_PLACEHOLDER"
 								    matcher = Regexp.escape(uri.to_s).sub(
 								      "HOSTNAME_PLACEHOLDER",
 								      "(#{Regexp.escape(host)}|#{Regexp.escape(ip)})"
 								    )
 								    stub_request(method, /\A#{matcher}\z/).with(headers: { "Host" => host })
 								  end
-												FIX: Follow the canonical URL when importing a remote topic. (#14489)

FinalDestination now supports the `follow_canonical` option, which will perform an initial GET request, parse the canonical link if present, and perform a HEAD request to it.

We use this mode during embeds to avoid treating URLs with different query parameters as different topics.
											
										
										
											2021-10-01 11:48:21 -04:00
+								  def canonical_follow(from, dest)
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								    fd_stub_request(:get, from).to_return(
-												FIX: Follow the canonical URL when importing a remote topic. (#14489)

FinalDestination now supports the `follow_canonical` option, which will perform an initial GET request, parse the canonical link if present, and perform a HEAD request to it.

We use this mode during embeds to avoid treating URLs with different query parameters as different topics.
											
										
										
											2021-10-01 11:48:21 -04:00
+								      status: 200,
 								      body: "<head><link rel=\"canonical\" href=\"#{dest}\"></head>"
 								    )
 								  end
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								  def redirect_response(from, dest)
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								    fd_stub_request(:head, from).to_return(
-												Revert "FIX: Use Excon to do its own stubbing"

This reverts commit 80af54460a81bf1593cacb1f0b23e0e0a42c8076.

											
										
										
											2017-05-26 01:04:25 -04:00
+								      status: 302,
 								      headers: { "Location" => dest }
 								    )
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								  end
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								  def fd(url)
 								    FinalDestination.new(url, opts)
 								  end
-												FIX: Ignore OneBox blacklisted domains.

											
										
										
											2018-08-26 10:31:02 -04:00
+								  it 'correctly parses ignored hostnames' do
 								    fd = FinalDestination.new('https://meta.discourse.org',
 								      ignore_redirects: ['http://google.com', 'youtube.com', 'https://meta.discourse.org', '://bing.com']
 								    )
 								    expect(fd.ignored).to eq(['test.localhost', 'google.com', 'meta.discourse.org'])
 								  end
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								  describe '.resolve' do
 								    it "has a ready status code before anything happens" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								      expect(fd('https://eviltrout.com').status).to eq(:ready)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    end
-												FIX: Don't raise an error if reading from URL timeout.

											
										
										
											2017-09-27 02:52:49 -04:00
+								    it "returns nil for an invalid url" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								      expect(fd(nil).resolve).to be_nil
 								      expect(fd('asdf').resolve).to be_nil
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    end
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								    it "returns nil for unresolvable url" do
 								      FinalDestination::SSRFDetector.stubs(:lookup_ips).raises(SocketError)
 								      expect(fd("https://example.com").resolve).to eq(nil)
 								    end
 								    it "returns nil for url timeout" do
 								      FinalDestination::SSRFDetector.stubs(:lookup_ips).raises(Timeout::Error)
 								      expect(fd("https://example.com").resolve).to eq(nil)
 								    end
-												FIX: Don't raise an error if reading from URL timeout.

											
										
										
											2017-09-27 02:52:49 -04:00
+								    it "returns nil when read timeouts" do
 								      Excon.expects(:public_send).raises(Excon::Errors::Timeout)
 								      expect(fd('https://discourse.org').resolve).to eq(nil)
 								    end
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    context "without redirects" do
 								      before do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, "https://eviltrout.com/").to_return(doc_response)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								      end
 								      it "returns the final url" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								        final = FinalDestination.new('https://eviltrout.com', opts)
 								        expect(final.resolve.to_s).to eq('https://eviltrout.com')
 								        expect(final.redirected?).to eq(false)
 								        expect(final.status).to eq(:resolved)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								      end
-												FIX: Use same code path for downloading images

											
										
										
											2017-05-23 13:31:20 -04:00
+								    end
-												FIX: Support for skipping redirects on certain domains (like steam)

											
										
										
											2017-06-26 15:38:23 -04:00
+								    it "ignores redirects" do
 								      final = FinalDestination.new('https://ignore-me.com/some-url', opts)
 								      expect(final.resolve.to_s).to eq('https://ignore-me.com/some-url')
 								      expect(final.redirected?).to eq(false)
 								      expect(final.status).to eq(:resolved)
 								    end
-												DEV: Use proper wording for contexts in specs

											
										
										
											2022-07-27 12:14:14 -04:00
+								    context "with underscores in URLs" do
-												FIX: Use same code path for downloading images

											
										
										
											2017-05-23 13:31:20 -04:00
+								      before do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, 'https://some_thing.example.com').to_return(doc_response)
-												FIX: Use same code path for downloading images

											
										
										
											2017-05-23 13:31:20 -04:00
+								      end
 								      it "doesn't raise errors with underscores in urls" do
 								        final = FinalDestination.new('https://some_thing.example.com', opts)
 								        expect(final.resolve.to_s).to eq('https://some_thing.example.com')
 								        expect(final.redirected?).to eq(false)
 								        expect(final.status).to eq(:resolved)
 								      end
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    end
 								    context "with a couple of redirects" do
 								      before do
-												Revert "FIX: Use Excon to do its own stubbing"

This reverts commit 80af54460a81bf1593cacb1f0b23e0e0a42c8076.

											
										
										
											2017-05-26 01:04:25 -04:00
+								        redirect_response("https://eviltrout.com", "https://codinghorror.com/blog")
 								        redirect_response("https://codinghorror.com/blog", "https://discourse.org")
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, "https://discourse.org").to_return(doc_response)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								      end
 								      it "returns the final url" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								        final = FinalDestination.new('https://eviltrout.com', opts)
 								        expect(final.resolve.to_s).to eq('https://discourse.org')
 								        expect(final.redirected?).to eq(true)
 								        expect(final.status).to eq(:resolved)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								      end
 								    end
 								    context "with too many redirects" do
 								      before do
-												Revert "FIX: Use Excon to do its own stubbing"

This reverts commit 80af54460a81bf1593cacb1f0b23e0e0a42c8076.

											
										
										
											2017-05-26 01:04:25 -04:00
+								        redirect_response("https://eviltrout.com", "https://codinghorror.com/blog")
 								        redirect_response("https://codinghorror.com/blog", "https://discourse.org")
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, "https://discourse.org").to_return(doc_response)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								      end
 								      it "returns the final url" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								        final = FinalDestination.new('https://eviltrout.com', opts.merge(max_redirects: 1))
 								        expect(final.resolve).to be_nil
 								        expect(final.redirected?).to eq(true)
 								        expect(final.status).to eq(:too_many_redirects)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								      end
 								    end
 								    context "with a redirect to an internal IP" do
 								      before do
-												Revert "FIX: Use Excon to do its own stubbing"

This reverts commit 80af54460a81bf1593cacb1f0b23e0e0a42c8076.

											
										
										
											2017-05-26 01:04:25 -04:00
+								        redirect_response("https://eviltrout.com", "https://private-host.com")
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        FinalDestination::SSRFDetector.stubs(:lookup_and_filter_ips).with("eviltrout.com").returns(["1.2.3.4"])
 								        FinalDestination::SSRFDetector.stubs(:lookup_and_filter_ips).with("private-host.com").raises(FinalDestination::SSRFDetector::DisallowedIpError)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								      end
 								      it "returns the final url" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								        final = FinalDestination.new('https://eviltrout.com', opts)
 								        expect(final.resolve).to be_nil
 								        expect(final.redirected?).to eq(true)
 								        expect(final.status).to eq(:invalid_address)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								      end
 								    end
-												FIX: If HEAD is not supported, try GET. Also set cookies

											
										
										
											2017-06-06 13:53:49 -04:00
-												FIX: do not follow redirect on same host with path /login or /session

											
										
										
											2019-08-07 06:56:03 -04:00
+								    context "with a redirect to login path" do
 								      before do
 								        redirect_response("https://eviltrout.com/t/xyz/1", "https://eviltrout.com/login")
 								      end
 								      it "does not follow redirect" do
 								        final = FinalDestination.new('https://eviltrout.com/t/xyz/1', opts)
 								        expect(final.resolve.to_s).to eq('https://eviltrout.com/t/xyz/1')
 								        expect(final.redirected?).to eq(false)
 								        expect(final.status).to eq(:resolved)
 								      end
 								    end
-												SECURITY: Onebox response timeout and size limit (#15927)

Validation to ensure that Onebox request is no longer than 10 seconds and response size is not bigger than 1 MB
											
										
										
											2022-02-13 20:11:09 -05:00
+								    it 'raises error when response is too big' do
 								      stub_const(described_class, "MAX_REQUEST_SIZE_BYTES", 1) do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:get, "https://codinghorror.com/blog").to_return(body_response)
-												SECURITY: Onebox response timeout and size limit (#15927)

Validation to ensure that Onebox request is no longer than 10 seconds and response size is not bigger than 1 MB
											
										
										
											2022-02-13 20:11:09 -05:00
+								        final = FinalDestination.new('https://codinghorror.com/blog', opts.merge(follow_canonical: true))
 								        expect { final.resolve }.to raise_error(Excon::Errors::ExpectationFailed, "response size too big: https://codinghorror.com/blog")
 								      end
 								    end
 								    it 'raises error when response is too slow' do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								      fd_stub_request(:get, "https://codinghorror.com/blog").to_return(lambda { |request| freeze_time(11.seconds.from_now) ; body_response })
-												SECURITY: Onebox response timeout and size limit (#15927)

Validation to ensure that Onebox request is no longer than 10 seconds and response size is not bigger than 1 MB
											
										
										
											2022-02-13 20:11:09 -05:00
+								      final = FinalDestination.new('https://codinghorror.com/blog', opts.merge(follow_canonical: true))
 								      expect { final.resolve }.to raise_error(Excon::Errors::ExpectationFailed, "connect timeout reached: https://codinghorror.com/blog")
 								    end
-												DEV: Use proper wording for contexts in specs

											
										
										
											2022-07-27 12:14:14 -04:00
+								    context 'when following canonical links' do
-												FIX: Follow the canonical URL when importing a remote topic. (#14489)

FinalDestination now supports the `follow_canonical` option, which will perform an initial GET request, parse the canonical link if present, and perform a HEAD request to it.

We use this mode during embeds to avoid treating URLs with different query parameters as different topics.
											
										
										
											2021-10-01 11:48:21 -04:00
+								      it 'resolves the canonical link as the final destination' do
 								        canonical_follow("https://eviltrout.com", "https://codinghorror.com/blog")
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, "https://codinghorror.com/blog").to_return(doc_response)
-												FIX: Follow the canonical URL when importing a remote topic. (#14489)

FinalDestination now supports the `follow_canonical` option, which will perform an initial GET request, parse the canonical link if present, and perform a HEAD request to it.

We use this mode during embeds to avoid treating URLs with different query parameters as different topics.
											
										
										
											2021-10-01 11:48:21 -04:00
 								        final = FinalDestination.new('https://eviltrout.com', opts.merge(follow_canonical: true))
 								        expect(final.resolve.to_s).to eq("https://codinghorror.com/blog")
 								        expect(final.redirected?).to eq(false)
 								        expect(final.status).to eq(:resolved)
 								      end
-												FIX: Canonical URLs may be relative (#14825)

FinalDestination's follow_canonical mode used for embedded topics should work when canonical URLs are relative, as specified in [RFC 6596](https://datatracker.ietf.org/doc/html/rfc6596)
											
										
										
											2021-11-05 13:20:14 -04:00
+								      it 'resolves the canonical link when the URL is relative' do
 								        host = "https://codinghorror.com"
 								        canonical_follow("#{host}/blog", "/blog/canonical")
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, "#{host}/blog/canonical").to_return(doc_response)
-												FIX: Canonical URLs may be relative (#14825)

FinalDestination's follow_canonical mode used for embedded topics should work when canonical URLs are relative, as specified in [RFC 6596](https://datatracker.ietf.org/doc/html/rfc6596)
											
										
										
											2021-11-05 13:20:14 -04:00
 								        final = FinalDestination.new("#{host}/blog", opts.merge(follow_canonical: true))
 								        expect(final.resolve.to_s).to eq("#{host}/blog/canonical")
 								        expect(final.redirected?).to eq(false)
 								        expect(final.status).to eq(:resolved)
 								      end
 								      it 'resolves the canonical link when the URL is relative and does not start with the / symbol' do
 								        host = "https://codinghorror.com"
 								        canonical_follow("#{host}/blog", "blog/canonical")
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, "#{host}/blog/canonical").to_return(doc_response)
-												FIX: Canonical URLs may be relative (#14825)

FinalDestination's follow_canonical mode used for embedded topics should work when canonical URLs are relative, as specified in [RFC 6596](https://datatracker.ietf.org/doc/html/rfc6596)
											
										
										
											2021-11-05 13:20:14 -04:00
 								        final = FinalDestination.new("#{host}/blog", opts.merge(follow_canonical: true))
 								        expect(final.resolve.to_s).to eq("#{host}/blog/canonical")
 								        expect(final.redirected?).to eq(false)
 								        expect(final.status).to eq(:resolved)
 								      end
-												FIX: Follow the canonical URL when importing a remote topic. (#14489)

FinalDestination now supports the `follow_canonical` option, which will perform an initial GET request, parse the canonical link if present, and perform a HEAD request to it.

We use this mode during embeds to avoid treating URLs with different query parameters as different topics.
											
										
										
											2021-10-01 11:48:21 -04:00
+								      it "does not follow the canonical link if it's the same as the current URL" do
 								        canonical_follow("https://eviltrout.com", "https://eviltrout.com")
 								        final = FinalDestination.new('https://eviltrout.com', opts.merge(follow_canonical: true))
 								        expect(final.resolve.to_s).to eq("https://eviltrout.com")
 								        expect(final.redirected?).to eq(false)
 								        expect(final.status).to eq(:resolved)
 								      end
 								      it "does not follow the canonical link if it's invalid" do
 								        canonical_follow("https://eviltrout.com", "")
 								        final = FinalDestination.new('https://eviltrout.com', opts.merge(follow_canonical: true))
 								        expect(final.resolve.to_s).to eq("https://eviltrout.com")
 								        expect(final.redirected?).to eq(false)
 								        expect(final.status).to eq(:resolved)
 								      end
 								    end
-												DEV: Use proper wording for contexts in specs

											
										
										
											2022-07-27 12:14:14 -04:00
+								    context "when forcing GET" do
-												FIX: allows onebox to force GET hosts returning wrong headers on HEAD


											
										
										
											2017-08-08 05:44:27 -04:00
+								      it "will do a GET when forced" do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        url = 'https://force.get.com/posts?page=4'
 								        get_stub = fd_stub_request(:get, url)
 								        head_stub = fd_stub_request(:head, url)
 								        final = FinalDestination.new(url, opts)
 								        expect(final.resolve.to_s).to eq(url)
-												FIX: allows onebox to force GET hosts returning wrong headers on HEAD


											
										
										
											2017-08-08 05:44:27 -04:00
+								        expect(final.status).to eq(:resolved)
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        expect(get_stub).to have_been_requested
 								        expect(head_stub).to_not have_been_requested
-												FIX: allows onebox to force GET hosts returning wrong headers on HEAD


											
										
										
											2017-08-08 05:44:27 -04:00
+								      end
 								      it "will do a HEAD if not forced" do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        url = 'https://eviltrout.com/posts?page=2'
 								        get_stub = fd_stub_request(:get, url)
 								        head_stub = fd_stub_request(:head, url)
 								        final = FinalDestination.new(url, opts)
 								        expect(final.resolve.to_s).to eq(url)
-												FIX: allows onebox to force GET hosts returning wrong headers on HEAD


											
										
										
											2017-08-08 05:44:27 -04:00
+								        expect(final.status).to eq(:resolved)
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        expect(get_stub).to_not have_been_requested
 								        expect(head_stub).to have_been_requested
-												FIX: allows onebox to force GET hosts returning wrong headers on HEAD


											
										
										
											2017-08-08 05:44:27 -04:00
+								      end
-												DEV: Oneboxer wildcard subdomains (#13015)

* DEV: Allow wildcards in Oneboxer optional domain Site Settings

Allows a wildcard to be used as a subdomain on Oneboxer-related SiteSettings, e.g.:

- `force_get_hosts`
- `cache_onebox_response_body_domains`
- `force_custom_user_agent_hosts`

* DEV: fix typos

* FIX: Try doing a GET after receiving a 500 error from a HEAD

By default we try to do a `HEAD` requests. If this results in a 500 error response, we should try to do a `GET`

* DEV: `force_get_hosts` should be a hidden setting

* DEV: Oneboxer Strategies

Have an alternative oneboxing ‘strategy’ (i.e., set of options) to use when an attempt to generate a Onebox fails. Keep track of any non-default strategies that were used on a particular host, and use that strategy for that host in the future.

Initially, the alternate strategy (`force_get_and_ua`) forces the FinalDestination step of Oneboxing to do a `GET` rather than `HEAD`, and forces a custom user agent.

* DEV: change stubbed return code

The stubbed status code needs to be a value not recognized by FinalDestination
											
										
										
											2021-05-13 15:48:35 -04:00
 								      it "will do a GET when forced on a wildcard subdomain" do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        url = 'https://any-subdomain.ihaveawildcard.com/some/other/content'
 								        get_stub = fd_stub_request(:get, url)
 								        head_stub = fd_stub_request(:head, url)
 								        final = FinalDestination.new(url, opts)
 								        expect(final.resolve.to_s).to eq(url)
-												DEV: Oneboxer wildcard subdomains (#13015)

* DEV: Allow wildcards in Oneboxer optional domain Site Settings

Allows a wildcard to be used as a subdomain on Oneboxer-related SiteSettings, e.g.:

- `force_get_hosts`
- `cache_onebox_response_body_domains`
- `force_custom_user_agent_hosts`

* DEV: fix typos

* FIX: Try doing a GET after receiving a 500 error from a HEAD

By default we try to do a `HEAD` requests. If this results in a 500 error response, we should try to do a `GET`

* DEV: `force_get_hosts` should be a hidden setting

* DEV: Oneboxer Strategies

Have an alternative oneboxing ‘strategy’ (i.e., set of options) to use when an attempt to generate a Onebox fails. Keep track of any non-default strategies that were used on a particular host, and use that strategy for that host in the future.

Initially, the alternate strategy (`force_get_and_ua`) forces the FinalDestination step of Oneboxing to do a `GET` rather than `HEAD`, and forces a custom user agent.

* DEV: change stubbed return code

The stubbed status code needs to be a value not recognized by FinalDestination
											
										
										
											2021-05-13 15:48:35 -04:00
+								        expect(final.status).to eq(:resolved)
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        expect(get_stub).to have_been_requested
 								        expect(head_stub).to_not have_been_requested
-												DEV: Oneboxer wildcard subdomains (#13015)

* DEV: Allow wildcards in Oneboxer optional domain Site Settings

Allows a wildcard to be used as a subdomain on Oneboxer-related SiteSettings, e.g.:

- `force_get_hosts`
- `cache_onebox_response_body_domains`
- `force_custom_user_agent_hosts`

* DEV: fix typos

* FIX: Try doing a GET after receiving a 500 error from a HEAD

By default we try to do a `HEAD` requests. If this results in a 500 error response, we should try to do a `GET`

* DEV: `force_get_hosts` should be a hidden setting

* DEV: Oneboxer Strategies

Have an alternative oneboxing ‘strategy’ (i.e., set of options) to use when an attempt to generate a Onebox fails. Keep track of any non-default strategies that were used on a particular host, and use that strategy for that host in the future.

Initially, the alternate strategy (`force_get_and_ua`) forces the FinalDestination step of Oneboxing to do a `GET` rather than `HEAD`, and forces a custom user agent.

* DEV: change stubbed return code

The stubbed status code needs to be a value not recognized by FinalDestination
											
										
										
											2021-05-13 15:48:35 -04:00
+								      end
 								      it "will do a HEAD if on a subdomain of a forced get domain without a wildcard" do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        url = 'https://particularly.eviltrout.com/has/a/secret/plan'
 								        get_stub = fd_stub_request(:get, url)
 								        head_stub = fd_stub_request(:head, url)
 								        final = FinalDestination.new(url, opts)
 								        expect(final.resolve.to_s).to eq(url)
-												DEV: Oneboxer wildcard subdomains (#13015)

* DEV: Allow wildcards in Oneboxer optional domain Site Settings

Allows a wildcard to be used as a subdomain on Oneboxer-related SiteSettings, e.g.:

- `force_get_hosts`
- `cache_onebox_response_body_domains`
- `force_custom_user_agent_hosts`

* DEV: fix typos

* FIX: Try doing a GET after receiving a 500 error from a HEAD

By default we try to do a `HEAD` requests. If this results in a 500 error response, we should try to do a `GET`

* DEV: `force_get_hosts` should be a hidden setting

* DEV: Oneboxer Strategies

Have an alternative oneboxing ‘strategy’ (i.e., set of options) to use when an attempt to generate a Onebox fails. Keep track of any non-default strategies that were used on a particular host, and use that strategy for that host in the future.

Initially, the alternate strategy (`force_get_and_ua`) forces the FinalDestination step of Oneboxing to do a `GET` rather than `HEAD`, and forces a custom user agent.

* DEV: change stubbed return code

The stubbed status code needs to be a value not recognized by FinalDestination
											
										
										
											2021-05-13 15:48:35 -04:00
+								        expect(final.status).to eq(:resolved)
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        expect(get_stub).to_not have_been_requested
 								        expect(head_stub).to have_been_requested
-												DEV: Oneboxer wildcard subdomains (#13015)

* DEV: Allow wildcards in Oneboxer optional domain Site Settings

Allows a wildcard to be used as a subdomain on Oneboxer-related SiteSettings, e.g.:

- `force_get_hosts`
- `cache_onebox_response_body_domains`
- `force_custom_user_agent_hosts`

* DEV: fix typos

* FIX: Try doing a GET after receiving a 500 error from a HEAD

By default we try to do a `HEAD` requests. If this results in a 500 error response, we should try to do a `GET`

* DEV: `force_get_hosts` should be a hidden setting

* DEV: Oneboxer Strategies

Have an alternative oneboxing ‘strategy’ (i.e., set of options) to use when an attempt to generate a Onebox fails. Keep track of any non-default strategies that were used on a particular host, and use that strategy for that host in the future.

Initially, the alternate strategy (`force_get_and_ua`) forces the FinalDestination step of Oneboxing to do a `GET` rather than `HEAD`, and forces a custom user agent.

* DEV: change stubbed return code

The stubbed status code needs to be a value not recognized by FinalDestination
											
										
										
											2021-05-13 15:48:35 -04:00
+								      end
-												FIX: allows onebox to force GET hosts returning wrong headers on HEAD


											
										
										
											2017-08-08 05:44:27 -04:00
+								    end
-												DEV: Use proper wording for contexts in specs

											
										
										
											2022-07-27 12:14:14 -04:00
+								    context "when HEAD not supported" do
-												FIX: If HEAD is not supported, try GET. Also set cookies

											
										
										
											2017-06-06 13:53:49 -04:00
+								      before do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:get, 'https://eviltrout.com').to_return(
-												FIX: If HEAD is not supported, try GET. Also set cookies

											
										
										
											2017-06-06 13:53:49 -04:00
+								          status: 301,
 								          headers: {
 								            "Location" => 'https://discourse.org',
 								            'Set-Cookie' => 'evil=trout'
 								          }
 								        )
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, 'https://discourse.org')
-												FIX: If HEAD is not supported, try GET. Also set cookies

											
										
										
											2017-06-06 13:53:49 -04:00
+								      end
 								      context "when the status code is 405" do
 								        before do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								          fd_stub_request(:head, 'https://eviltrout.com').to_return(status: 405)
-												FIX: If HEAD is not supported, try GET. Also set cookies

											
										
										
											2017-06-06 13:53:49 -04:00
+								        end
 								        it "will try a GET" do
 								          final = FinalDestination.new('https://eviltrout.com', opts)
 								          expect(final.resolve.to_s).to eq('https://discourse.org')
 								          expect(final.status).to eq(:resolved)
 								          expect(final.cookie).to eq('evil=trout')
 								        end
 								      end
 								      context "when the status code is 501" do
 								        before do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								          fd_stub_request(:head, 'https://eviltrout.com').to_return(status: 501)
-												FIX: If HEAD is not supported, try GET. Also set cookies

											
										
										
											2017-06-06 13:53:49 -04:00
+								        end
 								        it "will try a GET" do
 								          final = FinalDestination.new('https://eviltrout.com', opts)
 								          expect(final.resolve.to_s).to eq('https://discourse.org')
 								          expect(final.status).to eq(:resolved)
 								          expect(final.cookie).to eq('evil=trout')
 								        end
 								      end
-												FIX: Cookies header didn't have the right format

											
										
										
											2018-02-18 10:08:07 -05:00
 								      it "correctly extracts cookies during GET" do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, "https://eviltrout.com").to_return(status: 405)
-												FIX: Cookies header didn't have the right format

											
										
										
											2018-02-18 10:08:07 -05:00
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:get, "https://eviltrout.com")
-												FIX: Cookies header didn't have the right format

											
										
										
											2018-02-18 10:08:07 -05:00
+								          .to_return(status: 302, body: "" , headers: {
 								            "Location" => "https://eviltrout.com",
 								            "Set-Cookie" => ["foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com",
 								                             "bar=1",
 								                             "baz=2; expires=Tue, 19-Feb-2019 10:14:24 GMT; path=/; domain=eviltrout.com"]
 								          })
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, "https://eviltrout.com")
-												FIX: Cookies header didn't have the right format

											
										
										
											2018-02-18 10:08:07 -05:00
+								          .with(headers: { "Cookie" => "bar=1; baz=2; foo=219ffwef9w0f" })
 								        final = FinalDestination.new("https://eviltrout.com", opts)
 								        expect(final.resolve.to_s).to eq("https://eviltrout.com")
 								        expect(final.status).to eq(:resolved)
 								        expect(final.cookie).to eq("bar=1; baz=2; foo=219ffwef9w0f")
 								      end
 								    end
 								    it "should use the correct format for cookies when there is only one cookie" do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								      fd_stub_request(:head, "https://eviltrout.com")
-												FIX: don't double request when downloading a file

											
										
										
											2018-02-24 06:35:57 -05:00
+								        .to_return(status: 302, headers: {
-												FIX: Cookies header didn't have the right format

											
										
										
											2018-02-18 10:08:07 -05:00
+								          "Location" => "https://eviltrout.com",
 								          "Set-Cookie" => "foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com"
 								        })
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								      fd_stub_request(:head, "https://eviltrout.com")
-												FIX: Cookies header didn't have the right format

											
										
										
											2018-02-18 10:08:07 -05:00
+								        .with(headers: { "Cookie" => "foo=219ffwef9w0f" })
 								      final = FinalDestination.new("https://eviltrout.com", opts)
 								      expect(final.resolve.to_s).to eq("https://eviltrout.com")
 								      expect(final.status).to eq(:resolved)
 								      expect(final.cookie).to eq("foo=219ffwef9w0f")
 								    end
 								    it "should use the correct format for cookies when there are multiple cookies" do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								      fd_stub_request(:head, "https://eviltrout.com")
-												FIX: don't double request when downloading a file

											
										
										
											2018-02-24 06:35:57 -05:00
+								        .to_return(status: 302, headers: {
-												FIX: Cookies header didn't have the right format

											
										
										
											2018-02-18 10:08:07 -05:00
+								          "Location" => "https://eviltrout.com",
 								          "Set-Cookie" => ["foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com",
 								                           "bar=1",
 								                           "baz=2; expires=Tue, 19-Feb-2019 10:14:24 GMT; path=/; domain=eviltrout.com"]
 								        })
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								      fd_stub_request(:head, "https://eviltrout.com")
-												FIX: Cookies header didn't have the right format

											
										
										
											2018-02-18 10:08:07 -05:00
+								        .with(headers: { "Cookie" => "bar=1; baz=2; foo=219ffwef9w0f" })
 								      final = FinalDestination.new("https://eviltrout.com", opts)
 								      expect(final.resolve.to_s).to eq("https://eviltrout.com")
 								      expect(final.status).to eq(:resolved)
 								      expect(final.cookie).to eq("bar=1; baz=2; foo=219ffwef9w0f")
-												FIX: If HEAD is not supported, try GET. Also set cookies

											
										
										
											2017-06-06 13:53:49 -04:00
+								    end
-												FIX: preserve github fragment URL

											
										
										
											2018-12-19 01:27:07 -05:00
 								    it "persists fragment url" do
 								      origin_url = "https://eviltrout.com/origin/lib/code/foobar.rb"
 								      upstream_url = "https://eviltrout.com/upstream/lib/code/foobar.rb"
 								      redirect_response(origin_url, upstream_url)
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								      fd_stub_request(:head, upstream_url).to_return(doc_response)
-												FIX: preserve github fragment URL

											
										
										
											2018-12-19 01:27:07 -05:00
 								      final = FinalDestination.new("#{origin_url}#L154-L205", opts)
 								      expect(final.resolve.to_s).to eq("#{upstream_url}#L154-L205")
 								      expect(final.status).to eq(:resolved)
 								    end
-												FEATURE: Onebox can match engines based on the content_type (#13876)

* FEATURE: Onebox can match engines based on the content_type

`FinalDestination` now returns the `content_type` of a resolved URL.

`Oneboxer` passes this value to `Onebox` itself. Onebox engines can now specify a `matches_content_type` regex of content_types that the engine can handle, regardless of the URL.

`ImageOnebox` will match URLs with a content type of `image/png`, `jpg`, `gif`, `bmp`, `tif`, etc.

This will allow images that exist at a URL without a file type extension to be correctly rendered, assuming a valid `content_type` is returned.
											
										
										
											2021-07-30 13:36:30 -04:00
-												DEV: Use proper wording for contexts in specs

											
										
										
											2022-07-27 12:14:14 -04:00
+								    context "with content_type" do
-												FEATURE: Onebox can match engines based on the content_type (#13876)

* FEATURE: Onebox can match engines based on the content_type

`FinalDestination` now returns the `content_type` of a resolved URL.

`Oneboxer` passes this value to `Onebox` itself. Onebox engines can now specify a `matches_content_type` regex of content_types that the engine can handle, regardless of the URL.

`ImageOnebox` will match URLs with a content type of `image/png`, `jpg`, `gif`, `bmp`, `tif`, etc.

This will allow images that exist at a URL without a file type extension to be correctly rendered, assuming a valid `content_type` is returned.
											
										
										
											2021-07-30 13:36:30 -04:00
+								      before do
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								        fd_stub_request(:head, "https://eviltrout.com/this/is/an/image").to_return(image_response)
-												FEATURE: Onebox can match engines based on the content_type (#13876)

* FEATURE: Onebox can match engines based on the content_type

`FinalDestination` now returns the `content_type` of a resolved URL.

`Oneboxer` passes this value to `Onebox` itself. Onebox engines can now specify a `matches_content_type` regex of content_types that the engine can handle, regardless of the URL.

`ImageOnebox` will match URLs with a content type of `image/png`, `jpg`, `gif`, `bmp`, `tif`, etc.

This will allow images that exist at a URL without a file type extension to be correctly rendered, assuming a valid `content_type` is returned.
											
										
										
											2021-07-30 13:36:30 -04:00
+								      end
 								      it "returns a content_type" do
 								        final = FinalDestination.new("https://eviltrout.com/this/is/an/image", opts)
 								        expect(final.resolve.to_s).to eq("https://eviltrout.com/this/is/an/image")
 								        expect(final.content_type).to eq("image/jpeg")
 								        expect(final.status).to eq(:resolved)
 								      end
 								    end
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								  end
-												FIX: Don’t raise an error on onebox timeouts

Currently when generating oneboxes if the connection timeouts and we’re
using the `FinalDestination#get` method, then it raises an exception.

We already catch this exception when using the
`FinalDestination#resolve` method so this patch just applies the same
logic to `FinalDestination#get`.

											
										
										
											2022-07-21 09:58:17 -04:00
+								  describe '#get' do
 								    let(:fd) { FinalDestination.new("http://wikipedia.com", opts.merge(verbose: true)) }
-												PERF: ability to crawl for titles without extra HEAD req

Also, introduces a much more aggressive timeout for title crawling
and introduces gzip to body that is crawled

											
										
										
											2018-01-28 23:36:52 -05:00
-												DEV: Fix flaky `FinalDestination` specs

											
										
										
											2022-07-27 06:09:47 -04:00
+								    before do
 								      described_class.clear_https_cache!("wikipedia.com")
 								    end
-												FIX: Don’t raise an error on onebox timeouts

Currently when generating oneboxes if the connection timeouts and we’re
using the `FinalDestination#get` method, then it raises an exception.

We already catch this exception when using the
`FinalDestination#resolve` method so this patch just applies the same
logic to `FinalDestination#get`.

											
										
										
											2022-07-21 09:58:17 -04:00
+								    context "when there is a redirect" do
 								      before do
 								        stub_request(:get, "http://wikipedia.com/").
 								          to_return(status: 302, body: "" , headers: { "location" => "https://wikipedia.com/" })
 								        # webmock does not do chunks
 								        stub_request(:get, "https://wikipedia.com/").
 								          to_return(status: 200, body: "<html><head>" , headers: {})
 								      end
-												DEV: Fix `FinalDestination` specs (#17642)


											
										
										
											2022-07-25 06:38:54 -04:00
+								      after do
 								        WebMock.reset!
 								      end
-												FIX: Don’t raise an error on onebox timeouts

Currently when generating oneboxes if the connection timeouts and we’re
using the `FinalDestination#get` method, then it raises an exception.

We already catch this exception when using the
`FinalDestination#resolve` method so this patch just applies the same
logic to `FinalDestination#get`.

											
										
										
											2022-07-21 09:58:17 -04:00
+								      it "correctly streams" do
 								        chunk = nil
 								        result = fd.get do |resp, c|
 								          chunk = c
 								          throw :done
 								        end
-												PERF: ability to crawl for titles without extra HEAD req

Also, introduces a much more aggressive timeout for title crawling
and introduces gzip to body that is crawled

											
										
										
											2018-01-28 23:36:52 -05:00
-												FIX: Don’t raise an error on onebox timeouts

Currently when generating oneboxes if the connection timeouts and we’re
using the `FinalDestination#get` method, then it raises an exception.

We already catch this exception when using the
`FinalDestination#resolve` method so this patch just applies the same
logic to `FinalDestination#get`.

											
										
										
											2022-07-21 09:58:17 -04:00
+								        expect(result).to eq("https://wikipedia.com/")
 								        expect(chunk).to eq("<html><head>")
 								      end
 								    end
-												PERF: ability to crawl for titles without extra HEAD req

Also, introduces a much more aggressive timeout for title crawling
and introduces gzip to body that is crawled

											
										
										
											2018-01-28 23:36:52 -05:00
-												FIX: Don’t raise an error on onebox timeouts

Currently when generating oneboxes if the connection timeouts and we’re
using the `FinalDestination#get` method, then it raises an exception.

We already catch this exception when using the
`FinalDestination#resolve` method so this patch just applies the same
logic to `FinalDestination#get`.

											
										
										
											2022-07-21 09:58:17 -04:00
+								    context "when there is a timeout" do
 								      subject(:get) { fd.get {} }
-												PERF: ability to crawl for titles without extra HEAD req

Also, introduces a much more aggressive timeout for title crawling
and introduces gzip to body that is crawled

											
										
										
											2018-01-28 23:36:52 -05:00
-												FIX: Don’t raise an error on onebox timeouts

Currently when generating oneboxes if the connection timeouts and we’re
using the `FinalDestination#get` method, then it raises an exception.

We already catch this exception when using the
`FinalDestination#resolve` method so this patch just applies the same
logic to `FinalDestination#get`.

											
										
										
											2022-07-21 09:58:17 -04:00
+								      before do
 								        fd.stubs(:safe_session).raises(Timeout::Error)
 								      end
-												PERF: ability to crawl for titles without extra HEAD req

Also, introduces a much more aggressive timeout for title crawling
and introduces gzip to body that is crawled

											
										
										
											2018-01-28 23:36:52 -05:00
-												FIX: Don’t raise an error on onebox timeouts

Currently when generating oneboxes if the connection timeouts and we’re
using the `FinalDestination#get` method, then it raises an exception.

We already catch this exception when using the
`FinalDestination#resolve` method so this patch just applies the same
logic to `FinalDestination#get`.

											
										
										
											2022-07-21 09:58:17 -04:00
+								      it "logs the exception" do
-												DEV: Fix flaky `FinalDestination` specs

											
										
										
											2022-07-27 06:09:47 -04:00
+								        Rails.logger.expects(:warn).with(regexp_matches(/FinalDestination could not resolve URL \(timeout\)/))
-												FIX: Don’t raise an error on onebox timeouts

Currently when generating oneboxes if the connection timeouts and we’re
using the `FinalDestination#get` method, then it raises an exception.

We already catch this exception when using the
`FinalDestination#resolve` method so this patch just applies the same
logic to `FinalDestination#get`.

											
										
										
											2022-07-21 09:58:17 -04:00
+								        get
-												PERF: ability to crawl for titles without extra HEAD req

Also, introduces a much more aggressive timeout for title crawling
and introduces gzip to body that is crawled

											
										
										
											2018-01-28 23:36:52 -05:00
+								      end
-												FIX: Don’t raise an error on onebox timeouts

Currently when generating oneboxes if the connection timeouts and we’re
using the `FinalDestination#get` method, then it raises an exception.

We already catch this exception when using the
`FinalDestination#resolve` method so this patch just applies the same
logic to `FinalDestination#get`.

											
										
										
											2022-07-21 09:58:17 -04:00
+								      it "returns nothing" do
 								        expect(get).to be_blank
 								      end
-												PERF: ability to crawl for titles without extra HEAD req

Also, introduces a much more aggressive timeout for title crawling
and introduces gzip to body that is crawled

											
										
										
											2018-01-28 23:36:52 -05:00
+								    end
-												FIX: Swallow SSL errors when generating oneboxes

											
										
										
											2022-08-09 06:13:56 -04:00
 								    context "when there is an SSL error" do
 								      subject(:get) { fd.get {} }
 								      before do
 								        fd.stubs(:safe_session).raises(OpenSSL::SSL::SSLError)
 								      end
 								      it "logs the exception" do
 								        Rails.logger.expects(:warn).with(regexp_matches(/an error with ssl occurred/i))
 								        get
 								      end
 								      it "returns nothing" do
 								        expect(get).to be_blank
 								      end
 								    end
-												PERF: ability to crawl for titles without extra HEAD req

Also, introduces a much more aggressive timeout for title crawling
and introduces gzip to body that is crawled

											
										
										
											2018-01-28 23:36:52 -05:00
+								  end
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								  describe ".validate_url_format" do
 								    it "supports http urls" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								      expect(fd('http://eviltrout.com').validate_uri_format).to eq(true)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    end
 								    it "supports https urls" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								      expect(fd('https://eviltrout.com').validate_uri_format).to eq(true)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    end
 								    it "doesn't support ftp urls" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								      expect(fd('ftp://eviltrout.com').validate_uri_format).to eq(false)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    end
-												SECURITY: Never crawl by IP

											
										
										
											2017-05-23 13:07:18 -04:00
+								    it "doesn't support IP urls" do
 								      expect(fd('http://104.25.152.10').validate_uri_format).to eq(false)
 								      expect(fd('https://[2001:abc:de:01:0:3f0:6a65:c2bf]').validate_uri_format).to eq(false)
 								    end
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    it "returns false for schemeless URL" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								      expect(fd('eviltrout.com').validate_uri_format).to eq(false)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    end
 								    it "returns false for nil URL" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								      expect(fd(nil).validate_uri_format).to eq(false)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    end
 								    it "returns false for invalid ports" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								      expect(fd('http://eviltrout.com:21').validate_uri_format).to eq(false)
 								      expect(fd('https://eviltrout.com:8000').validate_uri_format).to eq(false)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    end
 								    it "returns true for valid ports" do
-												FEATURE: A site setting to prevent crawling on private IP blocks

											
										
										
											2017-05-23 11:51:23 -04:00
+								      expect(fd('http://eviltrout.com:80').validate_uri_format).to eq(true)
 								      expect(fd('https://eviltrout.com:443').validate_uri_format).to eq(true)
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								    end
 								  end
-												FEATURE: cache https redirects per hostname

If a hostname does an https redirect we cache that so next
lookup does not incur it.

Also, only rate limit per ip once per final destination

Raise final destination protection to 1000 ip lookups an hour

											
										
										
											2017-10-17 01:22:38 -04:00
+								  describe "https cache" do
 								    it 'will cache https lookups' do
 								      FinalDestination.clear_https_cache!("wikipedia.com")
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								      fd_stub_request(:head, "http://wikipedia.com/image.png")
-												FEATURE: cache https redirects per hostname

If a hostname does an https redirect we cache that so next
lookup does not incur it.

Also, only rate limit per ip once per final destination

Raise final destination protection to 1000 ip lookups an hour

											
										
										
											2017-10-17 01:22:38 -04:00
+								        .to_return(status: 302, body: "", headers: { location: 'https://wikipedia.com/image.png' })
-												FIX: don't double request when downloading a file

											
										
										
											2018-02-24 06:35:57 -05:00
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								      fd_stub_request(:head, "https://wikipedia.com/image.png")
-												FEATURE: cache https redirects per hostname

If a hostname does an https redirect we cache that so next
lookup does not incur it.

Also, only rate limit per ip once per final destination

Raise final destination protection to 1000 ip lookups an hour

											
										
										
											2017-10-17 01:22:38 -04:00
 								      fd('http://wikipedia.com/image.png').resolve
-												SECURITY: Expand and improve SSRF Protections (#18815)

See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
											
										
										
											2022-11-01 12:33:17 -04:00
+								      fd_stub_request(:head, "https://wikipedia.com/image2.png")
-												FEATURE: cache https redirects per hostname

If a hostname does an https redirect we cache that so next
lookup does not incur it.

Also, only rate limit per ip once per final destination

Raise final destination protection to 1000 ip lookups an hour

											
										
										
											2017-10-17 01:22:38 -04:00
 								      fd('http://wikipedia.com/image2.png').resolve
 								    end
 								  end
-												DEV: Rename `UriHelper.escape_uri` to `.normalized_encode`

This is a much better description of its function. It performs idempotent normalization of a URL. If consumers truly need to `encode` a URL (including double-encoding of existing encoded entities), they can use the existing `.encode` method.

											
										
										
											2022-08-09 06:28:29 -04:00
+								  describe "#normalized_url" do
 								    it "correctly normalizes url" do
-												extract url escaping to a dedicated class method and improved tests

											
										
										
											2017-07-29 12:42:04 -04:00
+								      fragment_url = "https://eviltrout.com/2016/02/25/fixing-android-performance.html#discourse-comments"
-												DEV: Rename `UriHelper.escape_uri` to `.normalized_encode`

This is a much better description of its function. It performs idempotent normalization of a URL. If consumers truly need to `encode` a URL (including double-encoding of existing encoded entities), they can use the existing `.encode` method.

											
										
										
											2022-08-09 06:28:29 -04:00
+								      expect(fd(fragment_url).normalized_url.to_s).to eq(fragment_url)
-												FIX: Onebox fails on encoded URL.

https://meta.discourse.org/t/onebox-breaks-if-theres-chinese-text-in-url/67364

											
										
										
											2017-09-26 06:34:54 -04:00
-												DEV: Rename `UriHelper.escape_uri` to `.normalized_encode`

This is a much better description of its function. It performs idempotent normalization of a URL. If consumers truly need to `encode` a URL (including double-encoding of existing encoded entities), they can use the existing `.encode` method.

											
										
										
											2022-08-09 06:28:29 -04:00
+								      expect(fd("https://eviltrout.com?s=180&#038;d=mm&#038;r=g").normalized_url.to_s)
-												DEV: More robust processing of URLs (#11361)

* DEV: More robust processing of URLs

The previous `UrlHelper.encode_component(CGI.unescapeHTML(UrlHelper.unencode(uri))` method would naively process URLs, which could result in a badly formed response.

`Addressable::URI.normalized_encode(uri)` appears to deal with these edge-cases in a more robust way.

* DEV: onebox should use UrlHelper

* DEV: fix spec

* DEV: Escape output when rendering local links
											
										
										
											2020-12-03 17:16:01 -05:00
+								        .to eq("https://eviltrout.com?s=180&#038;d=mm&%23038;r=g")
-												FIX: Onebox fails on encoded URL.

https://meta.discourse.org/t/onebox-breaks-if-theres-chinese-text-in-url/67364

											
										
										
											2017-09-26 06:34:54 -04:00
-												DEV: Rename `UriHelper.escape_uri` to `.normalized_encode`

This is a much better description of its function. It performs idempotent normalization of a URL. If consumers truly need to `encode` a URL (including double-encoding of existing encoded entities), they can use the existing `.encode` method.

											
										
										
											2022-08-09 06:28:29 -04:00
+								      expect(fd("http://example.com/?a=\11\15").normalized_url.to_s).to eq("http://example.com/?a=%09%0D")
-												FIX: Onebox fails on encoded URL.

https://meta.discourse.org/t/onebox-breaks-if-theres-chinese-text-in-url/67364

											
										
										
											2017-09-26 06:34:54 -04:00
-												DEV: Rename `UriHelper.escape_uri` to `.normalized_encode`

This is a much better description of its function. It performs idempotent normalization of a URL. If consumers truly need to `encode` a URL (including double-encoding of existing encoded entities), they can use the existing `.encode` method.

											
										
										
											2022-08-09 06:28:29 -04:00
+								      expect(fd("https://ru.wikipedia.org/wiki/%D0%A1%D0%B2%D0%BE%D0%B1%D0%BE").normalized_url.to_s)
-												FIX: Onebox fails on encoded URL.

https://meta.discourse.org/t/onebox-breaks-if-theres-chinese-text-in-url/67364

											
										
										
											2017-09-26 06:34:54 -04:00
+								        .to eq('https://ru.wikipedia.org/wiki/%D0%A1%D0%B2%D0%BE%D0%B1%D0%BE')
-												DEV: Rename `UriHelper.escape_uri` to `.normalized_encode`

This is a much better description of its function. It performs idempotent normalization of a URL. If consumers truly need to `encode` a URL (including double-encoding of existing encoded entities), they can use the existing `.encode` method.

											
										
										
											2022-08-09 06:28:29 -04:00
+								      expect(fd('https://ru.wikipedia.org/wiki/Свобо').normalized_url.to_s)
-												FIX: Onebox fails on encoded URL.

https://meta.discourse.org/t/onebox-breaks-if-theres-chinese-text-in-url/67364

											
										
										
											2017-09-26 06:34:54 -04:00
+								        .to eq('https://ru.wikipedia.org/wiki/%D0%A1%D0%B2%D0%BE%D0%B1%D0%BE')
-												extract url escaping to a dedicated class method and improved tests

											
										
										
											2017-07-29 12:42:04 -04:00
+								    end
 								  end
-												Helper to find the final destination for a URL

											
										
										
											2017-05-22 12:23:04 -04:00
+								end