FIX: Apply onebox blocked domain checks on every redirect (#16150)

The `blocked onebox domains` setting lets site owners change what sites
are allowed to be oneboxed. When a link is entered into a post,
Discourse checks the domain of the link against that setting and blocks
the onebox if the domain is blocked. But if there's a chain of
redirects, then only the final destination website is checked against
the site setting.

This commit amends that behavior so that every website in the redirect
chain is checked against the site setting, and if anything is blocked
the original link doesn't onebox at all in the post. The
`Discourse-No-Onebox` header is also checked in every response and the
onebox is blocked if the header is set to "1".

Additionally, Discourse will now include the `Discourse-No-Onebox`
header with every response if the site requires login to access content.
This is done to signal to a Discourse instance that it shouldn't attempt
to onebox other Discourse instances if they're login-only. Non-Discourse
websites can also use include that header if they don't wish to have
Discourse onebox their content.

Internal ticket: t59305.

spec/lib/oneboxer_spec.rb

@@ -182,11 +182,16 @@ describe Oneboxer do
         stub_request(:get, "https://kitten.com").to_return(status: 200, body: html, headers: {})
         stub_request(:head, "https://kitten.com").to_return(status: 200, body: "", headers: {})
 
-        expect(Oneboxer.external_onebox("http://cat.com/meow")[:onebox]).to be_empty
-        expect(Oneboxer.external_onebox("https://kitten.com")[:onebox]).to be_empty
+        result = Oneboxer.external_onebox("http://cat.com/meow")
+        expect(result[:onebox]).to be_empty
+        expect(result[:preview]).to be_empty
+
+        result = Oneboxer.external_onebox("http://kitten.com")
+        expect(result[:onebox]).to be_empty
+        expect(result[:preview]).to be_empty
       end
 
-      it "returns onebox if 'midway redirect' is blocked but final redirect uri is not blocked" do
+      it "does not return onebox if anything in the redirect chain is blocked" do
         SiteSetting.blocked_onebox_domains = "middle.com"
 
         stub_request(:get, "https://cat.com/start").to_return(status: 301, body: "a", headers: { "location" => "https://middle.com/midway" })
@@ -197,7 +202,44 @@ describe Oneboxer do
         stub_request(:get, "https://cat.com/end").to_return(status: 200, body: html)
         stub_request(:head, "https://cat.com/end").to_return(status: 200, body: "", headers: {})
 
-        expect(Oneboxer.external_onebox("https://cat.com/start")[:onebox]).to be_present
+        result = Oneboxer.external_onebox("https://cat.com/start")
+        expect(result[:onebox]).to be_empty
+        expect(result[:preview]).to be_empty
+      end
+
+      it "does not return onebox if the Discourse-No-Onebox header == 1" do
+        stub_request(:get, "https://website.com/discourse-no-onebox")
+          .to_return(status: 200, body: "abc", headers: { "Discourse-No-Onebox" => "1" })
+        stub_request(:head, "https://website.com/discourse-no-onebox")
+          .to_return(status: 200, body: "", headers: { "Discourse-No-Onebox" => "1" })
+
+        result = Oneboxer.external_onebox("https://website.com/discourse-no-onebox")
+        expect(result[:onebox]).to be_empty
+        expect(result[:preview]).to be_empty
+      end
+
+      it "does not return onebox if the Discourse-No-Onebox header == 1 anywhere in the redirect chain" do
+        stub_request(:get, "https://website.com/redirect-no-onebox")
+          .to_return(status: 301, body: "", headers: { "Discourse-No-Onebox" => "1", "location" => "https://willneverreach.com" })
+        stub_request(:head, "https://website.com/redirect-no-onebox")
+          .to_return(status: 301, body: "", headers: { "Discourse-No-Onebox" => "1", "location" => "https://willneverreach.com" })
+
+        result = Oneboxer.external_onebox("https://website.com/redirect-no-onebox")
+        expect(result[:onebox]).to be_empty
+        expect(result[:preview]).to be_empty
+
+        stub_request(:get, "https://website.com/redirect")
+          .to_return(status: 301, body: "", headers: { "location" => "https://website.com/redirect/dont-onebox" })
+        stub_request(:head, "https://website.com/redirect")
+          .to_return(status: 301, body: "", headers: { "location" => "https://website.com/redirect/dont-onebox" })
+        stub_request(:get, "https://website.com/redirect/dont-onebox")
+          .to_return(status: 301, body: "", headers: { "Discourse-No-Onebox" => "1", "location" => "https://wontreachme.com" })
+        stub_request(:head, "https://website.com/redirect/dont-onebox")
+          .to_return(status: 301, body: "", headers: { "Discourse-No-Onebox" => "1", "location" => "https://wontreachme.com" })
+
+        result = Oneboxer.external_onebox("https://website.com/redirect")
+        expect(result[:onebox]).to be_empty
+        expect(result[:preview]).to be_empty
       end
     end