FEATURE: Allow email image embed with secure media (#10563)

This PR introduces a few important changes to secure media redaction in emails. First of all, two new site settings have been introduced:

* `secure_media_allow_embed_images_in_emails`: If enabled we will embed secure images in emails instead of redacting them.
* `secure_media_max_email_embed_image_size_kb`: The cap to the size of the secure image we will embed, defaulting to 1mb, so the email does not become too big. Max is 10mb. Works in tandem with `email_total_attachment_size_limit_kb`.

`Email::Sender` will now attach images to the email based on these settings. The sender will also call `inline_secure_images` in `Email::Styles` after secure media is redacted and attachments are added to replace redaction messages with attached images. I went with attachment and `cid` URLs because base64 image support is _still_ flaky in email clients.

All redaction of secure media is now handled in `Email::Styles` and calls out to `PrettyText.strip_secure_media` to do the actual stripping and replacing with placeholders. `app/mailers/group_smtp_mailer.rb` and `app/mailers/user_notifications.rb` no longer do any stripping because they are earlier in the pipeline than `Email::Styles`.

Finally the redaction notice has been restyled and includes a link to the media that the user can click, which will show it to them if they have the necessary permissions.

![image](https://user-images.githubusercontent.com/920448/92341012-b9a2c380-f0ff-11ea-860e-b376b4528357.png)

spec/components/email/sender_spec.rb

@@ -404,6 +404,87 @@ describe Email::Sender do
         .to contain_exactly(*[small_pdf, large_pdf, csv_file].map(&:original_filename))
     end
 
+    context "when secure media enabled" do
+      def enable_s3_uploads
+        SiteSetting.enable_s3_uploads = true
+        SiteSetting.s3_upload_bucket = "s3-upload-bucket"
+        SiteSetting.s3_access_key_id = "some key"
+        SiteSetting.s3_secret_access_key = "some secrets3_region key"
+        stub_request(:head, "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/")
+        stub_request(
+          :put,
+          "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{image.sha1}.#{image.extension}?acl"
+        )
+        store = FileStore::S3Store.new
+        s3_helper = store.instance_variable_get(:@s3_helper)
+        client = Aws::S3::Client.new(stub_responses: true)
+        s3_helper.stubs(:s3_client).returns(client)
+        Discourse.stubs(:store).returns(store)
+      end
+
+      before do
+        enable_s3_uploads
+        SiteSetting.secure_media = true
+        SiteSetting.login_required = true
+        SiteSetting.email_total_attachment_size_limit_kb = 14_000
+        SiteSetting.secure_media_max_email_embed_image_size_kb = 5_000
+
+        Jobs.run_immediately!
+        Jobs::PullHotlinkedImages.any_instance.expects(:execute)
+        FileStore::S3Store.any_instance.expects(:has_been_uploaded?).returns(true).at_least_once
+        CookedPostProcessor.any_instance.stubs(:get_size).returns([244, 66])
+
+        @secure_image = UploadCreator.new(file_from_fixtures("logo.png", "images"), "logo.png")
+          .create_for(Discourse.system_user.id)
+        @secure_image.update_secure_status(secure_override_value: true)
+        @secure_image.update(access_control_post_id: reply.id)
+        reply.update(raw: reply.raw + "\n" + "#{UploadMarkdown.new(@secure_image).image_markdown}")
+        reply.rebake!
+      end
+
+      it "does not attach images when embedding them is not allowed" do
+        Email::Sender.new(message, :valid_type).send
+        expect(message.attachments.length).to eq(3)
+      end
+
+      context "when embedding secure images in email is allowed" do
+        before do
+          SiteSetting.secure_media_allow_embed_images_in_emails = true
+        end
+
+        it "does not attach images that are not marked as secure" do
+          Email::Sender.new(message, :valid_type).send
+          expect(message.attachments.length).to eq(4)
+        end
+
+        it "does not embed images that are too big" do
+          SiteSetting.secure_media_max_email_embed_image_size_kb = 1
+          Email::Sender.new(message, :valid_type).send
+          expect(message.attachments.length).to eq(3)
+        end
+
+        it "uses the email styles to inline secure images and attaches the secure image upload to the email" do
+          Email::Sender.new(message, :valid_type).send
+          expect(message.attachments.length).to eq(4)
+          expect(message.attachments.map(&:filename))
+            .to contain_exactly(*[small_pdf, large_pdf, csv_file, @secure_image].map(&:original_filename))
+          expect(message.html_part.body).to include("cid:")
+          expect(message.html_part.body).to include("embedded-secure-image")
+          expect(message.attachments.length).to eq(4)
+        end
+      end
+    end
+
+    it "adds only non-image uploads as attachments to the email and leaves the image intact with original source" do
+      SiteSetting.email_total_attachment_size_limit_kb = 10_000
+      Email::Sender.new(message, :valid_type).send
+
+      expect(message.attachments.length).to eq(3)
+      expect(message.attachments.map(&:filename))
+        .to contain_exactly(*[small_pdf, large_pdf, csv_file].map(&:original_filename))
+      expect(message.html_part.body).to include("<img src=\"#{Discourse.base_url}#{image.url}\"")
+    end
+
     it "respects the size limit and attaches only files that fit into the max email size" do
       SiteSetting.email_total_attachment_size_limit_kb = 40
       Email::Sender.new(message, :valid_type).send

spec/components/email/styles_spec.rb

@@ -4,6 +4,7 @@ require 'rails_helper'
 require 'email'
 
 describe Email::Styles do
+  let(:attachments) { {} }
 
   def basic_fragment(html)
     styler = Email::Styles.new(html)
@@ -186,23 +187,57 @@ describe Email::Styles do
     end
   end
 
-  context "replace_relative_urls" do
+  context "replace_secure_media_urls" do
+    let(:attachments) { { 'testimage.png' => stub(url: 'email/test.png') } }
     it "replaces secure media within a link with a placeholder" do
       frag = html_fragment("<a href=\"#{Discourse.base_url}\/secure-media-uploads/original/1X/testimage.png\"><img src=\"/secure-media-uploads/original/1X/testimage.png\"></a>")
-      expect(frag.at('p.secure-media-notice')).to be_present
       expect(frag.at('img')).not_to be_present
-      expect(frag.at('a')).not_to be_present
+      expect(frag.to_s).to include("Redacted")
     end
 
     it "replaces secure images with a placeholder" do
       frag = html_fragment("<img src=\"/secure-media-uploads/original/1X/testimage.png\">")
-      expect(frag.at('p.secure-media-notice')).to be_present
       expect(frag.at('img')).not_to be_present
+      expect(frag.to_s).to include("Redacted")
     end
 
     it "does not replace topic links with secure-media-uploads in the name" do
       frag = html_fragment("<a href=\"#{Discourse.base_url}\/t/secure-media-uploads/235723\">Visit Topic</a>")
-      expect(frag.at('p.secure-media-notice')).not_to be_present
+      expect(frag.to_s).not_to include("Redacted")
+    end
+  end
+
+  context "inline_secure_images" do
+    let(:attachments) { { 'testimage.png' => stub(url: 'cid:email/test.png') } }
+    fab!(:upload) { Fabricate(:upload, original_filename: 'testimage.png', secure: true, sha1: '123456') }
+
+    def strip_and_inline
+      html = "<a href=\"#{Discourse.base_url}\/secure-media-uploads/original/1X/123456.png\"><img src=\"/secure-media-uploads/original/1X/123456.png\"></a>"
+
+      # strip out the secure media
+      styler = Email::Styles.new(html)
+      styler.format_basic
+      styler.format_html
+      html = styler.to_html
+
+      # pass in the attachments to match uploads based on sha + original filename
+      styler = Email::Styles.new(html)
+      styler.inline_secure_images(attachments)
+      @frag = Nokogiri::HTML5.fragment(styler.to_s)
+    end
+
+    it "inlines attachments where stripped-secure-media data attr is present" do
+      strip_and_inline
+      expect(@frag.to_s).to include("cid:email/test.png")
+      expect(@frag.css('[data-stripped-secure-media]')).not_to be_present
+    end
+
+    it "does not inline anything if the upload cannot be found" do
+      upload.update(sha1: 'blah12')
+      strip_and_inline
+
+      expect(@frag.to_s).not_to include("cid:email/test.png")
+      expect(@frag.css('[data-stripped-secure-media]')).to be_present
     end
   end
 end

spec/components/pretty_text_spec.rb

@@ -928,6 +928,40 @@ describe PrettyText do
         expect(md.to_s).to match(I18n.t("emails.secure_media_placeholder"))
         expect(md.to_s).not_to match(SiteSetting.Upload.s3_cdn_url)
       end
+
+      it "replaces secure media within a link with a placeholder, keeping the url in an attribute" do
+        url = "#{Discourse.base_url}\/secure-media-uploads/original/1X/testimage.png"
+        html = <<~HTML
+        <a href=\"#{url}\"><img src=\"/secure-media-uploads/original/1X/testimage.png\"></a>
+        HTML
+        md = PrettyText.format_for_email(html, post)
+        expect(md).not_to include('<img')
+        expect(md).to include("Redacted")
+        expect(md).to include("data-stripped-secure-media=\"#{url}\"")
+      end
+
+      it "does not create nested redactions from double processing because of the view media link" do
+        url = "#{Discourse.base_url}\/secure-media-uploads/original/1X/testimage.png"
+        html = <<~HTML
+        <a href=\"#{url}\"><img src=\"/secure-media-uploads/original/1X/testimage.png\"></a>
+        HTML
+        md = PrettyText.format_for_email(html, post)
+        md = PrettyText.format_for_email(md, post)
+
+        expect(md.scan(/stripped-secure-view-media/).length).to eq(1)
+        expect(md.scan(/Redacted/).length).to eq(1)
+      end
+
+      it "replaces secure images with a placeholder, keeping the url in an attribute" do
+        url = "/secure-media-uploads/original/1X/testimage.png"
+        html = <<~HTML
+        <img src=\"#{url}\">
+        HTML
+        md = PrettyText.format_for_email(html, post)
+        expect(md).not_to include('<img')
+        expect(md).to include("Redacted")
+        expect(md).to include("data-stripped-secure-media=\"#{url}\"")
+      end
     end
   end
 

spec/mailers/user_notifications_spec.rb

@@ -699,50 +699,6 @@ describe UserNotifications do
     expect(mail.body.to_s).to match(I18n.t("user_notifications.reached_limit", count: 2))
   end
 
-  describe "secure media" do
-    let(:video_upload) { Fabricate(:upload, extension: "mov") }
-    let(:user) { Fabricate(:user) }
-    let(:post) { Fabricate(:post) }
-
-    before do
-      SiteSetting.s3_upload_bucket = "some-bucket-on-s3"
-      SiteSetting.s3_access_key_id = "s3-access-key-id"
-      SiteSetting.s3_secret_access_key = "s3-secret-access-key"
-      SiteSetting.s3_cdn_url = "https://s3.cdn.com"
-      SiteSetting.enable_s3_uploads = true
-      SiteSetting.secure_media = true
-      SiteSetting.login_required = true
-
-      video_upload.update!(url: "#{SiteSetting.s3_cdn_url}/#{Discourse.store.get_path_for_upload(video_upload)}")
-      user.email_logs.create!(
-        email_type: 'blah',
-        to_address: user.email,
-        user_id: user.id
-      )
-    end
-
-    it "replaces secure audio/video with placeholder" do
-      reply = Fabricate(:post, topic_id: post.topic_id, raw: "Video: #{video_upload.url}")
-
-      notification = Fabricate(
-        :notification,
-        topic_id: post.topic_id,
-        post_number: reply.post_number,
-        user: post.user,
-        data: { original_username: 'bob' }.to_json
-      )
-
-      mail = UserNotifications.user_replied(
-        user,
-        post: reply,
-        notification_type: notification.notification_type,
-        notification_data_hash: notification.data_hash
-      )
-
-      expect(mail.body.to_s).to match(I18n.t("emails.secure_media_placeholder"))
-    end
-  end
-
   def expects_build_with(condition)
     UserNotifications.any_instance.expects(:build_email).with(user.email, condition)
     mailer = UserNotifications.public_send(