From e92cd5318bff051659a8705f595a300583db826a Mon Sep 17 00:00:00 2001
From: Dan Ungureanu <dan@ungureanu.me>
Date: Mon, 15 Apr 2019 09:26:00 +0300
Subject: [PATCH] FEATURE: Add setting to strip whitespaces from incoming
 emails. (#7375)

Some email clients add leading whitespaces which get are transformed in
code blocks when processed.
---
 config/locales/server.en.yml           |  1 +
 config/site_settings.yml               |  1 +
 lib/email/receiver.rb                  | 27 +++++++-
 spec/components/email/receiver_spec.rb | 87 ++++++++++++++++++++++++++
 4 files changed, 113 insertions(+), 3 deletions(-)

diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index d4ec1b78a52..a2b27f1007c 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -1742,6 +1742,7 @@ en:
     reply_by_email_address: "Template for reply by email incoming email address, for example: %%{reply_key}@reply.example.com or replies+%%{reply_key}@example.com"
     alternative_reply_by_email_addresses: "List of alternative templates for reply by email incoming email addresses. Example: %%{reply_key}@reply.example.com|replies+%%{reply_key}@example.com"
     incoming_email_prefer_html: "Use HTML instead of text for incoming email."
+    strip_incoming_email_lines: "Remove leading and trailing whitespaces from each line of incoming emails."
 
     disable_emails: "Prevent Discourse from sending any kind of emails. Select 'yes' to disable emails for all users. Select 'non-staff' to disable emails for non-staff users only."
 
diff --git a/config/site_settings.yml b/config/site_settings.yml
index 8ac802f1ae0..02925176c0f 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -896,6 +896,7 @@ email:
   pop3_polling_delete_from_server: true
   log_mail_processing_failures: false
   incoming_email_prefer_html: true
+  strip_incoming_email_lines: false
   email_in:
     default: false
     client: true
diff --git a/lib/email/receiver.rb b/lib/email/receiver.rb
index 1e98de90ac3..07231c702ae 100644
--- a/lib/email/receiver.rb
+++ b/lib/email/receiver.rb
@@ -353,11 +353,32 @@ module Email
         end
       end
 
+      text_format = Receiver::formats[:plaintext]
       if text.blank? || (SiteSetting.incoming_email_prefer_html && markdown.present?)
-        return [markdown, elided_markdown, Receiver::formats[:markdown]]
-      else
-        return [text, elided_text, Receiver::formats[:plaintext]]
+        text, elided_text, text_format = markdown, elided_markdown, Receiver::formats[:markdown]
       end
+
+      if SiteSetting.strip_incoming_email_lines
+        in_code = nil
+
+        text = text.lines.map! do |line|
+          stripped = line.strip << "\n"
+
+          if !in_code && stripped[0..2] == '```'
+            in_code = '```'
+          elsif in_code == '```' && stripped[0..2] == '```'
+            in_code = nil
+          elsif !in_code && stripped[0..4] == '[code'
+            in_code = '[code]'
+          elsif in_code == '[code]' && stripped[0..6] == '[/code]'
+            in_code = nil
+          end
+
+          in_code ? line : stripped
+        end.join
+      end
+
+      [text, elided_text, text_format]
     end
 
     def to_markdown(html, elided_html)
diff --git a/spec/components/email/receiver_spec.rb b/spec/components/email/receiver_spec.rb
index 7d4568c2f3c..ecc056b6446 100644
--- a/spec/components/email/receiver_spec.rb
+++ b/spec/components/email/receiver_spec.rb
@@ -1256,4 +1256,91 @@ describe Email::Receiver do
     expect(email.to_addresses).to eq("foo@bar.com")
     expect(email.cc_addresses).to eq("bob@example.com;carol@example.com")
   end
+
+  context "#select_body" do
+
+    let(:email) {
+      <<~EOF
+      MIME-Version: 1.0
+      Date: Tue, 01 Jan 2019 00:00:00 +0300
+      Subject: An email with whitespaces
+      From: Foo <foo@discourse.org>
+      To: bar@discourse.org
+      Content-Type: text/plain; charset="UTF-8"
+
+          This is a line that will be stripped
+          This is another line that will be stripped
+
+      This is a line that will not be touched.
+      This is another line that will not be touched.
+
+      [code]
+        1.upto(10).each do |i|
+          puts i
+        end
+
+      ```
+        # comment
+      [/code]
+
+          This is going to be stripped too.
+
+      ```
+        1.upto(10).each do |i|
+          puts i
+        end
+
+      [/code]
+        # comment
+      ```
+
+              This is going to be stripped too.
+
+      Bye!
+      EOF
+    }
+
+    let(:stripped_text) {
+      <<~EOF
+      This is a line that will be stripped
+      This is another line that will be stripped
+
+      This is a line that will not be touched.
+      This is another line that will not be touched.
+
+      [code]
+        1.upto(10).each do |i|
+          puts i
+        end
+
+      ```
+        # comment
+      [/code]
+
+      This is going to be stripped too.
+
+      ```
+        1.upto(10).each do |i|
+          puts i
+        end
+
+      [/code]
+        # comment
+      ```
+
+      This is going to be stripped too.
+
+      Bye!
+      EOF
+    }
+
+    it "strips lines if strip_incoming_email_lines is enabled" do
+      SiteSetting.strip_incoming_email_lines = true
+
+      receiver = Email::Receiver.new(email)
+      text, elided, format = receiver.select_body
+      expect(text).to eq(stripped_text)
+    end
+
+  end
 end