FEATURE: Pull hotlinked images immediately after posting

Previously, with the default `editing_grace_period`, hotlinked images were pulled 5 minutes after a post is created. This delay was added to reduce the chance of automated edits clashing with user edits.

This commit refactors things so that we can pull hotlinked images immediately. URLs are immediately updated in the post's `cooked` HTML. The post's raw markdown is updated later, after the `editing_grace_period`.

This involves a number of behind-the-scenes changes including:

- Schedule Jobs::PullHotlinkedImages immediately after Jobs::ProcessPost. Move scheduling to after the `update_column` call to avoid race conditions

- Move raw changes into a separate job, which is delayed until after the ninja-edit window

- Move disable_if_low_on_disk_space logic into the `pull_hotlinked_images` job

- Move raw-parsing/replacing logic into `InlineUpload` so it can be easily be shared between `UpdateHotlinkedRaw` and `PullUserProfileHotlinkedImages`

spec/jobs/process_post_spec.rb

@@ -88,4 +88,35 @@ describe Jobs::ProcessPost do
       expect(post.topic.reload.excerpt).to eq("Some OP content")
     end
   end
+
+  context "#enqueue_pull_hotlinked_images" do
+    fab!(:post) { Fabricate(:post, created_at: 20.days.ago) }
+    let(:job) { Jobs::ProcessPost.new }
+
+    it "runs even when download_remote_images_to_local is disabled" do
+      # We want to run it to pull hotlinked optimized images
+      SiteSetting.download_remote_images_to_local = false
+      expect_enqueued_with(job: :pull_hotlinked_images, args: { post_id: post.id }) do
+        job.execute({ post_id: post.id })
+      end
+    end
+
+    context "when download_remote_images_to_local? is enabled" do
+      before do
+        SiteSetting.download_remote_images_to_local = true
+      end
+
+      it "enqueues" do
+        expect_enqueued_with(job: :pull_hotlinked_images, args: { post_id: post.id }) do
+          job.execute({ post_id: post.id })
+        end
+      end
+
+      it "does not run when requested to skip" do
+        job.execute({ post_id: post.id, skip_pull_hotlinked_images: true })
+        expect(Jobs::PullHotlinkedImages.jobs.size).to eq(0)
+      end
+    end
+
+  end
 end

spec/jobs/pull_hotlinked_images_spec.rb

@@ -67,6 +67,21 @@ describe Jobs::PullHotlinkedImages do
       expect(post.reload.raw).to eq("<img src=\"#{Upload.last.short_url}\">")
     end
 
+    it 'enqueues raw replacement job with a delay' do
+      Jobs.run_later!
+
+      post = Fabricate(:post, raw: "<img src='#{image_url}'>")
+      stub_image_size
+
+      freeze_time
+      Jobs.expects(:cancel_scheduled_job).with(:update_hotlinked_raw, post_id: post.id).once
+      delay = SiteSetting.editing_grace_period + 1
+
+      expect_enqueued_with(job: :update_hotlinked_raw, args: { post_id: post.id }, at: Time.zone.now + delay.seconds) do
+        Jobs::PullHotlinkedImages.new.execute(post_id: post.id)
+      end
+    end
+
     it 'removes downloaded images when they are no longer needed' do
       post = Fabricate(:post, raw: "<img src='#{image_url}'>")
       stub_image_size
@@ -196,14 +211,14 @@ describe Jobs::PullHotlinkedImages do
       expect(post.uploads).to contain_exactly(upload)
     end
 
-    it "skips raw_html posts" do
+    it "skips editing raw for raw_html posts" do
       raw = "<img src=\"#{image_url}\">"
       post = Fabricate(:post, raw: raw, cook_method: Post.cook_methods[:raw_html])
       stub_image_size
       expect do
         post.rebake!
         post.reload
-      end.not_to change { Upload.count }
+      end.to change { Upload.count }.by(1)
 
       expect(post.raw).to eq(raw)
     end
@@ -556,6 +571,43 @@ describe Jobs::PullHotlinkedImages do
     end
   end
 
+  context "#disable_if_low_on_disk_space" do
+    fab!(:post) { Fabricate(:post, created_at: 20.days.ago) }
+    let(:job) { Jobs::PullHotlinkedImages.new }
+
+    before do
+      SiteSetting.download_remote_images_to_local = true
+      SiteSetting.download_remote_images_threshold = 20
+      job.stubs(:available_disk_space).returns(50)
+    end
+
+    it "does nothing when there's enough disk space" do
+      SiteSetting.expects(:download_remote_images_to_local=).never
+      job.execute({ post_id: post.id })
+    end
+
+    context "when there's not enough disk space" do
+
+      before { SiteSetting.download_remote_images_threshold = 75 }
+
+      it "disables download_remote_images_threshold and send a notification to the admin" do
+        StaffActionLogger.any_instance.expects(:log_site_setting_change).once
+        SystemMessage.expects(:create_from_system_user).with(Discourse.site_contact_user, :download_remote_images_disabled).once
+        job.execute({ post_id: post.id })
+
+        expect(SiteSetting.download_remote_images_to_local).to eq(false)
+      end
+
+      it "doesn't disable download_remote_images_to_local if site uses S3" do
+        setup_s3
+        job.execute({ post_id: post.id })
+
+        expect(SiteSetting.download_remote_images_to_local).to eq(true)
+      end
+
+    end
+  end
+
   def stub_s3(upload)
     stub_upload(upload)
     stub_request(:get, "https:" + upload.url).to_return(status: 200, body: file_from_fixtures("smallest.png"))

spec/lib/cooked_post_processor_spec.rb

@@ -21,7 +21,6 @@ describe CookedPostProcessor do
       cpp.expects(:post_process_oneboxes).in_sequence(post_process)
       cpp.expects(:post_process_images).in_sequence(post_process)
       cpp.expects(:optimize_urls).in_sequence(post_process)
-      cpp.expects(:pull_hotlinked_images).in_sequence(post_process)
       cpp.post_process
 
       expect(PostUpload.exists?(post: post, upload: upload)).to eq(true)
@@ -1543,102 +1542,6 @@ describe CookedPostProcessor do
     end
   end
 
-  context "#pull_hotlinked_images" do
-
-    let(:post) { build(:post, created_at: 20.days.ago) }
-    let(:cpp) { CookedPostProcessor.new(post) }
-
-    before { cpp.stubs(:available_disk_space).returns(90) }
-
-    it "runs even when download_remote_images_to_local is disabled" do
-      # We want to run it to pull hotlinked optimized images
-      SiteSetting.download_remote_images_to_local = false
-      expect { cpp.pull_hotlinked_images }.
-        to change { Jobs::PullHotlinkedImages.jobs.count }.by 1
-    end
-
-    context "when download_remote_images_to_local? is enabled" do
-      before do
-        SiteSetting.download_remote_images_to_local = true
-      end
-
-      it "disables download_remote_images if there is not enough disk space" do
-        cpp.expects(:available_disk_space).returns(5)
-        cpp.pull_hotlinked_images
-        expect(SiteSetting.download_remote_images_to_local).to eq(false)
-      end
-
-      it "does not run when requested to skip" do
-        CookedPostProcessor.new(post, skip_pull_hotlinked_images: true).pull_hotlinked_images
-        expect(Jobs::PullHotlinkedImages.jobs.size).to eq(0)
-      end
-
-      context "and there is enough disk space" do
-        before { cpp.expects(:disable_if_low_on_disk_space).at_least_once }
-
-        context "and the post has been updated by an actual user" do
-
-          before { post.id = 42 }
-
-          it "ensures only one job is scheduled right after the editing_grace_period" do
-            freeze_time
-
-            Jobs.expects(:cancel_scheduled_job).with(:pull_hotlinked_images, post_id: post.id).once
-
-            delay = SiteSetting.editing_grace_period + 1
-
-            expect_enqueued_with(job: :pull_hotlinked_images, args: { post_id: post.id }, at: Time.zone.now + delay.seconds) do
-              cpp.pull_hotlinked_images
-            end
-          end
-
-        end
-
-      end
-
-    end
-
-  end
-
-  context "#disable_if_low_on_disk_space" do
-
-    let(:post) { build(:post, created_at: 20.days.ago) }
-    let(:cpp) { CookedPostProcessor.new(post) }
-
-    before do
-      SiteSetting.download_remote_images_to_local = true
-      SiteSetting.download_remote_images_threshold = 20
-      cpp.stubs(:available_disk_space).returns(50)
-    end
-
-    it "does nothing when there's enough disk space" do
-      SiteSetting.expects(:download_remote_images_to_local=).never
-      cpp.disable_if_low_on_disk_space
-    end
-
-    context "when there's not enough disk space" do
-
-      before { SiteSetting.download_remote_images_threshold = 75 }
-
-      it "disables download_remote_images_threshold and send a notification to the admin" do
-        StaffActionLogger.any_instance.expects(:log_site_setting_change).once
-        SystemMessage.expects(:create_from_system_user).with(Discourse.site_contact_user, :download_remote_images_disabled).once
-        cpp.disable_if_low_on_disk_space
-
-        expect(SiteSetting.download_remote_images_to_local).to eq(false)
-      end
-
-      it "doesn't disable download_remote_images_to_local if site uses S3" do
-        setup_s3
-        cpp.disable_if_low_on_disk_space
-
-        expect(SiteSetting.download_remote_images_to_local).to eq(true)
-      end
-
-    end
-
-  end
-
   context "#is_a_hyperlink?" do
 
     let(:post) { build(:post) }