mirror of
https://github.com/discourse/discourse.git
synced 2026-08-05 02:43:08 -05:00
enable thumbnailing on S3
- added url to optimized image model - refactored s3_store & local_store
This commit is contained in:
Régis Hanol
@@ -77,7 +77,7 @@ class CookedPostProcessor
|
||||
end
|
||||
|
||||
def relative_to_absolute(src)
|
||||
if src =~ /\A\/[^\/]/
|
||||
if src =~ /^\/[^\/]/
|
||||
Discourse.base_url_no_prefix + src
|
||||
else
|
||||
src
|
||||
@@ -98,7 +98,7 @@ class CookedPostProcessor
|
||||
|
||||
def associate_to_post(upload)
|
||||
return if PostUpload.where(post_id: @post.id, upload_id: upload.id).count > 0
|
||||
PostUpload.create({ post_id: @post.id, upload_id: upload.id })
|
||||
PostUpload.create(post_id: @post.id, upload_id: upload.id)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
# do not care if it's already associated
|
||||
end
|
||||
@@ -155,7 +155,7 @@ class CookedPostProcessor
|
||||
a.add_child(img)
|
||||
|
||||
# replace the image by its thumbnail
|
||||
img['src'] = upload.thumbnail_url if upload && upload.has_thumbnail?
|
||||
img['src'] = relative_to_absolute(upload.thumbnail.url) if upload && upload.has_thumbnail?
|
||||
|
||||
# then, some overlay informations
|
||||
meta = Nokogiri::XML::Node.new("div", @doc)
|
||||
@@ -206,12 +206,13 @@ class CookedPostProcessor
|
||||
end
|
||||
|
||||
def get_size(url)
|
||||
# make sure s3 urls have a scheme (otherwise, FastImage will fail)
|
||||
url = "http:" + url if Upload.is_on_s3?(url)
|
||||
return unless is_valid_image_uri?(url)
|
||||
uri = url
|
||||
# make sure urls have a scheme (otherwise, FastImage will fail)
|
||||
uri = (SiteSetting.use_ssl? ? "https:" : "http:") + url if url.start_with?("//")
|
||||
return unless is_valid_image_uri?(uri)
|
||||
# we can *always* crawl our own images
|
||||
return unless SiteSetting.crawl_images? || Upload.has_been_uploaded?(url)
|
||||
@size_cache[url] ||= FastImage.size(url)
|
||||
return unless SiteSetting.crawl_images? || Discourse.store.has_been_uploaded?(url)
|
||||
@size_cache[url] ||= FastImage.size(uri)
|
||||
rescue Zlib::BufError # FastImage.size raises BufError for some gifs
|
||||
end
|
||||
|
||||
@@ -222,14 +223,9 @@ class CookedPostProcessor
|
||||
end
|
||||
|
||||
def attachments
|
||||
if SiteSetting.enable_s3_uploads?
|
||||
@doc.css("a.attachment[href^=\"#{S3Store.base_url}\"]")
|
||||
else
|
||||
# local uploads are identified using a relative uri
|
||||
@doc.css("a.attachment[href^=\"#{LocalStore.directory}\"]") +
|
||||
# when cdn is enabled, we have the whole url
|
||||
@doc.css("a.attachment[href^=\"#{LocalStore.base_url}\"]")
|
||||
end
|
||||
attachments = @doc.css("a.attachment[href^=\"#{Discourse.store.absolute_base_url}\"]")
|
||||
attachments += @doc.css("a.attachment[href^=\"#{Discourse.store.relative_base_url}\"]") if Discourse.store.internal?
|
||||
attachments
|
||||
end
|
||||
|
||||
def dirty?
|
||||
|
||||
+11
-1
@@ -81,7 +81,7 @@ module Discourse
|
||||
|
||||
def self.git_version
|
||||
return $git_version if $git_version
|
||||
f = Rails.root.to_s + "/config/version"
|
||||
f = Rails.root.to_s + "/lib/version"
|
||||
require f if File.exists?("#{f}.rb")
|
||||
|
||||
begin
|
||||
@@ -98,6 +98,16 @@ module Discourse
|
||||
user
|
||||
end
|
||||
|
||||
def self.store
|
||||
if SiteSetting.enable_s3_uploads?
|
||||
@s3_store_loaded ||= require 'file_store/s3_store'
|
||||
S3Store.new
|
||||
else
|
||||
@local_store_loaded ||= require 'file_store/local_store'
|
||||
LocalStore.new
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def self.maintenance_mode_key
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
class LocalStore
|
||||
|
||||
def store_upload(file, upload)
|
||||
unique_sha1 = Digest::SHA1.hexdigest("#{Time.now.to_s}#{file.original_filename}")[0,16]
|
||||
extension = File.extname(file.original_filename)
|
||||
clean_name = "#{unique_sha1}#{extension}"
|
||||
path = "#{relative_base_url}/#{upload.id}/#{clean_name}"
|
||||
# copy the file to the right location
|
||||
copy_file(file, "#{public_dir}#{path}")
|
||||
# url
|
||||
Discourse.base_uri + path
|
||||
end
|
||||
|
||||
def store_optimized_image(file, optimized_image)
|
||||
# 1234567890ABCDEF_100x200.jpg
|
||||
filename = [
|
||||
optimized_image.sha1[6..16],
|
||||
"_#{optimized_image.width}x#{optimized_image.height}",
|
||||
optimized_image.extension,
|
||||
].join
|
||||
# <rails>/public/uploads/site/_optimized/123/456/<filename>
|
||||
path = File.join(
|
||||
relative_base_url,
|
||||
"_optimized",
|
||||
optimized_image.sha1[0..2],
|
||||
optimized_image.sha1[3..5],
|
||||
filename
|
||||
)
|
||||
# copy the file to the right location
|
||||
copy_file(file, "#{public_dir}#{path}")
|
||||
# url
|
||||
Discourse.base_uri + path
|
||||
end
|
||||
|
||||
def remove_file(url)
|
||||
File.delete("#{public_dir}#{url}") if has_been_uploaded?(url)
|
||||
rescue Errno::ENOENT
|
||||
# don't care if the file isn't there
|
||||
end
|
||||
|
||||
def has_been_uploaded?(url)
|
||||
is_relative?(url) || is_local?(url)
|
||||
end
|
||||
|
||||
def absolute_base_url
|
||||
url = asset_host.present? ? asset_host : Discourse.base_url_no_prefix
|
||||
"#{url}#{relative_base_url}"
|
||||
end
|
||||
|
||||
def relative_base_url
|
||||
"/uploads/#{RailsMultisite::ConnectionManagement.current_db}"
|
||||
end
|
||||
|
||||
def external?
|
||||
!internal?
|
||||
end
|
||||
|
||||
def internal?
|
||||
true
|
||||
end
|
||||
|
||||
def path_for(upload)
|
||||
"#{public_dir}#{upload.url}"
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def copy_file(file, path)
|
||||
FileUtils.mkdir_p Pathname.new(path).dirname
|
||||
# move the file to the right location
|
||||
# not using cause mv, cause permissions are no good on move
|
||||
File.open(path, "wb") do |f|
|
||||
f.write(file.read)
|
||||
end
|
||||
end
|
||||
|
||||
def is_relative?(url)
|
||||
url.start_with?(relative_base_url)
|
||||
end
|
||||
|
||||
def is_local?(url)
|
||||
url.start_with?(absolute_base_url)
|
||||
end
|
||||
|
||||
def public_dir
|
||||
"#{Rails.root}/public"
|
||||
end
|
||||
|
||||
def asset_host
|
||||
Rails.configuration.action_controller.asset_host
|
||||
end
|
||||
|
||||
end
|
||||
@@ -0,0 +1,119 @@
|
||||
require 'digest/sha1'
|
||||
require 'open-uri'
|
||||
|
||||
class S3Store
|
||||
|
||||
def store_upload(file, upload)
|
||||
extension = File.extname(file.original_filename)
|
||||
remote_filename = "#{upload.id}#{upload.sha1}#{extension}"
|
||||
|
||||
# if this fails, it will throw an exception
|
||||
upload(file.tempfile, remote_filename, file.content_type)
|
||||
|
||||
# returns the url of the uploaded file
|
||||
"#{absolute_base_url}/#{remote_filename}"
|
||||
end
|
||||
|
||||
def store_optimized_image(file, optimized_image)
|
||||
extension = File.extname(file.path)
|
||||
remote_filename = [
|
||||
optimized_image.id,
|
||||
optimized_image.sha1,
|
||||
"_#{optimized_image.width}x#{optimized_image.height}",
|
||||
extension
|
||||
].join
|
||||
|
||||
# if this fails, it will throw an exception
|
||||
upload(file, remote_filename)
|
||||
|
||||
# returns the url of the uploaded file
|
||||
"#{absolute_base_url}/#{remote_filename}"
|
||||
end
|
||||
|
||||
def remove_file(url)
|
||||
check_missing_site_settings
|
||||
return unless has_been_uploaded?(url)
|
||||
name = File.basename(url)
|
||||
remove(name)
|
||||
end
|
||||
|
||||
def has_been_uploaded?(url)
|
||||
url.start_with?(absolute_base_url)
|
||||
end
|
||||
|
||||
def absolute_base_url
|
||||
"//#{s3_bucket}.s3.amazonaws.com"
|
||||
end
|
||||
|
||||
def external?
|
||||
true
|
||||
end
|
||||
|
||||
def internal?
|
||||
!external?
|
||||
end
|
||||
|
||||
def download(upload)
|
||||
temp_file = Tempfile.new(["discourse-s3", File.extname(upload.original_filename)])
|
||||
url = (SiteSetting.use_ssl? ? "https:" : "http:") + upload.url
|
||||
|
||||
File.open(temp_file.path, "wb") do |f|
|
||||
f.write open(url, "rb", read_timeout: 20).read
|
||||
end
|
||||
|
||||
temp_file
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def s3_bucket
|
||||
SiteSetting.s3_upload_bucket.downcase
|
||||
end
|
||||
|
||||
def check_missing_site_settings
|
||||
raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.s3_upload_bucket.blank?
|
||||
raise Discourse::SiteSettingMissing.new("s3_access_key_id") if SiteSetting.s3_access_key_id.blank?
|
||||
raise Discourse::SiteSettingMissing.new("s3_secret_access_key") if SiteSetting.s3_secret_access_key.blank?
|
||||
end
|
||||
|
||||
def get_or_create_directory(name)
|
||||
check_missing_site_settings
|
||||
|
||||
@fog_loaded ||= require 'fog'
|
||||
|
||||
fog = Fog::Storage.new generate_options
|
||||
|
||||
directory = fog.directories.get(name)
|
||||
directory = fog.directories.create(key: name) unless directory
|
||||
directory
|
||||
end
|
||||
|
||||
def generate_options
|
||||
options = {
|
||||
provider: 'AWS',
|
||||
aws_access_key_id: SiteSetting.s3_access_key_id,
|
||||
aws_secret_access_key: SiteSetting.s3_secret_access_key,
|
||||
}
|
||||
options[:region] = SiteSetting.s3_region unless SiteSetting.s3_region.empty?
|
||||
options
|
||||
end
|
||||
|
||||
def upload(file, name, content_type=nil)
|
||||
args = {
|
||||
key: name,
|
||||
public: true,
|
||||
body: file,
|
||||
}
|
||||
args[:content_type] = content_type if content_type
|
||||
directory.files.create(args)
|
||||
end
|
||||
|
||||
def remove(name)
|
||||
directory.files.destroy(key: name)
|
||||
end
|
||||
|
||||
def directory
|
||||
get_or_create_directory(s3_bucket)
|
||||
end
|
||||
|
||||
end
|
||||
@@ -1,96 +0,0 @@
|
||||
#
|
||||
# This class is used to download and optimize images.
|
||||
#
|
||||
|
||||
require 'image_sorcery'
|
||||
require 'digest/sha1'
|
||||
require 'open-uri'
|
||||
|
||||
class ImageOptimizer
|
||||
attr_accessor :url
|
||||
|
||||
# url is a url of an image ex:
|
||||
# 'http://site.com/image.png'
|
||||
# '/uploads/site/image.png'
|
||||
def initialize(url)
|
||||
@url = url
|
||||
# make sure directories exists
|
||||
FileUtils.mkdir_p downloads_dir
|
||||
FileUtils.mkdir_p optimized_dir
|
||||
end
|
||||
|
||||
# return the path of an optimized image,
|
||||
# if already cached return cached, else download and cache
|
||||
# at the original size.
|
||||
# if size is specified return a resized image
|
||||
# if height or width are nil maintain aspect ratio
|
||||
#
|
||||
# Optimised image is the "most efficient" storage for an image
|
||||
# at the basic level it runs through image_optim https://github.com/toy/image_optim
|
||||
# it also has a failsafe that converts jpg to png or the opposite. if jpg size is 1.5*
|
||||
# as efficient as png it flips formats.
|
||||
def optimized_image_url (width = nil, height = nil)
|
||||
begin
|
||||
unless has_been_uploaded?
|
||||
return @url unless SiteSetting.crawl_images?
|
||||
# download the file if it hasn't been cached yet
|
||||
download! unless File.exists?(cached_path)
|
||||
end
|
||||
|
||||
# resize the image using Image Magick
|
||||
result = ImageSorcery.new(cached_path).convert(optimized_path, resize: "#{width}x#{height}")
|
||||
return optimized_url if result
|
||||
@url
|
||||
rescue
|
||||
@url
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def public_dir
|
||||
@public_dir ||= "#{Rails.root}/public"
|
||||
end
|
||||
|
||||
def downloads_dir
|
||||
@downloads_dir ||= "#{public_dir}/downloads/#{RailsMultisite::ConnectionManagement.current_db}"
|
||||
end
|
||||
|
||||
def optimized_dir
|
||||
@optimized_dir ||= "#{public_dir}/images/#{RailsMultisite::ConnectionManagement.current_db}"
|
||||
end
|
||||
|
||||
def has_been_uploaded?
|
||||
@url.start_with?(Discourse.base_url_no_prefix)
|
||||
end
|
||||
|
||||
def cached_path
|
||||
@cached_path ||= if has_been_uploaded?
|
||||
"#{public_dir}#{@url[Discourse.base_url_no_prefix.length..-1]}"
|
||||
else
|
||||
"#{downloads_dir}/#{file_name(@url)}"
|
||||
end
|
||||
end
|
||||
|
||||
def optimized_path
|
||||
@optimized_path ||= "#{optimized_dir}/#{file_name(cached_path)}"
|
||||
end
|
||||
|
||||
def file_name (uri)
|
||||
image_info = FastImage.new(uri)
|
||||
name = Digest::SHA1.hexdigest(uri)[0,16]
|
||||
name << ".#{image_info.type}"
|
||||
name
|
||||
end
|
||||
|
||||
def download!
|
||||
File.open(cached_path, "wb") do |f|
|
||||
f.write open(@url, "rb", read_timeout: 20).read
|
||||
end
|
||||
end
|
||||
|
||||
def optimized_url
|
||||
@optimized_url ||= Discourse.base_url_no_prefix + "/images/#{RailsMultisite::ConnectionManagement.current_db}/#{file_name(cached_path)}"
|
||||
end
|
||||
|
||||
end
|
||||
@@ -1,47 +0,0 @@
|
||||
module LocalStore
|
||||
|
||||
def self.store_file(file, sha1, upload_id)
|
||||
unique_sha1 = Digest::SHA1.hexdigest("#{Time.now.to_s}#{file.original_filename}")[0,16]
|
||||
extension = File.extname(file.original_filename)
|
||||
clean_name = "#{unique_sha1}#{extension}"
|
||||
url_root = "#{directory}/#{upload_id}"
|
||||
path = "#{Rails.root}/public#{url_root}"
|
||||
|
||||
FileUtils.mkdir_p path
|
||||
|
||||
# not using cause mv, cause permissions are no good on move
|
||||
File.open("#{path}/#{clean_name}", "wb") do |f|
|
||||
f.write File.read(file.tempfile)
|
||||
end
|
||||
|
||||
# url
|
||||
Discourse::base_uri + "#{url_root}/#{clean_name}"
|
||||
end
|
||||
|
||||
def self.remove_file(url)
|
||||
File.delete("#{Rails.root}/public#{url}")
|
||||
rescue Errno::ENOENT
|
||||
end
|
||||
|
||||
def self.uploaded_regex
|
||||
/\/uploads\/#{RailsMultisite::ConnectionManagement.current_db}\/(?<upload_id>\d+)\/[0-9a-f]{16}\.(png|jpg|jpeg|gif|tif|tiff|bmp)/
|
||||
end
|
||||
|
||||
def self.base_url
|
||||
url = asset_host.present? ? asset_host : Discourse.base_url_no_prefix
|
||||
"#{url}#{directory}"
|
||||
end
|
||||
|
||||
def self.base_path
|
||||
"#{Rails.root}/public#{directory}"
|
||||
end
|
||||
|
||||
def self.directory
|
||||
"/uploads/#{RailsMultisite::ConnectionManagement.current_db}"
|
||||
end
|
||||
|
||||
def self.asset_host
|
||||
Rails.configuration.action_controller.asset_host
|
||||
end
|
||||
|
||||
end
|
||||
@@ -1,70 +0,0 @@
|
||||
module S3Store
|
||||
|
||||
def self.store_file(file, sha1, upload_id)
|
||||
S3Store.check_missing_site_settings
|
||||
|
||||
directory = S3Store.get_or_create_directory(SiteSetting.s3_upload_bucket)
|
||||
extension = File.extname(file.original_filename)
|
||||
remote_filename = "#{upload_id}#{sha1}#{extension}"
|
||||
|
||||
# if this fails, it will throw an exception
|
||||
file = S3Store.upload(file, remote_filename, directory)
|
||||
"#{S3Store.base_url}/#{remote_filename}"
|
||||
end
|
||||
|
||||
def self.base_url
|
||||
"//#{SiteSetting.s3_upload_bucket.downcase}.s3.amazonaws.com"
|
||||
end
|
||||
|
||||
def self.remove_file(url)
|
||||
S3Store.check_missing_site_settings
|
||||
|
||||
directory = S3Store.get_or_create_directory(SiteSetting.s3_upload_bucket)
|
||||
|
||||
file = S3Store.destroy(url, directory)
|
||||
end
|
||||
|
||||
def self.check_missing_site_settings
|
||||
raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.s3_upload_bucket.blank?
|
||||
raise Discourse::SiteSettingMissing.new("s3_access_key_id") if SiteSetting.s3_access_key_id.blank?
|
||||
raise Discourse::SiteSettingMissing.new("s3_secret_access_key") if SiteSetting.s3_secret_access_key.blank?
|
||||
end
|
||||
|
||||
def self.get_or_create_directory(name)
|
||||
@fog_loaded = require 'fog' unless @fog_loaded
|
||||
|
||||
options = S3Store.generate_options
|
||||
|
||||
fog = Fog::Storage.new(options)
|
||||
|
||||
directory = fog.directories.get(name)
|
||||
directory = fog.directories.create(key: name) unless directory
|
||||
|
||||
directory
|
||||
end
|
||||
|
||||
def self.generate_options
|
||||
options = {
|
||||
provider: 'AWS',
|
||||
aws_access_key_id: SiteSetting.s3_access_key_id,
|
||||
aws_secret_access_key: SiteSetting.s3_secret_access_key
|
||||
}
|
||||
options[:region] = SiteSetting.s3_region unless SiteSetting.s3_region.empty?
|
||||
|
||||
options
|
||||
end
|
||||
|
||||
def self.upload(file, name, directory)
|
||||
directory.files.create(
|
||||
key: name,
|
||||
public: true,
|
||||
body: file.tempfile,
|
||||
content_type: file.content_type
|
||||
)
|
||||
end
|
||||
|
||||
def self.destroy(name, directory)
|
||||
directory.files.destroy(key: name)
|
||||
end
|
||||
|
||||
end
|
||||
Reference in New Issue
Block a user