mirror of
https://github.com/discourse/discourse.git
synced 2024-11-25 10:20:58 -06:00
b93b6c4299
This is a fix to address blurry onebox favicon images if the site you are linking to happens to have a favicon.ico file that contains multiple images. This fix detects of we are trying to create an upload for a favicon.ico file. We then convert it to a png and not a jpeg like we were doing. We want a png because it will preserve transparency, otherwise if we convert it to a jpeg we lose that and it looks bad on dark themed sites. This fix also addresses the fact that .ico files can include multiple images. The blurry images we were producing was caused by the ImageMagick `-flatten` option when the .ico file had multiple images which then squishes them all together. So for .ico files we are no longer flattening them and instead we are grabbing the last image in the .ico bundle and converting that single image to a png.
565 lines
19 KiB
Ruby
565 lines
19 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require "fastimage"
|
|
|
|
class UploadCreator
|
|
|
|
TYPES_TO_CROP ||= %w{avatar card_background custom_emoji profile_background}.each(&:freeze)
|
|
|
|
ALLOWED_SVG_ELEMENTS ||= %w{
|
|
circle clipPath defs ellipse feGaussianBlur filter g line linearGradient
|
|
marker path polygon polyline radialGradient rect stop style svg text
|
|
textPath tref tspan use
|
|
}.each(&:freeze)
|
|
|
|
include ActiveSupport::Deprecation::DeprecatedConstantAccessor
|
|
deprecate_constant 'WHITELISTED_SVG_ELEMENTS', 'UploadCreator::ALLOWED_SVG_ELEMENTS'
|
|
|
|
# Available options
|
|
# - type (string)
|
|
# - origin (string)
|
|
# - for_group_message (boolean)
|
|
# - for_theme (boolean)
|
|
# - for_private_message (boolean)
|
|
# - pasted (boolean)
|
|
# - for_export (boolean)
|
|
# - for_gravatar (boolean)
|
|
# - skip_validations (boolean)
|
|
def initialize(file, filename, opts = {})
|
|
@file = file
|
|
@filename = (filename || "").gsub(/[^[:print:]]/, "")
|
|
@upload = Upload.new(original_filename: @filename, filesize: 0)
|
|
@opts = opts
|
|
@filesize = @opts[:filesize] if @opts[:external_upload_too_big]
|
|
@opts[:validate] = opts[:skip_validations].present? ? !ActiveRecord::Type::Boolean.new.cast(opts[:skip_validations]) : true
|
|
|
|
# TODO (martin) Validate @opts[:type] to make sure only blessed types are passed
|
|
# in, since the clientside can pass any type it wants.
|
|
end
|
|
|
|
def create_for(user_id)
|
|
if filesize <= 0
|
|
@upload.errors.add(:base, I18n.t("upload.empty"))
|
|
return @upload
|
|
end
|
|
|
|
@image_info = FastImage.new(@file) rescue nil
|
|
is_image = FileHelper.is_supported_image?(@filename)
|
|
is_image ||= @image_info && FileHelper.is_supported_image?("test.#{@image_info.type}")
|
|
is_image = false if @opts[:for_theme]
|
|
|
|
# if this is present then it means we are creating an upload record from
|
|
# an external_upload_stub and the file is > ExternalUploadManager::DOWNLOAD_LIMIT,
|
|
# so we have not downloaded it to a tempfile. no modifications can be made to the
|
|
# file in this case because it does not exist; we simply move it to its new location
|
|
# in S3
|
|
#
|
|
# TODO (martin) I've added a bunch of external_upload_too_big checks littered
|
|
# throughout the UploadCreator code. It would be better to have two seperate
|
|
# classes with shared methods, rather than doing all these checks all over the
|
|
# place. Needs a refactor.
|
|
external_upload_too_big = @opts[:external_upload_too_big]
|
|
sha1_before_changes = Upload.generate_digest(@file) if @file
|
|
|
|
DistributedMutex.synchronize("upload_#{user_id}_#{@filename}") do
|
|
# We need to convert HEIFs early because FastImage does not consider them as images
|
|
if convert_heif_to_jpeg? && !external_upload_too_big
|
|
convert_heif!
|
|
is_image = FileHelper.is_supported_image?("test.#{@image_info.type}")
|
|
end
|
|
|
|
if is_image && !external_upload_too_big
|
|
extract_image_info!
|
|
return @upload if @upload.errors.present?
|
|
|
|
if @image_info.type == :svg
|
|
clean_svg!
|
|
elsif @image_info.type == :ico
|
|
convert_favicon_to_png!
|
|
elsif !Rails.env.test? || @opts[:force_optimize]
|
|
convert_to_jpeg! if convert_png_to_jpeg? || should_alter_quality?
|
|
fix_orientation! if should_fix_orientation?
|
|
crop! if should_crop?
|
|
optimize! if should_optimize?
|
|
downsize! if should_downsize?
|
|
return @upload if is_still_too_big?
|
|
end
|
|
|
|
# conversion may have switched the type
|
|
image_type = @image_info.type.to_s
|
|
end
|
|
|
|
# compute the sha of the file and generate a unique hash
|
|
# which is only used for secure uploads
|
|
if !external_upload_too_big
|
|
sha1 = Upload.generate_digest(@file)
|
|
end
|
|
if SiteSetting.secure_media || external_upload_too_big
|
|
unique_hash = generate_fake_sha1_hash
|
|
end
|
|
|
|
# we do not check for duplicate uploads if secure media is
|
|
# enabled because we use a unique access hash to differentiate
|
|
# between uploads instead of the sha1, and to get around various
|
|
# access/permission issues for uploads
|
|
if !SiteSetting.secure_media && !external_upload_too_big
|
|
# do we already have that upload?
|
|
@upload = Upload.find_by(sha1: sha1)
|
|
|
|
# make sure the previous upload has not failed
|
|
if @upload && @upload.url.blank?
|
|
@upload.destroy
|
|
@upload = nil
|
|
end
|
|
|
|
# return the previous upload if any
|
|
if @upload
|
|
add_metadata!
|
|
UserUpload.find_or_create_by!(user_id: user_id, upload_id: @upload.id) if user_id
|
|
return @upload
|
|
end
|
|
end
|
|
|
|
fixed_original_filename = nil
|
|
|
|
if is_image && !external_upload_too_big
|
|
current_extension = File.extname(@filename).downcase.sub("jpeg", "jpg")
|
|
expected_extension = ".#{image_type}".downcase.sub("jpeg", "jpg")
|
|
|
|
# we have to correct original filename here, no choice
|
|
# otherwise validation will fail and we can not save
|
|
# TODO decide if we only run the validation on the extension
|
|
if current_extension != expected_extension
|
|
basename = File.basename(@filename, current_extension).presence || "image"
|
|
fixed_original_filename = "#{basename}#{expected_extension}"
|
|
end
|
|
end
|
|
|
|
# create the upload otherwise
|
|
@upload = Upload.new
|
|
@upload.user_id = user_id
|
|
@upload.original_filename = fixed_original_filename || @filename
|
|
@upload.filesize = filesize
|
|
@upload.sha1 = (SiteSetting.secure_media? || external_upload_too_big) ? unique_hash : sha1
|
|
@upload.original_sha1 = SiteSetting.secure_media? ? sha1 : nil
|
|
@upload.url = ""
|
|
@upload.origin = @opts[:origin][0...1000] if @opts[:origin]
|
|
@upload.extension = image_type || File.extname(@filename)[1..10]
|
|
|
|
if is_image && !external_upload_too_big
|
|
if @image_info.type.to_s == 'svg'
|
|
w, h = [0, 0]
|
|
|
|
# identify can behave differently depending on how it's compiled and
|
|
# what programs (e.g. inkscape) are installed on your system.
|
|
# 'MSVG:' forces ImageMagick to use internal routines and behave
|
|
# consistently whether it's running from our docker container or not
|
|
begin
|
|
w, h = Discourse::Utils
|
|
.execute_command("identify", "-ping", "-format", "%w %h", "MSVG:#{@file.path}", timeout: Upload::MAX_IDENTIFY_SECONDS)
|
|
.split(' ').map(&:to_i)
|
|
rescue
|
|
# use default 0, 0
|
|
end
|
|
else
|
|
w, h = @image_info.size
|
|
end
|
|
|
|
@upload.thumbnail_width, @upload.thumbnail_height = ImageSizer.resize(w, h)
|
|
@upload.width, @upload.height = w, h
|
|
@upload.animated = animated?
|
|
end
|
|
|
|
add_metadata!
|
|
|
|
if SiteSetting.secure_media
|
|
secure, reason = UploadSecurity.new(@upload, @opts.merge(creating: true)).should_be_secure_with_reason
|
|
attrs = @upload.secure_params(secure, reason, "upload creator")
|
|
@upload.assign_attributes(attrs)
|
|
end
|
|
|
|
# Callbacks using this event to validate the upload or the file must add errors to the
|
|
# upload errors object.
|
|
#
|
|
# Can't do any validation of the file if external_upload_too_big because we don't have
|
|
# the actual file.
|
|
if !external_upload_too_big
|
|
DiscourseEvent.trigger(:before_upload_creation, @file, is_image, @upload, @opts[:validate])
|
|
end
|
|
return @upload unless @upload.errors.empty? && @upload.save(validate: @opts[:validate])
|
|
|
|
should_move = false
|
|
upload_changed = \
|
|
if external_upload_too_big
|
|
false
|
|
else
|
|
Upload.generate_digest(@file) != sha1_before_changes
|
|
end
|
|
|
|
if @opts[:existing_external_upload_key] && Discourse.store.external?
|
|
should_move = external_upload_too_big || !upload_changed
|
|
end
|
|
|
|
if should_move
|
|
# move the file in the store instead of reuploading
|
|
url = Discourse.store.move_existing_stored_upload(
|
|
existing_external_upload_key: @opts[:existing_external_upload_key],
|
|
upload: @upload
|
|
)
|
|
else
|
|
# store the file and update its url
|
|
File.open(@file.path) do |f|
|
|
url = Discourse.store.store_upload(f, @upload)
|
|
end
|
|
end
|
|
|
|
if url.present?
|
|
@upload.url = url
|
|
@upload.save!(validate: @opts[:validate])
|
|
else
|
|
@upload.errors.add(:url, I18n.t("upload.store_failure", upload_id: @upload.id, user_id: user_id))
|
|
end
|
|
|
|
if @upload.errors.empty? && is_image && @opts[:type] == "avatar" && @upload.extension != "svg"
|
|
Jobs.enqueue(:create_avatar_thumbnails, upload_id: @upload.id)
|
|
end
|
|
|
|
if @upload.errors.empty?
|
|
UserUpload.find_or_create_by!(user_id: user_id, upload_id: @upload.id) if user_id
|
|
end
|
|
|
|
@upload
|
|
end
|
|
ensure
|
|
if @file
|
|
@file.respond_to?(:close!) ? @file.close! : @file.close
|
|
end
|
|
end
|
|
|
|
def extract_image_info!
|
|
@image_info = FastImage.new(@file) rescue nil
|
|
@file.rewind
|
|
|
|
if @image_info.nil?
|
|
@upload.errors.add(:base, I18n.t("upload.images.not_supported_or_corrupted"))
|
|
elsif filesize <= 0
|
|
@upload.errors.add(:base, I18n.t("upload.empty"))
|
|
elsif pixels == 0 && @image_info.type.to_s != 'svg'
|
|
@upload.errors.add(:base, I18n.t("upload.images.size_not_found"))
|
|
elsif max_image_pixels > 0 && pixels >= max_image_pixels * 2
|
|
@upload.errors.add(:base, I18n.t("upload.images.larger_than_x_megapixels", max_image_megapixels: SiteSetting.max_image_megapixels * 2))
|
|
end
|
|
end
|
|
|
|
MIN_PIXELS_TO_CONVERT_TO_JPEG ||= 1280 * 720
|
|
|
|
def convert_png_to_jpeg?
|
|
return false unless @image_info.type == :png
|
|
return true if @opts[:pasted]
|
|
return false if SiteSetting.png_to_jpg_quality == 100
|
|
pixels > MIN_PIXELS_TO_CONVERT_TO_JPEG
|
|
end
|
|
|
|
MIN_CONVERT_TO_JPEG_BYTES_SAVED = 75_000
|
|
MIN_CONVERT_TO_JPEG_SAVING_RATIO = 0.70
|
|
|
|
def convert_favicon_to_png!
|
|
png_tempfile = Tempfile.new(["image", ".png"])
|
|
|
|
from = @file.path
|
|
to = png_tempfile.path
|
|
|
|
OptimizedImage.ensure_safe_paths!(from, to)
|
|
|
|
from = OptimizedImage.prepend_decoder!(from, nil, filename: "image.#{@image_info.type}")
|
|
to = OptimizedImage.prepend_decoder!(to)
|
|
|
|
from = "#{from}[-1]" # We only want the last(largest) image of the .ico file
|
|
|
|
opts = { flatten: false } # Preserve transparency
|
|
|
|
begin
|
|
execute_convert(from, to, opts)
|
|
rescue
|
|
# retry with debugging enabled
|
|
execute_convert(from, to, opts.merge(debug: true))
|
|
end
|
|
|
|
@file.respond_to?(:close!) ? @file.close! : @file.close
|
|
@file = png_tempfile
|
|
extract_image_info!
|
|
end
|
|
|
|
def convert_to_jpeg!
|
|
return if @opts[:for_site_setting]
|
|
return if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED
|
|
|
|
jpeg_tempfile = Tempfile.new(["image", ".jpg"])
|
|
|
|
from = @file.path
|
|
to = jpeg_tempfile.path
|
|
|
|
OptimizedImage.ensure_safe_paths!(from, to)
|
|
|
|
from = OptimizedImage.prepend_decoder!(from, nil, filename: "image.#{@image_info.type}")
|
|
to = OptimizedImage.prepend_decoder!(to)
|
|
|
|
opts = {}
|
|
desired_quality = [SiteSetting.png_to_jpg_quality, SiteSetting.recompress_original_jpg_quality].compact.min
|
|
target_quality = @upload.target_image_quality(from, desired_quality)
|
|
opts = { quality: target_quality } if target_quality
|
|
|
|
begin
|
|
execute_convert(from, to, opts)
|
|
rescue
|
|
# retry with debugging enabled
|
|
execute_convert(from, to, opts.merge(debug: true))
|
|
end
|
|
|
|
new_size = File.size(jpeg_tempfile.path)
|
|
|
|
keep_jpeg = new_size < filesize * MIN_CONVERT_TO_JPEG_SAVING_RATIO
|
|
keep_jpeg &&= (filesize - new_size) > MIN_CONVERT_TO_JPEG_BYTES_SAVED
|
|
|
|
if keep_jpeg
|
|
@file.respond_to?(:close!) ? @file.close! : @file.close
|
|
@file = jpeg_tempfile
|
|
extract_image_info!
|
|
else
|
|
jpeg_tempfile.close!
|
|
end
|
|
end
|
|
|
|
def convert_heif_to_jpeg?
|
|
File.extname(@filename).downcase.match?(/\.hei(f|c)$/)
|
|
end
|
|
|
|
def convert_heif!
|
|
jpeg_tempfile = Tempfile.new(["image", ".jpg"])
|
|
from = @file.path
|
|
to = jpeg_tempfile.path
|
|
OptimizedImage.ensure_safe_paths!(from, to)
|
|
|
|
begin
|
|
execute_convert(from, to)
|
|
rescue
|
|
# retry with debugging enabled
|
|
execute_convert(from, to, { debug: true })
|
|
end
|
|
|
|
@file.respond_to?(:close!) ? @file.close! : @file.close
|
|
@file = jpeg_tempfile
|
|
extract_image_info!
|
|
end
|
|
|
|
MAX_CONVERT_FORMAT_SECONDS = 20
|
|
def execute_convert(from, to, opts = {})
|
|
command = [
|
|
"convert",
|
|
from,
|
|
"-auto-orient",
|
|
"-background", "white",
|
|
"-interlace", "none",
|
|
]
|
|
command << "-flatten" unless opts[:flatten] == false
|
|
command << "-debug" << "all" if opts[:debug]
|
|
command << "-quality" << opts[:quality].to_s if opts[:quality]
|
|
command << to
|
|
|
|
Discourse::Utils.execute_command(
|
|
*command,
|
|
failure_message: I18n.t("upload.png_to_jpg_conversion_failure_message"),
|
|
timeout: MAX_CONVERT_FORMAT_SECONDS
|
|
)
|
|
end
|
|
|
|
def should_alter_quality?
|
|
return false if animated?
|
|
|
|
desired_quality = @image_info.type == :png ? SiteSetting.png_to_jpg_quality : SiteSetting.recompress_original_jpg_quality
|
|
@upload.target_image_quality(@file.path, desired_quality).present?
|
|
end
|
|
|
|
def should_downsize?
|
|
max_image_size > 0 && filesize >= max_image_size && !animated?
|
|
end
|
|
|
|
def downsize!
|
|
3.times do
|
|
original_size = filesize
|
|
down_tempfile = Tempfile.new(["down", ".#{@image_info.type}"])
|
|
|
|
from = @file.path
|
|
to = down_tempfile.path
|
|
|
|
OptimizedImage.ensure_safe_paths!(from, to)
|
|
|
|
OptimizedImage.downsize(
|
|
from,
|
|
to,
|
|
"50%",
|
|
scale_image: true,
|
|
raise_on_error: true
|
|
)
|
|
|
|
@file.respond_to?(:close!) ? @file.close! : @file.close
|
|
@file = down_tempfile
|
|
|
|
extract_image_info!
|
|
|
|
return if filesize >= original_size || pixels == 0 || !should_downsize?
|
|
end
|
|
rescue
|
|
@upload.errors.add(:base, I18n.t("upload.optimize_failure_message"))
|
|
end
|
|
|
|
def is_still_too_big?
|
|
if max_image_pixels > 0 && pixels >= max_image_pixels
|
|
@upload.errors.add(:base, I18n.t("upload.images.larger_than_x_megapixels", max_image_megapixels: SiteSetting.max_image_megapixels))
|
|
true
|
|
elsif max_image_size > 0 && filesize >= max_image_size
|
|
@upload.errors.add(:base, I18n.t(
|
|
"upload.images.too_large_humanized", max_size: ActiveSupport::NumberHelper.number_to_human_size(max_image_size)
|
|
))
|
|
true
|
|
else
|
|
false
|
|
end
|
|
end
|
|
|
|
def clean_svg!
|
|
doc = Nokogiri::XML(@file)
|
|
doc.xpath(svg_allowlist_xpath).remove
|
|
doc.xpath("//@*[starts-with(name(), 'on')]").remove
|
|
doc.css('use').each do |use_el|
|
|
if use_el.attr('href')
|
|
use_el.remove_attribute('href') unless use_el.attr('href').starts_with?('#')
|
|
end
|
|
end
|
|
File.write(@file.path, doc.to_s)
|
|
@file.rewind
|
|
end
|
|
|
|
def should_fix_orientation?
|
|
# orientation is between 1 and 8, 1 being the default
|
|
# cf. http://www.daveperrett.com/articles/2012/07/28/exif-orientation-handling-is-a-ghetto/
|
|
@image_info.orientation.to_i > 1
|
|
end
|
|
|
|
MAX_FIX_ORIENTATION_TIME = 5
|
|
def fix_orientation!
|
|
path = @file.path
|
|
|
|
OptimizedImage.ensure_safe_paths!(path)
|
|
path = OptimizedImage.prepend_decoder!(path, nil, filename: "image.#{@image_info.type}")
|
|
|
|
Discourse::Utils.execute_command('convert', path, '-auto-orient', path, timeout: MAX_FIX_ORIENTATION_TIME)
|
|
|
|
extract_image_info!
|
|
end
|
|
|
|
def should_crop?
|
|
return false if ['profile_background', 'card_background', 'custom_emoji'].include?(@opts[:type]) && animated?
|
|
|
|
TYPES_TO_CROP.include?(@opts[:type])
|
|
end
|
|
|
|
def crop!
|
|
max_pixel_ratio = Discourse::PIXEL_RATIOS.max
|
|
filename_with_correct_ext = "image.#{@image_info.type}"
|
|
|
|
case @opts[:type]
|
|
when "avatar"
|
|
width = height = Discourse.avatar_sizes.max
|
|
OptimizedImage.resize(@file.path, @file.path, width, height, filename: filename_with_correct_ext)
|
|
when "profile_background"
|
|
max_width = 850 * max_pixel_ratio
|
|
width, height = ImageSizer.resize(@image_info.size[0], @image_info.size[1], max_width: max_width, max_height: max_width)
|
|
OptimizedImage.downsize(@file.path, @file.path, "#{width}x#{height}\>", filename: filename_with_correct_ext)
|
|
when "card_background"
|
|
max_width = 590 * max_pixel_ratio
|
|
width, height = ImageSizer.resize(@image_info.size[0], @image_info.size[1], max_width: max_width, max_height: max_width)
|
|
OptimizedImage.downsize(@file.path, @file.path, "#{width}x#{height}\>", filename: filename_with_correct_ext)
|
|
when "custom_emoji"
|
|
OptimizedImage.downsize(@file.path, @file.path, "100x100\>", filename: filename_with_correct_ext)
|
|
end
|
|
|
|
extract_image_info!
|
|
end
|
|
|
|
def should_optimize?
|
|
# GIF is too slow (plus, we'll soon be converting them to MP4)
|
|
# Optimizing SVG is useless
|
|
return false if @file.path =~ /\.(gif|svg)$/i
|
|
# Safeguard for large PNGs
|
|
return pixels < 2_000_000 if @file.path =~ /\.png/i
|
|
# Everything else is fine!
|
|
true
|
|
end
|
|
|
|
def optimize!
|
|
OptimizedImage.ensure_safe_paths!(@file.path)
|
|
FileHelper.optimize_image!(@file.path)
|
|
extract_image_info!
|
|
end
|
|
|
|
def filesize
|
|
@filesize || File.size?(@file.path).to_i
|
|
end
|
|
|
|
def max_image_size
|
|
@max_image_size ||= SiteSetting.max_image_size_kb.kilobytes
|
|
end
|
|
|
|
def max_image_pixels
|
|
@max_image_pixels ||= SiteSetting.max_image_megapixels * 1_000_000
|
|
end
|
|
|
|
def pixels
|
|
@image_info.size&.reduce(:*).to_i
|
|
end
|
|
|
|
def svg_allowlist_xpath
|
|
@@svg_allowlist_xpath ||= "//*[#{ALLOWED_SVG_ELEMENTS.map { |e| "name()!='#{e}'" }.join(" and ") }]"
|
|
end
|
|
|
|
def add_metadata!
|
|
@upload.for_private_message = true if @opts[:for_private_message]
|
|
@upload.for_group_message = true if @opts[:for_group_message]
|
|
@upload.for_theme = true if @opts[:for_theme]
|
|
@upload.for_export = true if @opts[:for_export]
|
|
@upload.for_site_setting = true if @opts[:for_site_setting]
|
|
@upload.for_gravatar = true if @opts[:for_gravatar]
|
|
end
|
|
|
|
private
|
|
|
|
def animated?
|
|
return @animated if @animated != nil
|
|
|
|
@animated ||= begin
|
|
is_animated = FastImage.animated?(@file)
|
|
type = @image_info.type.to_s
|
|
|
|
if is_animated != nil
|
|
# FastImage will return nil if it cannot determine if animated
|
|
is_animated
|
|
elsif type == "gif" || type == "webp"
|
|
# Only GIFs, WEBPs and a few other unsupported image types can be animated
|
|
OptimizedImage.ensure_safe_paths!(@file.path)
|
|
|
|
command = ["identify", "-format", "%n\\n", @file.path]
|
|
frames = Discourse::Utils.execute_command(*command, timeout: Upload::MAX_IDENTIFY_SECONDS).to_i rescue 1
|
|
|
|
frames > 1
|
|
else
|
|
false
|
|
end
|
|
end
|
|
end
|
|
|
|
def generate_fake_sha1_hash
|
|
SecureRandom.hex(20)
|
|
end
|
|
end
|