FEATURE: Native theme support

This feature introduces the concept of themes. Themes are an evolution
of site customizations.

Themes introduce two very big conceptual changes:

- A theme may include other "child themes", children can include grand
children and so on.

- A theme may specify a color scheme

The change does away with the idea of "enabled" color schemes.

It also adds a bunch of big niceties like

- You can source a theme from a git repo

- History for themes is much improved

- You can only have a single enabled theme. Themes can be selected by
    users, if you opt for it.

On a technical level this change comes with a whole bunch of goodies

- All CSS is now compiled using a custom pipeline that uses libsass
    see /lib/stylesheet

- There is a single pipeline for css compilation (in the past we used
    one for customizations and another one for the rest of the app

- The stylesheet pipeline is now divorced of sprockets, there is no
   reliance on sprockets for CSS bundling

- CSS is generated with source maps everywhere (including themes) this
    makes debugging much easier

- Our "live reloader" is smarter and avoid a flash of unstyled content
   we run a file watcher in "puma" in dev so you no longer need to run
   rake autospec to watch for CSS changes

lib/stylesheet/common.rb

@@ -0,0 +1,5 @@
+require 'sassc'
+
+module Stylesheet
+  ASSET_ROOT = "#{Rails.root}/app/assets/stylesheets" unless defined? ASSET_ROOT
+end

lib/stylesheet/compiler.rb

@@ -0,0 +1,60 @@
+require_dependency 'stylesheet/common'
+require_dependency 'stylesheet/importer'
+require_dependency 'stylesheet/functions'
+
+module Stylesheet
+
+  class Compiler
+
+    def self.error_as_css(error, label)
+      error = error.message
+      error.gsub!("\n", '\A ')
+      error.gsub!("'", '\27 ')
+
+      "footer { white-space: pre; }
+      footer:after { content: '#{error}' }"
+    end
+
+    def self.compile_asset(asset, options={})
+
+      if Importer.special_imports[asset.to_s]
+        filename = "theme.scss"
+        file = "@import \"#{asset}\";"
+      else
+        filename = "#{asset}.scss"
+        path = "#{ASSET_ROOT}/#{filename}"
+        file = File.read path
+      end
+
+      compile(file,filename,options)
+
+    end
+
+    def self.compile(stylesheet, filename, options={})
+
+
+      source_map_file = options[:source_map_file] || "#{filename.sub(".scss","")}.css.map";
+      engine = SassC::Engine.new(stylesheet,
+                                 importer: Importer,
+                                 filename: filename,
+                                 style: :compressed,
+                                 source_map_file: source_map_file,
+                                 source_map_contents: true,
+                                 theme_id: options[:theme_id],
+                                 load_paths: [ASSET_ROOT])
+
+
+      result = engine.render
+
+      if options[:rtl]
+        require 'r2'
+        [R2.r2(result), nil]
+      else
+        source_map = engine.source_map
+        source_map.force_encoding("UTF-8")
+
+        [result, source_map]
+      end
+    end
+  end
+end

lib/stylesheet/functions.rb

@@ -0,0 +1,9 @@
+module Stylesheet
+  module ScssFunctions
+    def asset_url(path)
+      SassC::Script::String.new("url('#{ActionController::Base.helpers.asset_path(path.value)}')")
+    end
+  end
+end
+
+::SassC::Script::Functions.send :include, Stylesheet::ScssFunctions

lib/stylesheet/importer.rb

@@ -0,0 +1,126 @@
+require_dependency 'stylesheet/common'
+
+module Stylesheet
+  class Importer < SassC::Importer
+
+    @special_imports = {}
+
+    def self.special_imports
+      @special_imports
+    end
+
+    def self.register_import(name, &blk)
+      @special_imports[name] = blk
+    end
+
+    register_import "plugins" do
+      import_files(DiscoursePluginRegistry.stylesheets)
+    end
+
+    register_import "plugins_mobile" do
+      import_files(DiscoursePluginRegistry.mobile_stylesheets)
+    end
+
+    register_import "plugins_desktop" do
+      import_files(DiscoursePluginRegistry.desktop_stylesheets)
+    end
+
+    register_import "plugins_variables" do
+      import_files(DiscoursePluginRegistry.sass_variables)
+    end
+
+    register_import "theme_variables" do
+      contents = ""
+      colors = (@theme_id && theme.color_scheme) ? theme.color_scheme.resolved_colors : ColorScheme.base_colors
+      colors.each do |n, hex|
+        contents << "$#{n}: ##{hex} !default;\n"
+      end
+      Import.new("theme_variable.scss", source: contents)
+    end
+
+    register_import "category_backgrounds" do
+      contents = ""
+      Category.where('uploaded_background_id IS NOT NULL').each do |c|
+        contents << category_css(c) if c.uploaded_background
+      end
+
+      Import.new("categoy_background.scss", source: contents)
+    end
+
+    register_import "embedded_theme" do
+      next unless @theme_id
+
+      theme_import(:common, :embedded_scss)
+    end
+
+    register_import "mobile_theme" do
+      next unless @theme_id
+
+      theme_import(:mobile, :scss)
+    end
+
+    register_import "desktop_theme" do
+      next unless @theme_id
+
+      theme_import(:desktop, :scss)
+    end
+
+    def initialize(options)
+      @theme_id = options[:theme_id]
+    end
+
+    def import_files(files)
+      files.map do |file|
+        # we never want inline css imports, they are a mess
+        # this tricks libsass so it imports inline instead
+        if file =~ /\.css$/
+          file = file[0..-5]
+        end
+        Import.new(file)
+      end
+    end
+
+    def theme_import(target, attr)
+      fields = theme.list_baked_fields(target, attr)
+
+      fields.map do |field|
+        value = field.value
+        if value.present?
+          filename = "#{field.theme.id}/#{field.target_name}-#{field.name}-#{field.theme.name.parameterize}.scss"
+          with_comment = <<COMMENT
+// Theme: #{field.theme.name}
+// Target: #{field.target_name} #{field.name}
+// Last Edited: #{field.updated_at}
+
+#{value}
+COMMENT
+          Import.new(filename, source: with_comment)
+        end
+      end.compact
+    end
+
+    def theme
+      @theme ||= Theme.find(@theme_id)
+    end
+
+    def apply_cdn(url)
+      "#{GlobalSetting.cdn_url}#{url}"
+    end
+
+    def category_css(category)
+      "body.category-#{category.full_slug} { background-image: url(#{apply_cdn(category.uploaded_background.url)}) }\n"
+    end
+
+    def imports(asset, parent_path)
+      if asset[-1] == "*"
+        Dir["#{Stylesheet::ASSET_ROOT}/#{asset}.scss"].map do |path|
+          Import.new(asset[0..-2] + File.basename(path, ".*"))
+        end
+      elsif callback = Importer.special_imports[asset]
+        instance_eval(&callback)
+      else
+        Import.new(asset + ".scss")
+      end
+    end
+  end
+end

lib/stylesheet/manager.rb

@@ -0,0 +1,270 @@
+require_dependency 'distributed_cache'
+require_dependency 'stylesheet/compiler'
+
+module Stylesheet; end
+
+class Stylesheet::Manager
+
+  CACHE_PATH ||= 'tmp/stylesheet-cache'
+  MANIFEST_DIR ||= "#{Rails.root}/tmp/cache/assets/#{Rails.env}"
+  MANIFEST_FULL_PATH ||= "#{MANIFEST_DIR}/stylesheet-manifest"
+
+  @lock = Mutex.new
+
+  def self.cache
+    @cache ||= DistributedCache.new("discourse_stylesheet")
+  end
+
+  def self.clear_theme_cache!
+    cache.hash.keys.select{|k| k =~ /theme/}.each{|k|cache.delete(k)}
+  end
+
+  def self.stylesheet_link_tag(target = :desktop, media = 'all', theme_key = :missing)
+
+    target = target.to_sym
+
+    if theme_key == :missing
+      theme_key = SiteSetting.default_theme_key
+    end
+
+    cache_key = "#{target}_#{theme_key}"
+    tag = cache[cache_key]
+
+    return tag.dup.html_safe if tag
+
+    @lock.synchronize do
+      builder = self.new(target, theme_key)
+      builder.compile unless File.exists?(builder.stylesheet_fullpath)
+      builder.ensure_digestless_file
+      tag = %[<link href="#{builder.stylesheet_path}" media="#{media}" rel="stylesheet" data-target="#{target}"/>]
+      cache[cache_key] = tag
+
+      tag.dup.html_safe
+    end
+  end
+
+  def self.precompile_css
+    themes = Theme.where('user_selectable OR key = ?', SiteSetting.default_theme_key).pluck(:key,:name)
+    themes << nil
+    themes.each do |key,name|
+      [:desktop, :mobile, :desktop_rtl, :mobile_rtl].each do |target|
+        STDERR.puts "precompile target: #{target} #{name}"
+        stylesheet_link_tag(target, nil, key)
+      end
+    end
+    nil
+  end
+
+  def self.last_file_updated
+    if Rails.env.production?
+      @last_file_updated ||= if File.exists?(MANIFEST_FULL_PATH)
+        File.readlines(MANIFEST_FULL_PATH, 'r')[0]
+      else
+        mtime = max_file_mtime
+        FileUtils.mkdir_p(MANIFEST_DIR)
+        File.open(MANIFEST_FULL_PATH, "w") { |f| f.print(mtime) }
+        mtime
+      end
+    else
+      max_file_mtime
+    end
+  end
+
+  def self.max_file_mtime
+    globs = ["#{Rails.root}/app/assets/stylesheets/**/*.*css"]
+
+    Discourse.plugins.map { |plugin| File.dirname(plugin.path) }.each do |path|
+      globs += [
+        "#{path}/plugin.rb",
+        "#{path}/**/*.*css",
+      ]
+    end
+
+    globs.map do |pattern|
+      Dir.glob(pattern).map { |x| File.mtime(x) }.max
+    end.compact.max.to_i
+  end
+
+  def initialize(target = :desktop, theme_key)
+    @target = target
+    @theme_key = theme_key
+  end
+
+  def compile(opts={})
+    unless opts[:force]
+      if File.exists?(stylesheet_fullpath)
+        unless StylesheetCache.where(target: qualified_target, digest: digest).exists?
+          begin
+            source_map = File.read(source_map_fullpath) rescue nil
+            StylesheetCache.add(qualified_target, digest, File.read(stylesheet_fullpath), source_map)
+          rescue => e
+            Rails.logger.warn "Completely unexpected error adding contents of '#{stylesheet_fullpath}' to cache #{e}"
+          end
+        end
+        return true
+      end
+    end
+
+    rtl = @target.to_s =~ /_rtl$/
+    css,source_map = begin
+      Stylesheet::Compiler.compile_asset(
+        @target,
+         rtl: rtl,
+         theme_id: theme&.id,
+         source_map_file: source_map_filename
+      )
+    rescue SassC::SyntaxError => e
+      Rails.logger.error "Failed to compile #{@target} stylesheet: #{e.message}"
+      [Stylesheet::Compiler.error_as_css(e, "#{@target} stylesheet"), nil]
+    end
+
+    FileUtils.mkdir_p(cache_fullpath)
+
+    File.open(stylesheet_fullpath, "w") do |f|
+      f.puts css
+    end
+
+    if source_map.present?
+      File.open(source_map_fullpath, "w") do |f|
+        f.puts source_map
+      end
+    end
+
+    begin
+      StylesheetCache.add(qualified_target, digest, css, source_map)
+    rescue => e
+      Rails.logger.warn "Completely unexpected error adding item to cache #{e}"
+    end
+    css
+  end
+
+  def ensure_digestless_file
+    # file without digest is only for auto-reloading css in dev env
+    unless Rails.env.production? || (File.exist?(stylesheet_fullpath_no_digest) && File.mtime(stylesheet_fullpath) == File.mtime(stylesheet_fullpath_no_digest))
+      FileUtils.cp(stylesheet_fullpath, stylesheet_fullpath_no_digest)
+    end
+  end
+
+  def self.cache_fullpath
+    "#{Rails.root}/#{CACHE_PATH}"
+  end
+
+  def cache_fullpath
+    self.class.cache_fullpath
+  end
+
+  def stylesheet_fullpath
+    "#{cache_fullpath}/#{stylesheet_filename}"
+  end
+
+  def source_map_fullpath
+    "#{cache_fullpath}/#{source_map_filename}"
+  end
+
+  def source_map_filename
+    "#{stylesheet_filename}.map"
+  end
+
+  def stylesheet_fullpath_no_digest
+    "#{cache_fullpath}/#{stylesheet_filename_no_digest}"
+  end
+
+  def stylesheet_cdnpath
+    "#{GlobalSetting.cdn_url}#{stylesheet_relpath}?__ws=#{Discourse.current_hostname}"
+  end
+
+  def stylesheet_path
+    if Rails.env.development?
+      if @target.to_s =~ /theme/
+        stylesheet_relpath
+      else
+        stylesheet_relpath_no_digest
+      end
+    else
+      stylesheet_cdnpath
+    end
+  end
+
+  def root_path
+    "#{GlobalSetting.relative_url_root}/"
+  end
+
+  def stylesheet_relpath
+    "#{root_path}stylesheets/#{stylesheet_filename}"
+  end
+
+  def stylesheet_relpath_no_digest
+    "#{root_path}stylesheets/#{stylesheet_filename_no_digest}"
+  end
+
+  def qualified_target
+    if is_theme?
+      "#{@target}_#{theme.id}"
+    else
+      scheme_string = theme && theme.color_scheme ? "_#{theme.color_scheme.id}" : ""
+      "#{@target}#{scheme_string}"
+    end
+  end
+
+  def stylesheet_filename(with_digest = true)
+    digest_string = "_#{self.digest}" if with_digest
+    "#{qualified_target}#{digest_string}.css"
+  end
+
+  def stylesheet_filename_no_digest
+    stylesheet_filename(_with_digest=false)
+  end
+
+  def is_theme?
+    !!(@target.to_s =~ /_theme$/)
+  end
+
+  # digest encodes the things that trigger a recompile
+  def digest
+    @digest ||= begin
+      if is_theme?
+        theme_digest
+      else
+        color_scheme_digest
+      end
+    end
+  end
+
+  def theme
+    @theme ||= (Theme.find_by(key: @theme_key) || :nil)
+    @theme == :nil ? nil : @theme
+  end
+
+  def theme_digest
+    scss = ""
+
+    if [:mobile_theme, :desktop_theme].include?(@target)
+      scss = theme.resolve_baked_field(:common, :scss)
+      scss += theme.resolve_baked_field(@target.to_s.sub("_theme", ""), :scss)
+    elsif @target == :embedded_theme
+      scss = theme.resolve_baked_field(:common, :embedded_scss)
+    else
+      raise "attempting to look up theme digest for invalid field"
+    end
+
+    Digest::SHA1.hexdigest scss.to_s
+  end
+
+  def color_scheme_digest
+
+    cs = theme&.color_scheme
+    category_updated = Category.where("uploaded_background_id IS NOT NULL").last_updated_at
+
+    if cs || category_updated > 0
+      Digest::SHA1.hexdigest "#{RailsMultisite::ConnectionManagement.current_db}-#{cs&.id}-#{cs&.version}-#{Stylesheet::Manager.last_file_updated}-#{category_updated}"
+    else
+      digest_string = "defaults-#{Stylesheet::Manager.last_file_updated}"
+
+      if cdn_url = GlobalSetting.cdn_url
+        digest_string = "#{digest_string}-#{cdn_url}"
+      end
+
+      Digest::SHA1.hexdigest digest_string
+    end
+  end
+end

lib/stylesheet/watcher.rb

@@ -0,0 +1,70 @@
+require 'listen'
+
+module Stylesheet
+  class Watcher
+
+    def self.watch(paths=nil)
+      watcher = new(paths)
+      watcher.start
+      watcher
+    end
+
+    def initialize(paths)
+      @paths = paths || ["app/assets/stylesheets", "plugins"]
+      @queue = Queue.new
+    end
+
+    def start
+
+      Thread.new do
+        begin
+          while true
+            worker_loop
+          end
+        rescue => e
+          STDERR.puts "CSS change notifier crashed #{e}"
+        end
+      end
+
+
+      root = Rails.root.to_s
+      @paths.each do |watch|
+        Thread.new do
+          begin
+            Listen.to("#{root}/#{watch}") do |modified, added, _|
+              paths = [modified, added].flatten
+              paths.compact!
+              paths.map!{|long| long[(root.length+1)..-1]}
+              process_change(paths)
+            end
+          rescue => e
+            STDERR.puts "Failed to listen for CSS changes at: #{watch}\n#{e}"
+          end
+        end
+      end
+    end
+
+    def worker_loop
+      @queue.pop
+      while @queue.length > 0
+        @queue.pop
+      end
+
+      message = ["desktop", "mobile", "admin"].map do |name|
+        {hash: SecureRandom.hex, name: "/stylesheets/#{name}.css"}
+      end
+
+      Stylesheet::Manager.cache.clear
+      MessageBus.publish '/file-change', message
+    end
+
+    def process_change(paths)
+      paths.each do |path|
+        if path =~ /\.(css|scss)$/
+          @queue.push path
+        end
+      end
+    end
+
+  end
+end