From c6c824639900c1e0a3f9417de18c38c54e36eb6d Mon Sep 17 00:00:00 2001 From: Sam Date: Sun, 7 Jul 2013 14:30:52 +1000 Subject: [PATCH] added sample unicorn config added utility to measure real memory usage don't require thin by default --- Gemfile | 4 +- Gemfile.lock | 7 ++ config/unicorn.sample.conf.rb | 80 ++++++++++++++++++++ script/memstats.rb | 136 ++++++++++++++++++++++++++++++++++ 4 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 config/unicorn.sample.conf.rb create mode 100644 script/memstats.rb diff --git a/Gemfile b/Gemfile index d66fa0a29c0..bc8abdb3cd8 100644 --- a/Gemfile +++ b/Gemfile @@ -66,7 +66,7 @@ gem 'sinatra', require: nil gem 'slim' # required for sidekiq-web gem 'strong_parameters' # remove when we upgrade to Rails 4 gem 'therubyracer', require: 'v8' -gem 'thin' +gem 'thin', require: false gem 'diffy', require: false gem 'highline', require: false @@ -146,8 +146,8 @@ gem 'rack-mini-profiler', '0.1.27', require: false # require: false #, git: 'gi # https://github.com/jodosha/redis-store/pull/183 gem 'redis-rack-cache', git: 'https://github.com/SamSaffron/redis-rack-cache.git', require: false gem 'rack-cache', require: false - gem 'rack-cors', require: false +gem 'unicorn', require: false # perftools only works on 1.9 atm group :profile do diff --git a/Gemfile.lock b/Gemfile.lock index 15203c20dcc..7d3adaffa06 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -230,6 +230,7 @@ GEM json (1.7.7) jwt (0.1.8) multi_json (>= 1.5) + kgio (2.8.0) librarian (0.1.0) highline thor (~> 0.15) @@ -334,6 +335,7 @@ GEM rake (>= 0.8.7) rdoc (~> 3.4) thor (>= 0.14.6, < 2.0) + raindrops (0.11.0) rake (10.0.4) rake-compiler (0.8.3) rake @@ -455,6 +457,10 @@ GEM uglifier (2.0.1) execjs (>= 0.3.0) multi_json (~> 1.0, >= 1.0.2) + unicorn (4.6.3) + kgio (~> 2.6) + rack + raindrops (~> 0.7) PLATFORMS ruby @@ -549,4 +555,5 @@ DEPENDENCIES timecop turbo-sprockets-rails3 uglifier + unicorn vestal_versions! diff --git a/config/unicorn.sample.conf.rb b/config/unicorn.sample.conf.rb new file mode 100644 index 00000000000..9887490dbba --- /dev/null +++ b/config/unicorn.sample.conf.rb @@ -0,0 +1,80 @@ +# See http://unicorn.bogomips.org/Unicorn/Configurator.html + +discourse_path = File.expand_path(File.expand_path(File.dirname(__FILE__)) + "/../") + +# tune down if not enough ram +worker_processes 2 + +working_directory discourse_path + +listen 8080, :tcp_nopush => true + +# nuke workers after 30 seconds instead of 60 seconds (the default) +timeout 30 + +# feel free to point this anywhere accessible on the filesystem +pid "#{discourse_path}/tmp/pids/unicorn.pid" + +# By default, the Unicorn logger will write to stderr. +# Additionally, some applications/frameworks log to stderr or stdout, +# so prevent them from going to /dev/null when daemonized here: +stderr_path "#{discourse_path}/log/unicorn.stderr.log" +stdout_path "#{discourse_path}/log/unicorn.stdout.log" + +# important for Ruby 2.0 +preload_app true + +# Enable this flag to have unicorn test client connections by writing the +# beginning of the HTTP headers before calling the application. This +# prevents calling the application for connections that have disconnected +# while queued. This is only guaranteed to detect clients on the same +# host unicorn runs on, and unlikely to detect disconnects even on a +# fast LAN. +check_client_connection false + +initialized = false +before_fork do |server, worker| + unless initialized + # load up the yaml for the localization bits, in master process + I18n.t(:posts) + # get rid of rubbish so we don't share it + GC.start + end + ActiveRecord::Base.connection.disconnect! + $redis.client.disconnect + + + #TODO + # at this point we want to fork out sidekiq, it will let us reuse the shared memory + + # The following is only recommended for memory/DB-constrained + # installations. It is not needed if your system can house + # twice as many worker_processes as you have configured. + # + # # This allows a new master process to incrementally + # # phase out the old master process with SIGTTOU to avoid a + # # thundering herd (especially in the "preload_app false" case) + # # when doing a transparent upgrade. The last worker spawned + # # will then kill off the old master process with a SIGQUIT. + # old_pid = "#{server.config[:pid]}.oldbin" + # if old_pid != server.pid + # begin + # sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU + # Process.kill(sig, File.read(old_pid).to_i) + # rescue Errno::ENOENT, Errno::ESRCH + # end + # end + # + # Throttle the master from forking too quickly by sleeping. Due + # to the implementation of standard Unix signal handlers, this + # helps (but does not completely) prevent identical, repeated signals + # from being lost when the receiving process is busy. + # sleep 1 +end + +after_fork do |server, worker| + ActiveRecord::Base.establish_connection + $redis.client.reconnect + Rails.cache.reconnect + MessageBus.after_fork +end diff --git a/script/memstats.rb b/script/memstats.rb new file mode 100644 index 00000000000..70cc3ce50eb --- /dev/null +++ b/script/memstats.rb @@ -0,0 +1,136 @@ +#!/usr/bin/env ruby +# from: https://gist.github.com/kenn/5105061/raw/ac7ebc6be7008c35b72560cc4e05b7cc14eb4919/memstats.rb + +#------------------------------------------------------------------------------ +# Aggregate Print useful information from /proc/[pid]/smaps +# +# pss - Roughly the amount of memory that is "really" being used by the pid +# swap - Amount of swap this process is currently using +# +# Reference: +# http://www.mjmwired.net/kernel/Documentation/filesystems/proc.txt#361 +# +# Example: +# # ./memstats.rb 4386 +# Process: 4386 +# Command Line: /usr/bin/mongod -f /etc/mongo/mongod.conf +# Memory Summary: +# private_clean 107,132 kB +# private_dirty 2,020,676 kB +# pss 2,127,860 kB +# rss 2,128,536 kB +# shared_clean 728 kB +# shared_dirty 0 kB +# size 149,281,668 kB +# swap 1,719,792 kB +#------------------------------------------------------------------------------ + +class Mapping + FIELDS = %w[ size rss shared_clean shared_dirty private_clean private_dirty swap pss ] + attr_reader :address_start + attr_reader :address_end + attr_reader :perms + attr_reader :offset + attr_reader :device_major + attr_reader :device_minor + attr_reader :inode + attr_reader :region + + attr_accessor :size + attr_accessor :rss + attr_accessor :shared_clean + attr_accessor :shared_dirty + attr_accessor :private_dirty + attr_accessor :private_clean + attr_accessor :swap + attr_accessor :pss + + def initialize( lines ) + + FIELDS.each do |field| + self.send("#{field}=", 0) + end + + parse_first_line( lines.shift ) + lines.each do |l| + parse_field_line(l) + end + end + + def parse_first_line( line ) + parts = line.strip.split + @address_start, @address_end = parts[0].split("-") + @perms = parts[1] + @offset = parts[2] + @device_major, @device_minor = parts[3].split(":") + @inode = parts[4] + @region = parts[5] || "anonymous" + end + + def parse_field_line( line ) + parts = line.strip.split + field = parts[0].downcase.sub(':','') + if respond_to? "#{field}=" + value = Float(parts[1]).to_i + self.send( "#{field}=", value ) + end + end +end + +def consume_mapping( map_lines, totals ) + m = Mapping.new( map_lines ) + + Mapping::FIELDS.each do |field| + totals[field] += m.send( field ) + end + return m +end + +abort 'usage: memstats [pid]' unless ARGV.first +pid = ARGV.shift.to_i +totals = Hash.new(0) +mappings = [] + +File.open( "/proc/#{pid}/smaps" ) do |smaps| + + map_lines = [] + + loop do + break if smaps.eof? + line = smaps.readline.strip + case line + when /\w+:\s+/ + map_lines << line + when /[0-9a-f]+:[0-9a-f]+\s+/ + if map_lines.size > 0 then + mappings << consume_mapping( map_lines, totals ) + end + map_lines.clear + map_lines << line + else + break + end + end +end + +# http://rubyforge.org/snippet/download.php?type=snippet&id=511 +def format_number( n ) + n.to_s.gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2') +end + +def get_commandline( pid ) + commandline = IO.read( "/proc/#{pid}/cmdline" ).split("\0") + if commandline.first =~ /java$/ then + loop { break if commandline.shift == "-jar" } + return "[java] #{commandline.shift}" + end + return commandline.join(' ') +end + + +puts "#{"Process:".ljust(20)} #{pid}" +puts "#{"Command Line:".ljust(20)} #{get_commandline(pid)}" +puts "Memory Summary:" +totals.keys.sort.each do |k| + puts " #{k.ljust(20)} #{format_number( totals[k] ).rjust(12)} kB" +end