added support for disabling indexing by google using SiteSetting.allow_index_in_robots_txt = false

app/controllers/robots_txt_controller.rb

@@ -0,0 +1,14 @@
+class RobotsTxtController < ApplicationController
+  layout false
+  skip_before_filter :check_xhr
+
+  def index
+    path = if SiteSetting.allow_index_in_robots_txt && !SiteSetting.restrict_access
+      :index
+    else
+      :no_index
+    end
+    
+    render path, content_type: 'text/plain'
+  end
+end