mirror of
https://github.com/discourse/discourse.git
synced 2025-02-25 18:55:32 -06:00
DEV: Apply syntax_tree formatting to spec/*
This commit is contained in:
@@ -1,13 +1,13 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
RSpec.describe RobotsTxtController do
|
||||
describe '#builder' do
|
||||
describe "#builder" do
|
||||
it "returns json information for building a robots.txt" do
|
||||
get "/robots-builder.json"
|
||||
json = response.parsed_body
|
||||
expect(json).to be_present
|
||||
expect(json['header']).to be_present
|
||||
expect(json['agents']).to be_present
|
||||
expect(json["header"]).to be_present
|
||||
expect(json["agents"]).to be_present
|
||||
end
|
||||
|
||||
it "includes overridden content if robots.txt is is overridden" do
|
||||
@@ -16,64 +16,76 @@ RSpec.describe RobotsTxtController do
|
||||
get "/robots-builder.json"
|
||||
expect(response.status).to eq(200)
|
||||
json = response.parsed_body
|
||||
expect(json['header']).to be_present
|
||||
expect(json['agents']).to be_present
|
||||
expect(json['overridden']).to eq("something")
|
||||
expect(json["header"]).to be_present
|
||||
expect(json["agents"]).to be_present
|
||||
expect(json["overridden"]).to eq("something")
|
||||
end
|
||||
end
|
||||
|
||||
describe '#index' do
|
||||
describe "#index" do
|
||||
context "when the content is overridden" do
|
||||
it "is not prepended if there are no overrides" do
|
||||
sign_in(Fabricate(:admin))
|
||||
get '/robots.txt'
|
||||
get "/robots.txt"
|
||||
expect(response.body).not_to start_with(RobotsTxtController::OVERRIDDEN_HEADER)
|
||||
end
|
||||
|
||||
it "is prepended if there are overrides and the user is admin" do
|
||||
SiteSetting.overridden_robots_txt = "overridden_content"
|
||||
sign_in(Fabricate(:admin))
|
||||
get '/robots.txt'
|
||||
get "/robots.txt"
|
||||
expect(response.body).to start_with(RobotsTxtController::OVERRIDDEN_HEADER)
|
||||
end
|
||||
|
||||
it "is not prepended if the user is not admin" do
|
||||
SiteSetting.overridden_robots_txt = "overridden_content"
|
||||
get '/robots.txt'
|
||||
get "/robots.txt"
|
||||
expect(response.body).not_to start_with(RobotsTxtController::OVERRIDDEN_HEADER)
|
||||
end
|
||||
end
|
||||
|
||||
context 'with subfolder' do
|
||||
it 'prefixes the rules with the directory' do
|
||||
context "with subfolder" do
|
||||
it "prefixes the rules with the directory" do
|
||||
set_subfolder "/forum"
|
||||
|
||||
get '/robots.txt'
|
||||
get "/robots.txt"
|
||||
expect(response.body).to include("\nDisallow: /forum/email/")
|
||||
end
|
||||
end
|
||||
|
||||
context 'when allow_index_in_robots_txt is true' do
|
||||
context "when allow_index_in_robots_txt is true" do
|
||||
def expect_allowed_and_disallowed_sections(allow_index, disallow_index)
|
||||
expect(allow_index).to be_present
|
||||
expect(disallow_index).to be_present
|
||||
|
||||
allow_section = allow_index < disallow_index ?
|
||||
response.body[allow_index...disallow_index] : response.body[allow_index..-1]
|
||||
allow_section =
|
||||
(
|
||||
if allow_index < disallow_index
|
||||
response.body[allow_index...disallow_index]
|
||||
else
|
||||
response.body[allow_index..-1]
|
||||
end
|
||||
)
|
||||
|
||||
expect(allow_section).to include('Disallow: /auth/')
|
||||
expect(allow_section).to include("Disallow: /auth/")
|
||||
expect(allow_section).to_not include("Disallow: /\n")
|
||||
|
||||
disallowed_section = allow_index < disallow_index ?
|
||||
response.body[disallow_index..-1] : response.body[disallow_index...allow_index]
|
||||
disallowed_section =
|
||||
(
|
||||
if allow_index < disallow_index
|
||||
response.body[disallow_index..-1]
|
||||
else
|
||||
response.body[disallow_index...allow_index]
|
||||
end
|
||||
)
|
||||
expect(disallowed_section).to include("Disallow: /\n")
|
||||
end
|
||||
|
||||
it "returns index when indexing is allowed" do
|
||||
SiteSetting.allow_index_in_robots_txt = true
|
||||
get '/robots.txt'
|
||||
get "/robots.txt"
|
||||
|
||||
i = response.body.index('User-agent: *')
|
||||
i = response.body.index("User-agent: *")
|
||||
expect(i).to be_present
|
||||
expect(response.body[i..-1]).to include("Disallow: /auth/")
|
||||
# we have to insert Googlebot for special handling
|
||||
@@ -82,24 +94,30 @@ RSpec.describe RobotsTxtController do
|
||||
|
||||
it "can allowlist user agents" do
|
||||
SiteSetting.allowed_crawler_user_agents = "Googlebot|Twitterbot"
|
||||
get '/robots.txt'
|
||||
expect(response.body).to include('User-agent: Googlebot')
|
||||
expect(response.body).to include('User-agent: Twitterbot')
|
||||
get "/robots.txt"
|
||||
expect(response.body).to include("User-agent: Googlebot")
|
||||
expect(response.body).to include("User-agent: Twitterbot")
|
||||
|
||||
allowed_index = [response.body.index('User-agent: Googlebot'), response.body.index('User-agent: Twitterbot')].min
|
||||
disallow_all_index = response.body.index('User-agent: *')
|
||||
allowed_index = [
|
||||
response.body.index("User-agent: Googlebot"),
|
||||
response.body.index("User-agent: Twitterbot"),
|
||||
].min
|
||||
disallow_all_index = response.body.index("User-agent: *")
|
||||
|
||||
expect_allowed_and_disallowed_sections(allowed_index, disallow_all_index)
|
||||
end
|
||||
|
||||
it "can blocklist user agents" do
|
||||
SiteSetting.blocked_crawler_user_agents = "Googlebot|Twitterbot"
|
||||
get '/robots.txt'
|
||||
expect(response.body).to include('User-agent: Googlebot')
|
||||
expect(response.body).to include('User-agent: Twitterbot')
|
||||
get "/robots.txt"
|
||||
expect(response.body).to include("User-agent: Googlebot")
|
||||
expect(response.body).to include("User-agent: Twitterbot")
|
||||
|
||||
disallow_index = [response.body.index('User-agent: Googlebot'), response.body.index('User-agent: Twitterbot')].min
|
||||
allow_index = response.body.index('User-agent: *')
|
||||
disallow_index = [
|
||||
response.body.index("User-agent: Googlebot"),
|
||||
response.body.index("User-agent: Twitterbot"),
|
||||
].min
|
||||
allow_index = response.body.index("User-agent: *")
|
||||
|
||||
expect_allowed_and_disallowed_sections(allow_index, disallow_index)
|
||||
end
|
||||
@@ -107,16 +125,16 @@ RSpec.describe RobotsTxtController do
|
||||
it "ignores blocklist if allowlist is set" do
|
||||
SiteSetting.allowed_crawler_user_agents = "Googlebot|Twitterbot"
|
||||
SiteSetting.blocked_crawler_user_agents = "Bananabot"
|
||||
get '/robots.txt'
|
||||
expect(response.body).to_not include('Bananabot')
|
||||
expect(response.body).to include('User-agent: Googlebot')
|
||||
expect(response.body).to include('User-agent: Twitterbot')
|
||||
get "/robots.txt"
|
||||
expect(response.body).to_not include("Bananabot")
|
||||
expect(response.body).to include("User-agent: Googlebot")
|
||||
expect(response.body).to include("User-agent: Twitterbot")
|
||||
end
|
||||
end
|
||||
|
||||
it "returns noindex when indexing is disallowed" do
|
||||
SiteSetting.allow_index_in_robots_txt = false
|
||||
get '/robots.txt'
|
||||
get "/robots.txt"
|
||||
|
||||
expect(response.body).to_not include("Disallow: /auth/")
|
||||
expect(response.body).to include("User-agent: googlebot\nAllow")
|
||||
@@ -124,19 +142,21 @@ RSpec.describe RobotsTxtController do
|
||||
|
||||
it "returns overridden robots.txt if the file is overridden" do
|
||||
SiteSetting.overridden_robots_txt = "blah whatever"
|
||||
get '/robots.txt'
|
||||
get "/robots.txt"
|
||||
expect(response.status).to eq(200)
|
||||
expect(response.body).to eq(SiteSetting.overridden_robots_txt)
|
||||
end
|
||||
|
||||
describe 'sitemap' do
|
||||
let(:sitemap_line) { "Sitemap: #{Discourse.base_protocol}://#{Discourse.current_hostname}/sitemap.xml" }
|
||||
describe "sitemap" do
|
||||
let(:sitemap_line) do
|
||||
"Sitemap: #{Discourse.base_protocol}://#{Discourse.current_hostname}/sitemap.xml"
|
||||
end
|
||||
|
||||
it 'include sitemap location when enabled' do
|
||||
it "include sitemap location when enabled" do
|
||||
SiteSetting.enable_sitemap = true
|
||||
SiteSetting.login_required = false
|
||||
|
||||
get '/robots.txt'
|
||||
get "/robots.txt"
|
||||
|
||||
expect(response.body).to include(sitemap_line)
|
||||
end
|
||||
@@ -145,7 +165,7 @@ RSpec.describe RobotsTxtController do
|
||||
SiteSetting.enable_sitemap = false
|
||||
SiteSetting.login_required = false
|
||||
|
||||
get '/robots.txt'
|
||||
get "/robots.txt"
|
||||
|
||||
expect(response.body).not_to include(sitemap_line)
|
||||
end
|
||||
@@ -154,27 +174,23 @@ RSpec.describe RobotsTxtController do
|
||||
SiteSetting.enable_sitemap = true
|
||||
SiteSetting.login_required = true
|
||||
|
||||
get '/robots.txt'
|
||||
get "/robots.txt"
|
||||
|
||||
expect(response.body).not_to include(sitemap_line)
|
||||
end
|
||||
end
|
||||
|
||||
describe 'plugins' do
|
||||
describe "plugins" do
|
||||
let(:event_handler) do
|
||||
Proc.new { |robots_info| robots_info[:agents] << { name: 'Test', disallow: ['/test/'] } }
|
||||
Proc.new { |robots_info| robots_info[:agents] << { name: "Test", disallow: ["/test/"] } }
|
||||
end
|
||||
|
||||
before do
|
||||
DiscourseEvent.on(:robots_info, &event_handler)
|
||||
end
|
||||
before { DiscourseEvent.on(:robots_info, &event_handler) }
|
||||
|
||||
after do
|
||||
DiscourseEvent.off(:robots_info, &event_handler)
|
||||
end
|
||||
after { DiscourseEvent.off(:robots_info, &event_handler) }
|
||||
|
||||
it 'can add to robots.txt' do
|
||||
get '/robots.txt'
|
||||
it "can add to robots.txt" do
|
||||
get "/robots.txt"
|
||||
|
||||
expect(response.parsed_body).to include("User-agent: Test\nDisallow: /test/")
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user