mirror of
https://github.com/discourse/discourse.git
synced 2026-08-05 18:57:20 -05:00
Adds the ability to serve a custom llms.txt file at /llms.txt, allowing site admins to provide information for LLM crawlers about the site's structure and important content. The implementation uses a new `llms_txt` upload site setting where admins can upload a .txt or .md file (max 512KB). The file is served via the StaticController, following the same pattern as favicon. For external storage (S3), the content is cached in Redis using the upload's SHA1 as the cache key - this means the cache automatically invalidates when a new file is uploaded. For local storage, the file is served directly via send_file. Blocked crawlers are allowed to access /llms.txt, similar to robots.txt. Ref - t/162690 Requires - https://github.com/discourse/discourse/pull/36939
464 lines
14 KiB
Ruby
464 lines
14 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
RSpec.describe StaticController do
|
|
fab!(:upload)
|
|
|
|
describe "#favicon" do
|
|
let(:filename) { "smallest.png" }
|
|
let(:file) { file_from_fixtures(filename) }
|
|
|
|
let(:upload) { UploadCreator.new(file, filename).create_for(Discourse.system_user.id) }
|
|
|
|
after { Discourse.redis.scan_each(match: "memoize_*").each { |key| Discourse.redis.del(key) } }
|
|
|
|
context "with local store" do
|
|
it "returns the default favicon if favicon has not been configured" do
|
|
get "/favicon/proxied"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.media_type).to eq("image/png")
|
|
expect(response.body.bytesize).to eq(SiteIconManager.favicon.filesize)
|
|
end
|
|
|
|
it "returns the configured favicon" do
|
|
SiteSetting.favicon = upload
|
|
|
|
get "/favicon/proxied"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.media_type).to eq("image/png")
|
|
expect(response.body.bytesize).to eq(upload.filesize)
|
|
end
|
|
end
|
|
|
|
context "with external store" do
|
|
let(:upload) do
|
|
Upload.create!(
|
|
url: "//s3-upload-bucket.s3-us-east-1.amazonaws.com/somewhere/a.png",
|
|
original_filename: filename,
|
|
filesize: file.size,
|
|
user_id: Discourse.system_user.id,
|
|
)
|
|
end
|
|
|
|
before { setup_s3 }
|
|
|
|
it "can proxy a favicon correctly" do
|
|
SiteSetting.favicon = upload
|
|
|
|
stub_request(:get, "https:/#{upload.url}").to_return(status: 200, body: file)
|
|
|
|
get "/favicon/proxied"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.media_type).to eq("image/png")
|
|
expect(response.body.bytesize).to eq(upload.filesize)
|
|
end
|
|
|
|
context "when favicon fails to load" do
|
|
before { FileHelper.stubs(:download).raises(SocketError) }
|
|
|
|
it "creates an admin notice" do
|
|
expect { get "/favicon/proxied" }.to change { AdminNotice.problem.count }.by(1)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "#cdn_asset" do
|
|
let(:site) { RailsMultisite::ConnectionManagement.current_db }
|
|
|
|
it "can serve assets" do
|
|
begin
|
|
assets_path = Rails.public_path.join("assets")
|
|
|
|
FileUtils.mkdir_p(assets_path)
|
|
|
|
file_path = assets_path.join("test.js.br")
|
|
File.write(file_path, "fake brotli file")
|
|
|
|
get "/cdn_asset/#{site}/test.js.br"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.headers["Cache-Control"]).to match(/public/)
|
|
ensure
|
|
File.delete(file_path)
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "#show" do
|
|
before do
|
|
post = create_post
|
|
SiteSetting.tos_topic_id = post.topic.id
|
|
SiteSetting.guidelines_topic_id = post.topic.id
|
|
SiteSetting.privacy_topic_id = post.topic.id
|
|
end
|
|
|
|
context "with a static file that's present" do
|
|
it "should return the right response for /faq" do
|
|
get "/faq"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.body).to include(I18n.t("js.faq"))
|
|
expect(response.body).to include("<title>FAQ - Discourse</title>")
|
|
end
|
|
end
|
|
|
|
[
|
|
["tos", :tos_url, I18n.t("js.tos")],
|
|
["privacy", :privacy_policy_url, I18n.t("js.privacy")],
|
|
].each do |id, setting_name, text|
|
|
context "with #{id}" do
|
|
context "when #{setting_name} site setting is NOT set" do
|
|
it "renders the #{id} page" do
|
|
get "/#{id}"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.body).to include(text)
|
|
end
|
|
end
|
|
|
|
context "when #{setting_name} site setting is set" do
|
|
before { SiteSetting.set(setting_name, "http://example.com/page") }
|
|
|
|
it "redirects to the #{setting_name}" do
|
|
get "/#{id}"
|
|
|
|
expect(response).to redirect_to("http://example.com/page")
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
context "with a missing file" do
|
|
it "should respond 404" do
|
|
get "/static/does-not-exist"
|
|
expect(response.status).to eq(404)
|
|
end
|
|
|
|
context "with modal pages" do
|
|
it "should return the right response for /signup" do
|
|
get "/signup"
|
|
expect(response.status).to eq(200)
|
|
end
|
|
|
|
it "should return the right response for /password-reset" do
|
|
get "/password-reset"
|
|
expect(response.status).to eq(200)
|
|
end
|
|
end
|
|
end
|
|
|
|
it "should redirect to / when logged in and path is /login without redirect" do
|
|
sign_in(Fabricate(:user))
|
|
get "/login"
|
|
expect(response).to redirect_to("/")
|
|
end
|
|
|
|
it "should display the login template when login is required" do
|
|
SiteSetting.login_required = true
|
|
|
|
get "/login"
|
|
|
|
expect(response.status).to eq(200)
|
|
|
|
expect(response.body).to include(
|
|
PrettyText.cook(I18n.t("login_required.welcome_message", title: SiteSetting.title)),
|
|
)
|
|
end
|
|
|
|
context "when login_required is enabled" do
|
|
before { SiteSetting.login_required = true }
|
|
|
|
%w[faq guidelines rules conduct].each do |page_name|
|
|
it "#{page_name} page redirects to login page for anon" do
|
|
get "/#{page_name}"
|
|
expect(response).to redirect_to "/login"
|
|
end
|
|
|
|
it "#{page_name} page loads for logged in user" do
|
|
sign_in(Fabricate(:user))
|
|
|
|
get "/#{page_name}"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.body).to include(I18n.t("js.guidelines"))
|
|
end
|
|
end
|
|
end
|
|
|
|
context "with crawler view" do
|
|
it "should include correct title" do
|
|
get "/faq", headers: { "HTTP_USER_AGENT" => "Googlebot" }
|
|
expect(response.status).to eq(200)
|
|
expect(response.body).to include("<title>FAQ - Discourse</title>")
|
|
end
|
|
end
|
|
|
|
context "with plugin api extensions" do
|
|
after do
|
|
Rails.application.reload_routes!
|
|
StaticController::CUSTOM_PAGES.clear
|
|
end
|
|
|
|
it "adds new topic-backed pages" do
|
|
routes = Proc.new { get "contact" => "static#show", :id => "contact" }
|
|
Discourse::Application.routes.send(:eval_block, routes)
|
|
|
|
topic_id = Fabricate(:post, cooked: "contact info").topic_id
|
|
SiteSetting.test_some_topic_id = topic_id
|
|
|
|
Plugin::Instance.new.add_topic_static_page("contact", topic_id: "test_some_topic_id")
|
|
|
|
get "/contact"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.body).to include("contact info")
|
|
end
|
|
|
|
it "replaces existing topic-backed pages" do
|
|
topic_id = Fabricate(:post, cooked: "Regular FAQ").topic_id
|
|
SiteSetting.test_some_topic_id = topic_id
|
|
|
|
polish_topic_id = Fabricate(:post, cooked: "Polish FAQ").topic_id
|
|
SiteSetting.test_some_other_topic_id = polish_topic_id
|
|
|
|
Plugin::Instance
|
|
.new
|
|
.add_topic_static_page("faq") do
|
|
current_user&.locale == "pl" ? "test_some_other_topic_id" : "test_some_topic_id"
|
|
end
|
|
|
|
get "/faq"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.body).to include("Regular FAQ")
|
|
|
|
sign_in(Fabricate(:user, locale: "pl"))
|
|
get "/faq"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.body).to include("Polish FAQ")
|
|
end
|
|
end
|
|
|
|
it "does not pollute SiteSetting.title (regression)" do
|
|
SiteSetting.title = "test"
|
|
SiteSetting.short_site_description = "something"
|
|
|
|
expect do
|
|
get "/login"
|
|
get "/login"
|
|
end.to_not change { SiteSetting.title }
|
|
end
|
|
|
|
context "without a subfolder" do
|
|
it "redirects as requested when logged in and path is /login with valid redirect param" do
|
|
sign_in(Fabricate(:user))
|
|
get "/login", params: { redirect: "/foo" }
|
|
expect(response).to redirect_to("/foo")
|
|
end
|
|
|
|
it "redirects to / when logged in and path is /login with invalid redirect param" do
|
|
sign_in(Fabricate(:user))
|
|
get "/login", params: { redirect: "//foo" }
|
|
expect(response).to redirect_to("/")
|
|
get "/login", params: { redirect: "foo" }
|
|
expect(response).to redirect_to("/")
|
|
get "/login", params: { redirect: "http://foo" }
|
|
expect(response).to redirect_to("/")
|
|
get "/login", params: { redirect: "www.foo.bar" }
|
|
expect(response).to redirect_to("/")
|
|
end
|
|
|
|
context "when setting the destination_url cookie" do
|
|
it "respects the redirect parameter in the cookie" do
|
|
get "/login", params: { redirect: "/foo" }
|
|
expect(response.cookies["destination_url"]).to eq("/foo")
|
|
end
|
|
|
|
it "respects the redirect parameter including the query params" do
|
|
get "/login", params: { redirect: "/foo?filter=test" }
|
|
expect(response.cookies["destination_url"]).to eq("/foo?filter=test")
|
|
end
|
|
end
|
|
end
|
|
|
|
context "with a subfolder" do
|
|
before { set_subfolder "/sub_test" }
|
|
|
|
it "redirects as requested when logged in and path is /login with valid redirect param" do
|
|
sign_in(Fabricate(:user))
|
|
get "/login", params: { redirect: "/foo" }
|
|
expect(response).to redirect_to("/sub_test/foo")
|
|
end
|
|
|
|
it "redirects to / when logged in and path is /login with invalid redirect param" do
|
|
sign_in(Fabricate(:user))
|
|
get "/login", params: { redirect: "//foo" }
|
|
expect(response).to redirect_to("/sub_test/")
|
|
get "/login", params: { redirect: "foo" }
|
|
expect(response).to redirect_to("/sub_test/")
|
|
get "/login", params: { redirect: "http://foo" }
|
|
expect(response).to redirect_to("/sub_test/")
|
|
get "/login", params: { redirect: "www.foo.bar" }
|
|
expect(response).to redirect_to("/sub_test/")
|
|
end
|
|
|
|
context "when sets the destination_url cookie" do
|
|
it "respects the redirect parameter in the cookie" do
|
|
get "/login", params: { redirect: "/foo" }
|
|
expect(response.cookies["destination_url"]).to eq("/sub_test/foo")
|
|
end
|
|
|
|
it "respects the redirect parameter including the query params" do
|
|
get "/login", params: { redirect: "/foo?filter=test" }
|
|
expect(response.cookies["destination_url"]).to eq("/sub_test/foo?filter=test")
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "#enter" do
|
|
context "without a redirect path" do
|
|
it "redirects to the root url" do
|
|
post "/login.json"
|
|
expect(response).to redirect_to("/")
|
|
end
|
|
end
|
|
|
|
context "with a redirect path" do
|
|
it "redirects to the redirect path" do
|
|
post "/login.json", params: { redirect: "/foo" }
|
|
expect(response).to redirect_to("/foo")
|
|
end
|
|
end
|
|
|
|
context "with a full url" do
|
|
it "redirects to the correct path" do
|
|
post "/login.json", params: { redirect: "#{Discourse.base_url}/foo" }
|
|
expect(response).to redirect_to("/foo")
|
|
end
|
|
end
|
|
|
|
context "with a redirect path with query params" do
|
|
it "redirects to the redirect path and preserves query params" do
|
|
post "/login.json", params: { redirect: "/foo?bar=1" }
|
|
expect(response).to redirect_to("/foo?bar=1")
|
|
end
|
|
end
|
|
|
|
context "with a period to force a new host" do
|
|
it "redirects to the root path" do
|
|
post "/login.json", params: { redirect: ".org/foo" }
|
|
expect(response).to redirect_to("/")
|
|
end
|
|
end
|
|
|
|
context "with a full url to an external host" do
|
|
it "redirects to the root path" do
|
|
post "/login.json", params: { redirect: "http://eviltrout.com/foo" }
|
|
expect(response).to redirect_to("/")
|
|
end
|
|
end
|
|
|
|
context "with an invalid URL" do
|
|
it "redirects to the root" do
|
|
post "/login.json", params: { redirect: "javascript:alert('trout')" }
|
|
expect(response).to redirect_to("/")
|
|
end
|
|
end
|
|
|
|
context "with an array" do
|
|
it "redirects to the root" do
|
|
post "/login.json", params: { redirect: ["/foo"] }
|
|
expect(response).to redirect_to("/")
|
|
end
|
|
end
|
|
|
|
context "when the redirect path is the login page" do
|
|
it "redirects to the root url" do
|
|
post "/login.json", params: { redirect: login_path }
|
|
expect(response).to redirect_to("/")
|
|
end
|
|
end
|
|
|
|
context "when the redirect path contains the '/login' string" do
|
|
it "redirects to the requested path" do
|
|
post "/login.json", params: { redirect: "/page/login/1" }
|
|
expect(response).to redirect_to("/page/login/1")
|
|
end
|
|
end
|
|
context "when the redirect path is invalid" do
|
|
it "redirects to the root URL" do
|
|
post "/login.json", params: { redirect: "test" }
|
|
expect(response).to redirect_to("/")
|
|
end
|
|
end
|
|
|
|
context "with a subfolder" do
|
|
before { set_subfolder "/sub_test" }
|
|
|
|
context "without a redirect path" do
|
|
it "redirects to the subfolder root" do
|
|
post "/login.json"
|
|
expect(response).to redirect_to("/sub_test/")
|
|
end
|
|
end
|
|
|
|
context "when the redirect path is the login page" do
|
|
it "redirects to the subfolder root" do
|
|
post "/login.json", params: { redirect: "#{Discourse.base_path}/login" }
|
|
expect(response).to redirect_to("/sub_test/")
|
|
end
|
|
end
|
|
|
|
context "when the redirect path is invalid" do
|
|
it "redirects to the subfolder root" do
|
|
post "/login.json", params: { redirect: "test" }
|
|
expect(response).to redirect_to("/sub_test/")
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "#service_worker_asset" do
|
|
it "works" do
|
|
get "/service-worker.js"
|
|
expect(response.status).to eq(200)
|
|
expect(response.content_type).to start_with("text/javascript")
|
|
expect(response.body).to include("addEventListener")
|
|
end
|
|
end
|
|
|
|
describe "#llms_txt" do
|
|
it "returns 404 when no upload is set" do
|
|
get "/llms.txt"
|
|
expect(response.status).to eq(404)
|
|
end
|
|
|
|
context "with local store" do
|
|
it "returns content as plain text" do
|
|
SiteSetting.authorized_extensions = "txt"
|
|
|
|
file = Tempfile.new(%w[llms .txt])
|
|
file.write("# Test LLMs Content")
|
|
file.rewind
|
|
|
|
upload = UploadCreator.new(file, "llms.txt").create_for(Discourse.system_user.id)
|
|
SiteSetting.llms_txt = upload
|
|
|
|
get "/llms.txt"
|
|
|
|
expect(response.status).to eq(200)
|
|
expect(response.content_type).to start_with("text/plain")
|
|
expect(response.body).to eq("# Test LLMs Content")
|
|
ensure
|
|
file.close
|
|
file.unlink
|
|
end
|
|
end
|
|
end
|
|
end
|