diff --git a/lib/tasks/autospec.rake b/lib/tasks/autospec.rake index 0a6c9fadd5c..971eadc0601 100644 --- a/lib/tasks/autospec.rake +++ b/lib/tasks/autospec.rake @@ -22,3 +22,21 @@ task "autospec" => :environment do Autospec::Manager.run(force_polling: force_polling, latency: latency, debug: debug) end + +desc "Regenerate swagger docs on API spec change" +task "autospec:swagger" => :environment do + require 'listen' + require 'open3' + + puts "Listening to changes in spec/requests/api to regenerate Swagger docs." + listener = Listen.to("spec/requests/api") do |modified, added, removed| + puts "API doc file changed." + Open3.popen3("spec/regenerate_swagger_docs") do |stdin, stdout, stderr, wait_thr| + while line = stdout.gets + puts line + end + end + end + listener.start + sleep +end diff --git a/spec/fabricators/external_upload_stub_fabricator.rb b/spec/fabricators/external_upload_stub_fabricator.rb index d05ba626b32..5f8eb3e25e0 100644 --- a/spec/fabricators/external_upload_stub_fabricator.rb +++ b/spec/fabricators/external_upload_stub_fabricator.rb @@ -22,3 +22,8 @@ Fabricator(:attachment_external_upload_stub, from: :external_upload_stub) do filesize 1024 key { |attrs| FileStore::BaseStore.temporary_upload_path("file.pdf", folder_prefix: attrs[:folder_prefix] || "") } end + +Fabricator(:multipart_external_upload_stub, from: :external_upload_stub) do + multipart true + external_upload_identifier { "#{SecureRandom.hex(6)}._#{SecureRandom.hex(6)}_#{SecureRandom.hex(6)}.d.ghQ" } +end diff --git a/spec/regenerate_swagger_docs b/spec/regenerate_swagger_docs new file mode 100755 index 00000000000..6dc7046ef7e --- /dev/null +++ b/spec/regenerate_swagger_docs @@ -0,0 +1,10 @@ +#!/bin/sh +if [[ -z "${DISCOURSE_REPO_BASE_DIRECTORY}" ]]; then + echo "Set DISCOURSE_REPO_BASE_DIRECTORY before running this script." +else + discourse_api_docs_dir="${DISCOURSE_REPO_BASE_DIRECTORY}/discourse_api_docs/" + RUBYOPT="W0" rake rswag:specs:swaggerize && cp openapi/openapi.yaml ${discourse_api_docs_dir}openapi.yml + (cd $discourse_api_docs_dir ; sh ${discourse_api_docs_dir}openapi_changed.sh) + + echo "Swagger openapi.yml file copied to $discourse_api_docs_dir" +fi diff --git a/spec/requests/api/schemas/json/upload_abort_multipart_request.json b/spec/requests/api/schemas/json/upload_abort_multipart_request.json new file mode 100644 index 00000000000..43ae92d4bc6 --- /dev/null +++ b/spec/requests/api/schemas/json/upload_abort_multipart_request.json @@ -0,0 +1,13 @@ +{ + "additionalProperties": false, + "properties": { + "external_upload_identifier": { + "type": "string", + "description": "The identifier of the multipart upload in the external storage provider. This is the multipart upload_id in AWS S3.", + "example": "84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ" + } + }, + "required": [ + "external_upload_identifier" + ] +} diff --git a/spec/requests/api/schemas/json/upload_batch_presign_multipart_parts_request.json b/spec/requests/api/schemas/json/upload_batch_presign_multipart_parts_request.json new file mode 100644 index 00000000000..a538001430c --- /dev/null +++ b/spec/requests/api/schemas/json/upload_batch_presign_multipart_parts_request.json @@ -0,0 +1,19 @@ +{ + "additionalProperties": false, + "properties": { + "part_numbers": { + "type": "array", + "description": "The part numbers to generate the presigned URLs for, must be between 1 and 10000.", + "example": [1, 2, 3] + }, + "unique_identifier": { + "type": "string", + "description": "The unique identifier returned in the original /create-multipart request.", + "example": "66e86218-80d9-4bda-b4d5-2b6def968705" + } + }, + "required": [ + "part_numbers", + "unique_identifier" + ] +} diff --git a/spec/requests/api/schemas/json/upload_batch_presign_multipart_parts_response.json b/spec/requests/api/schemas/json/upload_batch_presign_multipart_parts_response.json new file mode 100644 index 00000000000..62efc45e831 --- /dev/null +++ b/spec/requests/api/schemas/json/upload_batch_presign_multipart_parts_response.json @@ -0,0 +1,15 @@ +{ + "additionalProperties": false, + "properties": { + "presigned_urls": { + "type": "object", + "description": "The presigned URLs for each part number, which has the part numbers as keys.", + "example": { + "1": "https://discourse-martin-uploads-test.s3.us-east-2.amazonaws.com/temp/uploads/default/123abc/123abc.jpg?partNumber=1&uploadId=123456abcd&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=test&X-Amz-Date=20211222T012336Z&X-Amz-Expires=600&X-Amz-SignedHeaders=host&X-Amz-Signature=abc123" + } + } + }, + "required": [ + "presigned_urls" + ] +} diff --git a/spec/requests/api/schemas/json/upload_complete_external_upload_request.json b/spec/requests/api/schemas/json/upload_complete_external_upload_request.json new file mode 100644 index 00000000000..02c575f4a8c --- /dev/null +++ b/spec/requests/api/schemas/json/upload_complete_external_upload_request.json @@ -0,0 +1,28 @@ +{ + "additionalProperties": false, + "properties": { + "unique_identifier": { + "type": "string", + "example": "66e86218-80d9-4bda-b4d5-2b6def968705", + "description": "The unique identifier returned in the original /generate-presigned-put request." + }, + "for_private_message": { + "type": "string", + "example": "true", + "description": "Optionally set this to true if the upload is for a private message." + }, + "for_site_setting": { + "type": "string", + "example": "true", + "description": "Optionally set this to true if the upload is for a site setting." + }, + "pasted": { + "type": "string", + "example": "true", + "description": "Optionally set this to true if the upload was pasted into the upload area. This will convert PNG files to JPEG." + } + }, + "required": [ + "unique_identifier" + ] +} diff --git a/spec/requests/api/schemas/json/upload_complete_multipart_request.json b/spec/requests/api/schemas/json/upload_complete_multipart_request.json new file mode 100644 index 00000000000..4dc666008cf --- /dev/null +++ b/spec/requests/api/schemas/json/upload_complete_multipart_request.json @@ -0,0 +1,28 @@ +{ + "additionalProperties": false, + "properties": { + "unique_identifier": { + "type": "string", + "example": "66e86218-80d9-4bda-b4d5-2b6def968705", + "description": "The unique identifier returned in the original /create-multipart request." + }, + "parts": { + "type": "array", + "example": [ + { + "part_number": 1, + "etag": "0c376dcfcc2606f4335bbc732de93344" + }, + { + "part_number": 2, + "etag": "09ert8cfcc2606f4335bbc732de91122" + } + ], + "description": "All of the part numbers and their corresponding ETags that have been uploaded must be provided." + } + }, + "required": [ + "unique_identifier", + "parts" + ] +} diff --git a/spec/requests/api/schemas/json/upload_create_multipart_request.json b/spec/requests/api/schemas/json/upload_create_multipart_request.json new file mode 100644 index 00000000000..b9fa7fac21b --- /dev/null +++ b/spec/requests/api/schemas/json/upload_create_multipart_request.json @@ -0,0 +1,39 @@ +{ + "additionalProperties": false, + "properties": { + "upload_type": { + "type": "string", + "enum": [ + "avatar", + "profile_background", + "card_background", + "custom_emoji", + "composer" + ] + }, + "file_name": { + "type": "string", + "example": "IMG_2021.jpeg" + }, + "file_size": { + "type": "integer", + "description": "File size should be represented in bytes.", + "example": 4096 + }, + "metadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "sha1-checksum": { + "type": "string", + "description": "The SHA1 checksum of the upload binary blob. Optionally be provided and serves as an additional security check when later processing the file in complete-external-upload endpoint." + } + } + } + }, + "required": [ + "upload_type", + "file_name", + "file_size" + ] +} diff --git a/spec/requests/api/schemas/json/upload_create_multipart_response.json b/spec/requests/api/schemas/json/upload_create_multipart_response.json new file mode 100644 index 00000000000..0750c821f67 --- /dev/null +++ b/spec/requests/api/schemas/json/upload_create_multipart_response.json @@ -0,0 +1,25 @@ +{ + "additionalProperties": false, + "properties": { + "key": { + "type": "string", + "description": "The path of the temporary file on the external storage service.", + "example": "temp/site/uploads/default/12345/67890.jpg" + }, + "external_upload_identifier": { + "type": "string", + "description": "The identifier of the multipart upload in the external storage provider. This is the multipart upload_id in AWS S3.", + "example": "84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ" + }, + "unique_identifier": { + "type": "string", + "description": "A unique string that identifies the external upload. This must be stored and then sent in the /complete-multipart and /batch-presign-multipart-parts endpoints.", + "example": "66e86218-80d9-4bda-b4d5-2b6def968705" + } + }, + "required": [ + "external_upload_identifier", + "key", + "unique_identifier" + ] +} diff --git a/spec/requests/api/schemas/json/upload_generate_presigned_put_request.json b/spec/requests/api/schemas/json/upload_generate_presigned_put_request.json new file mode 100644 index 00000000000..1d64e11600d --- /dev/null +++ b/spec/requests/api/schemas/json/upload_generate_presigned_put_request.json @@ -0,0 +1,39 @@ +{ + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "avatar", + "profile_background", + "card_background", + "custom_emoji", + "composer" + ] + }, + "file_name": { + "type": "string", + "example": "IMG_2021.jpeg" + }, + "file_size": { + "type": "integer", + "description": "File size should be represented in bytes.", + "example": 4096 + }, + "metadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "sha1-checksum": { + "type": "string", + "description": "The SHA1 checksum of the upload binary blob. Optionally be provided and serves as an additional security check when later processing the file in complete-external-upload endpoint." + } + } + } + }, + "required": [ + "type", + "file_name", + "file_size" + ] +} diff --git a/spec/requests/api/schemas/json/upload_generate_presigned_put_response.json b/spec/requests/api/schemas/json/upload_generate_presigned_put_response.json new file mode 100644 index 00000000000..244fd9909a4 --- /dev/null +++ b/spec/requests/api/schemas/json/upload_generate_presigned_put_response.json @@ -0,0 +1,21 @@ +{ + "additionalProperties": false, + "properties": { + "key": { + "type": "string", + "description": "The path of the temporary file on the external storage service.", + "example": "temp/site/uploads/default/12345/67890.jpg" + }, + "url": { + "type": "string", + "description": "A presigned PUT URL which must be used to upload the file binary blob to.", + "example": "https://file-uploads.s3.us-west-2.amazonaws.com/temp/site/uploads/default/123/456.jpg?x-amz-acl=private&x-amz-meta-sha1-checksum=sha1&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AAAAus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20211221T011246Z&X-Amz-Expires=600&X-Amz-SignedHeaders=host&X-Amz-Signature=12345678" + }, + "unique_identifier": { + "type": "string", + "description": "A unique string that identifies the external upload. This must be stored and then sent in the /complete-external-upload endpoint to complete the direct upload.", + "example": "66e86218-80d9-4bda-b4d5-2b6def968705" + } + } +} + diff --git a/spec/requests/api/uploads_spec.rb b/spec/requests/api/uploads_spec.rb index 17307b4129f..540d350598a 100644 --- a/spec/requests/api/uploads_spec.rb +++ b/spec/requests/api/uploads_spec.rb @@ -22,10 +22,10 @@ describe 'uploads' do parameter name: :params, in: :body, schema: expected_request_schema let(:params) { { - type: 'avatar', - user_id: admin.id, - synchronous: true, - file: logo + 'type' => 'avatar', + 'user_id' => admin.id, + 'synchronous' => true, + 'file' => logo } } produces 'application/json' @@ -40,4 +40,299 @@ describe 'uploads' do end end + + describe "external and multipart uploads" do + before do + setup_s3 + SiteSetting.enable_direct_s3_uploads = true + end + + path '/uploads/generate-presigned-put.json' do + post 'Initiates a direct external upload' do + tags 'Uploads' + operationId 'generatePresignedPut' + consumes 'application/json' + description <<~HEREDOC + Direct external uploads bypass the usual method of creating uploads + via the POST /uploads route, and upload directly to an external provider, + which by default is S3. This route begins the process, and will return + a unique identifier for the external upload as well as a presigned URL + which is where the file binary blob should be uploaded to. + + Once the upload is complete to the external service, you must call the + POST /complete-external-upload route using the unique identifier returned + by this route, which will create any required Upload record in the Discourse + database and also move file from its temporary location to the final + destination in the external storage service. + + #{direct_uploads_disclaimer} + HEREDOC + + expected_request_schema = load_spec_schema('upload_generate_presigned_put_request') + parameter name: :params, in: :body, schema: expected_request_schema + + produces 'application/json' + response '200', 'external upload initialized' do + expected_response_schema = load_spec_schema('upload_generate_presigned_put_response') + schema(expected_response_schema) + + let(:params) { { + 'file_name' => "test.png", + 'type' => "composer", + 'file_size' => 4096, + 'metadata' => { + 'sha1-checksum' => "830869e4ed99128e4352aa72ff5b0ffc26fdc390" + } + } } + + it_behaves_like "a JSON endpoint", 200 do + let(:expected_response_schema) { expected_response_schema } + let(:expected_request_schema) { expected_request_schema } + end + end + end + end + + path '/uploads/complete-external-upload.json' do + post 'Completes a direct external upload' do + let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" } + let!(:external_stub) { Fabricate(:external_upload_stub, created_by: admin) } + let!(:upload) { Fabricate(:upload) } + + before do + ExternalUploadManager.any_instance.stubs(:transform!).returns(upload) + ExternalUploadManager.any_instance.stubs(:destroy!) + external_stub.update(unique_identifier: unique_identifier) + end + + tags 'Uploads' + operationId 'completeExternalUpload' + consumes 'application/json' + description <<~HEREDOC + Completes an external upload initialized with /get-presigned-put. The + file will be moved from its temporary location in external storage to + a final destination in the S3 bucket. An Upload record will also be + created in the database in most cases. + + If a sha1-checksum was provided in the initial request it will also + be compared with the uploaded file in storage to make sure the same + file was uploaded. The file size will be compared for the same reason. + + #{direct_uploads_disclaimer} + HEREDOC + + expected_request_schema = load_spec_schema('upload_complete_external_upload_request') + parameter name: :params, in: :body, schema: expected_request_schema + + produces 'application/json' + response '200', 'external upload initialized' do + expected_response_schema = load_spec_schema('upload_create_response') + schema(expected_response_schema) + + let(:params) { { + 'unique_identifier' => unique_identifier, + } } + + it_behaves_like "a JSON endpoint", 200 do + let(:expected_response_schema) { expected_response_schema } + let(:expected_request_schema) { expected_request_schema } + end + end + end + end + + path '/uploads/create-multipart.json' do + post 'Creates a multipart external upload' do + before do + ExternalUploadManager.stubs(:create_direct_multipart_upload).returns({ + external_upload_identifier: "66e86218-80d9-4bda-b4d5-2b6def968705", + key: "temp/site/uploads/default/12345/67890.jpg", + unique_identifier: "84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ" + }) + end + + tags 'Uploads' + operationId 'createMultipartUpload' + consumes 'application/json' + description <<~HEREDOC + Creates a multipart upload in the external storage provider, storing + a temporary reference to the external upload similar to /get-presigned-put. + + #{direct_uploads_disclaimer} + HEREDOC + + expected_request_schema = load_spec_schema('upload_create_multipart_request') + parameter name: :params, in: :body, schema: expected_request_schema + + produces 'application/json' + response '200', 'external upload initialized' do + expected_response_schema = load_spec_schema('upload_create_multipart_response') + schema(expected_response_schema) + + let(:params) { { + 'file_name' => "test.png", + 'upload_type' => "composer", + 'file_size' => 4096, + 'metadata' => { + 'sha1-checksum' => "830869e4ed99128e4352aa72ff5b0ffc26fdc390" + } + } } + + it_behaves_like "a JSON endpoint", 200 do + let(:expected_response_schema) { expected_response_schema } + let(:expected_request_schema) { expected_request_schema } + end + end + end + end + + path '/uploads/batch-presign-multipart-parts.json' do + post 'Generates batches of presigned URLs for multipart parts' do + let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" } + let!(:external_stub) { Fabricate(:multipart_external_upload_stub, created_by: admin) } + let!(:upload) { Fabricate(:upload) } + + before do + stub_s3_store + external_stub.update(unique_identifier: unique_identifier) + end + + tags 'Uploads' + operationId 'batchPresignMultipartParts' + consumes 'application/json' + description <<~HEREDOC + Multipart uploads are uploaded in chunks or parts to individual presigned + URLs, similar to the one genreated by /generate-presigned-put. The part + numbers provided must be between 1 and 10000. The total number of parts + will depend on the chunk size in bytes that you intend to use to upload + each chunk. For example a 12MB file may have 2 5MB chunks and a final + 2MB chunk, for part numbers 1, 2, and 3. + + This endpoint will return a presigned URL for each part number provided, + which you can then use to send PUT requests for the binary chunk corresponding + to that part. When the part is uploaded, the provider should return an + ETag for the part, and this should be stored along with the part number, + because this is needed to complete the multipart upload. + + #{direct_uploads_disclaimer} + HEREDOC + + expected_request_schema = load_spec_schema('upload_batch_presign_multipart_parts_request') + parameter name: :params, in: :body, schema: expected_request_schema + + produces 'application/json' + response '200', 'external upload initialized' do + expected_response_schema = load_spec_schema('upload_batch_presign_multipart_parts_response') + schema(expected_response_schema) + + let(:params) { { + 'part_numbers' => [1, 2, 3], + 'unique_identifier' => "66e86218-80d9-4bda-b4d5-2b6def968705" + } } + + it_behaves_like "a JSON endpoint", 200 do + let(:expected_response_schema) { expected_response_schema } + let(:expected_request_schema) { expected_request_schema } + end + end + end + end + + path '/uploads/abort-multipart.json' do + post 'Abort multipart upload' do + let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" } + let!(:external_stub) { Fabricate(:multipart_external_upload_stub, created_by: admin) } + let!(:upload) { Fabricate(:upload) } + + before do + stub_s3_store + external_stub.update( + unique_identifier: unique_identifier, + external_upload_identifier: "84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ" + ) + end + + tags 'Uploads' + operationId 'abortMultipart' + consumes 'application/json' + description <<~HEREDOC + This endpoint aborts the multipart upload initiated with /create-multipart. + This should be used when cancelling the upload. It does not matter if parts + were already uploaded into the external storage provider. + + #{direct_uploads_disclaimer} + HEREDOC + + expected_request_schema = load_spec_schema('upload_abort_multipart_request') + parameter name: :params, in: :body, schema: expected_request_schema + + produces 'application/json' + response '200', 'external upload initialized' do + expected_response_schema = load_spec_schema('success_ok_response') + schema(expected_response_schema) + + let(:params) { { + 'external_upload_identifier' => "84x83tmxy398t3y._Q_z8CoJYVr69bE6D7f8J6Oo0434QquLFoYdGVerWFx9X5HDEI_TP_95c34n853495x35345394.d.ghQ" + } } + + it_behaves_like "a JSON endpoint", 200 do + let(:expected_response_schema) { expected_response_schema } + let(:expected_request_schema) { expected_request_schema } + end + end + end + end + + path '/uploads/complete-multipart.json' do + post 'Complete multipart upload' do + let(:unique_identifier) { "66e86218-80d9-4bda-b4d5-2b6def968705" } + let!(:external_stub) { Fabricate(:multipart_external_upload_stub, created_by: admin) } + let!(:upload) { Fabricate(:upload) } + + before do + ExternalUploadManager.any_instance.stubs(:transform!).returns(upload) + ExternalUploadManager.any_instance.stubs(:destroy!) + stub_s3_store + external_stub.update(unique_identifier: unique_identifier) + end + + tags 'Uploads' + operationId 'completeMultipart' + consumes 'application/json' + description <<~HEREDOC + Completes the multipart upload in the external store, and copies the + file from its temporary location to its final location in the store. + All of the parts must have been uploaded to the external storage provider. + An Upload record will be completed in most cases once the file is copied + to its final location. + + #{direct_uploads_disclaimer} + HEREDOC + + expected_request_schema = load_spec_schema('upload_complete_multipart_request') + parameter name: :params, in: :body, schema: expected_request_schema + + produces 'application/json' + response '200', 'external upload initialized' do + expected_response_schema = load_spec_schema('upload_create_response') + schema(expected_response_schema) + + let(:params) { { + 'unique_identifier' => unique_identifier, + 'parts' => [ + { + 'part_number' => 1, + 'etag' => '0c376dcfcc2606f4335bbc732de93344' + } + ] + } } + + it_behaves_like "a JSON endpoint", 200 do + let(:expected_response_schema) { expected_response_schema } + let(:expected_request_schema) { expected_request_schema } + end + end + end + end + end end diff --git a/spec/swagger_helper.rb b/spec/swagger_helper.rb index e72ae826456..884b2029490 100644 --- a/spec/swagger_helper.rb +++ b/spec/swagger_helper.rb @@ -74,6 +74,18 @@ def api_docs_description HEREDOC end +def direct_uploads_disclaimer + <<~HEREDOC + You must have the correct permissions and CORS settings configured in your + external provider. We support AWS S3 as the default. See: + + https://meta.discourse.org/t/-/210469#s3-multipart-direct-uploads-4. + + An external file store must be set up and `enable_direct_s3_uploads` must + be set to true for this endpoint to function. + HEREDOC +end + RSpec.configure do |config| # Specify a root folder where Swagger JSON files are generated # NOTE: If you're using the rswag-api to serve API descriptions, you'll need