diff --git a/.github/actions/reusable-prepare-peertube-run/action.yml b/.github/actions/reusable-prepare-peertube-run/action.yml index 1a6cd2cfd..aa5b897c9 100644 --- a/.github/actions/reusable-prepare-peertube-run/action.yml +++ b/.github/actions/reusable-prepare-peertube-run/action.yml @@ -8,7 +8,7 @@ runs: - name: Setup system dependencies shell: bash run: | - sudo apt-get install postgresql-client-common redis-tools parallel + sudo apt-get install postgresql-client-common redis-tools parallel libimage-exiftool-perl wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz" tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz mkdir -p $HOME/bin diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dfd9fd1c..e5e69b3f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## v4.1.1 + +### Security + + * Strip EXIF data when processing images + +### Docker + + * Fix videos import by installing python 3 + * Install `git` package (may be needed to install some plugins) + +### Bug fixes + + * Fix error when updating a live + * Fix performance regression when rendering HTML and feeds + * Fix player stuck by HTTP request error + + ## v4.1.0 ### IMPORTANT NOTES diff --git a/client/package.json b/client/package.json index 26ca15210..690e3b982 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "peertube-client", - "version": "4.1.0", + "version": "4.1.1", "private": true, "license": "AGPL-3.0", "author": { diff --git a/package.json b/package.json index d96f9675c..fd94620dd 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "peertube", "description": "PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.", - "version": "4.1.0", + "version": "4.1.1", "private": true, "licence": "AGPL-3.0", "engines": { diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts index 9d0c09051..7d6451db9 100644 --- a/server/helpers/image-utils.ts +++ b/server/helpers/image-utils.ts @@ -118,6 +118,8 @@ async function autoResize (options: { const sourceIsPortrait = sourceImage.getWidth() < sourceImage.getHeight() const destIsPortraitOrSquare = newSize.width <= newSize.height + removeExif(sourceImage) + if (sourceIsPortrait && !destIsPortraitOrSquare) { const baseImage = sourceImage.cloneQuiet().cover(newSize.width, newSize.height) .color([ { apply: 'shade', params: [ 50 ] } ]) @@ -144,6 +146,7 @@ function skipProcessing (options: { const { sourceImage, newSize, imageBytes, inputExt, outputExt } = options const { width, height } = newSize + if (hasExif(sourceImage)) return false if (sourceImage.getWidth() > width || sourceImage.getHeight() > height) return false if (inputExt !== outputExt) return false @@ -154,3 +157,11 @@ function skipProcessing (options: { return imageBytes <= 15 * kB } + +function hasExif (image: Jimp) { + return !!(image.bitmap as any).exifBuffer +} + +function removeExif (image: Jimp) { + (image.bitmap as any).exifBuffer = null +} diff --git a/server/helpers/markdown.ts b/server/helpers/markdown.ts index 41c1186ec..a20ac22d4 100644 --- a/server/helpers/markdown.ts +++ b/server/helpers/markdown.ts @@ -7,8 +7,13 @@ const sanitizeHtml = require('sanitize-html') const markdownItEmoji = require('markdown-it-emoji/light') const MarkdownItClass = require('markdown-it') -const markdownItWithHTML = new MarkdownItClass('default', { linkify: true, breaks: true, html: true }) -const markdownItWithoutHTML = new MarkdownItClass('default', { linkify: false, breaks: true, html: false }) +const markdownItForSafeHtml = new MarkdownItClass('default', { linkify: true, breaks: true, html: true }) + .enable(TEXT_WITH_HTML_RULES) + .use(markdownItEmoji) + +const markdownItForPlainText = new MarkdownItClass('default', { linkify: false, breaks: true, html: false }) + .use(markdownItEmoji) + .use(plainTextPlugin) const toSafeHtml = (text: string) => { if (!text) return '' @@ -17,9 +22,7 @@ const toSafeHtml = (text: string) => { const textWithLineFeed = text.replace(//g, '\r\n') // Convert possible markdown (emojis, emphasis and lists) to html - const html = markdownItWithHTML.enable(TEXT_WITH_HTML_RULES) - .use(markdownItEmoji) - .render(textWithLineFeed) + const html = markdownItForSafeHtml.render(textWithLineFeed) // Convert to safe Html return sanitizeHtml(html, defaultSanitizeOptions) @@ -28,12 +31,10 @@ const toSafeHtml = (text: string) => { const mdToOneLinePlainText = (text: string) => { if (!text) return '' - markdownItWithoutHTML.use(markdownItEmoji) - .use(plainTextPlugin) - .render(text) + markdownItForPlainText.render(text) // Convert to safe Html - return sanitizeHtml(markdownItWithoutHTML.plainText, textOnlySanitizeOptions) + return sanitizeHtml(markdownItForPlainText.plainText, textOnlySanitizeOptions) } // --------------------------------------------------------------------------- @@ -47,30 +48,38 @@ export { // Thanks: https://github.com/wavesheep/markdown-it-plain-text function plainTextPlugin (markdownIt: any) { - let lastSeparator = '' - function plainTextRule (state: any) { const text = scan(state.tokens) - markdownIt.plainText = text.replace(/\s+/g, ' ') + markdownIt.plainText = text } function scan (tokens: any[]) { + let lastSeparator = '' let text = '' - for (const token of tokens) { - if (token.children !== null) { - text += scan(token.children) - continue - } - + function buildSeparator (token: any) { if (token.type === 'list_item_close') { lastSeparator = ', ' - } else if (token.type.endsWith('_close')) { + } + + if (token.tag === 'br' || token.type === 'paragraph_close') { lastSeparator = ' ' - } else if (token.content) { - text += lastSeparator - text += token.content + } + } + + for (const token of tokens) { + buildSeparator(token) + + if (token.type !== 'inline') continue + + for (const child of token.children) { + buildSeparator(child) + + if (!child.content) continue + + text += lastSeparator + child.content + lastSeparator = '' } } diff --git a/server/middlewares/validators/videos/video-transcoding.ts b/server/middlewares/validators/videos/video-transcoding.ts index 34f231d45..da6638f4d 100644 --- a/server/middlewares/validators/videos/video-transcoding.ts +++ b/server/middlewares/validators/videos/video-transcoding.ts @@ -37,7 +37,7 @@ const createTranscodingValidator = [ // Prefer using job info table instead of video state because before 4.0 failed transcoded video were stuck in "TO_TRANSCODE" state const info = await VideoJobInfoModel.load(video.id) - if (info && info.pendingTranscode !== 0) { + if (info && info.pendingTranscode > 0) { return res.fail({ status: HttpStatusCode.CONFLICT_409, message: 'This video is already being transcoded' diff --git a/server/tests/fixtures/banner-resized.jpg b/server/tests/fixtures/banner-resized.jpg index 13ea422cb..952732d61 100644 Binary files a/server/tests/fixtures/banner-resized.jpg and b/server/tests/fixtures/banner-resized.jpg differ diff --git a/server/tests/fixtures/exif.jpg b/server/tests/fixtures/exif.jpg new file mode 100644 index 000000000..2997b38e9 Binary files /dev/null and b/server/tests/fixtures/exif.jpg differ diff --git a/server/tests/fixtures/exif.png b/server/tests/fixtures/exif.png new file mode 100644 index 000000000..a1a0113f8 Binary files /dev/null and b/server/tests/fixtures/exif.png differ diff --git a/server/tests/helpers/image.ts b/server/tests/helpers/image.ts index 64bd373cc..475ca8fb2 100644 --- a/server/tests/helpers/image.ts +++ b/server/tests/helpers/image.ts @@ -4,6 +4,7 @@ import 'mocha' import { expect } from 'chai' import { readFile, remove } from 'fs-extra' import { join } from 'path' +import { execPromise } from '@server/helpers/core-utils' import { buildAbsoluteFixturePath, root } from '@shared/core-utils' import { processImage } from '../../../server/helpers/image-utils' @@ -20,40 +21,77 @@ async function checkBuffers (path1: string, path2: string, equals: boolean) { } } +async function hasTitleExif (path: string) { + const result = JSON.parse(await execPromise(`exiftool -json ${path}`)) + + return result[0]?.Title === 'should be removed' +} + describe('Image helpers', function () { const imageDestDir = join(root(), 'test-images') - const imageDest = join(imageDestDir, 'test.jpg') + + const imageDestJPG = join(imageDestDir, 'test.jpg') + const imageDestPNG = join(imageDestDir, 'test.png') + const thumbnailSize = { width: 223, height: 122 } it('Should skip processing if the source image is okay', async function () { const input = buildAbsoluteFixturePath('thumbnail.jpg') - await processImage(input, imageDest, thumbnailSize, true) + await processImage(input, imageDestJPG, thumbnailSize, true) - await checkBuffers(input, imageDest, true) + await checkBuffers(input, imageDestJPG, true) }) it('Should not skip processing if the source image does not have the appropriate extension', async function () { const input = buildAbsoluteFixturePath('thumbnail.png') - await processImage(input, imageDest, thumbnailSize, true) + await processImage(input, imageDestJPG, thumbnailSize, true) - await checkBuffers(input, imageDest, false) + await checkBuffers(input, imageDestJPG, false) }) it('Should not skip processing if the source image does not have the appropriate size', async function () { const input = buildAbsoluteFixturePath('preview.jpg') - await processImage(input, imageDest, thumbnailSize, true) + await processImage(input, imageDestJPG, thumbnailSize, true) - await checkBuffers(input, imageDest, false) + await checkBuffers(input, imageDestJPG, false) }) it('Should not skip processing if the source image does not have the appropriate size', async function () { const input = buildAbsoluteFixturePath('thumbnail-big.jpg') - await processImage(input, imageDest, thumbnailSize, true) + await processImage(input, imageDestJPG, thumbnailSize, true) - await checkBuffers(input, imageDest, false) + await checkBuffers(input, imageDestJPG, false) + }) + + it('Should strip exif for a jpg file that can not be copied', async function () { + const input = buildAbsoluteFixturePath('exif.jpg') + expect(await hasTitleExif(input)).to.be.true + + await processImage(input, imageDestJPG, { width: 100, height: 100 }, true) + await checkBuffers(input, imageDestJPG, false) + + expect(await hasTitleExif(imageDestJPG)).to.be.false + }) + + it('Should strip exif for a jpg file that could be copied', async function () { + const input = buildAbsoluteFixturePath('exif.jpg') + expect(await hasTitleExif(input)).to.be.true + + await processImage(input, imageDestJPG, thumbnailSize, true) + await checkBuffers(input, imageDestJPG, false) + + expect(await hasTitleExif(imageDestJPG)).to.be.false + }) + + it('Should strip exif for png', async function () { + const input = buildAbsoluteFixturePath('exif.png') + expect(await hasTitleExif(input)).to.be.true + + await processImage(input, imageDestPNG, thumbnailSize, true) + expect(await hasTitleExif(imageDestPNG)).to.be.false }) after(async function () { - await remove(imageDest) + await remove(imageDestDir) }) }) diff --git a/server/tests/helpers/markdown.ts b/server/tests/helpers/markdown.ts index 0488a1a05..8177477f6 100644 --- a/server/tests/helpers/markdown.ts +++ b/server/tests/helpers/markdown.ts @@ -30,5 +30,11 @@ describe('Markdown helpers', function () { expect(result).to.equal('Hello coucou') }) + + it('Should convert tags to plain text', function () { + const result = mdToOneLinePlainText(`#déconversion\n#newage\n#histoire`) + + expect(result).to.equal('#déconversion #newage #histoire') + }) }) }) diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts index 9ecc84b5d..dcc16d7ea 100644 --- a/server/tests/shared/checks.ts +++ b/server/tests/shared/checks.ts @@ -25,21 +25,21 @@ async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { expect(content.toString()).to.not.contain(str) } -async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { +async function testImage (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') { const res = await makeGetRequest({ url, - path: imagePath, + path: imageHTTPPath, expectedStatus: HttpStatusCode.OK_200 }) const body = res.body const data = await readFile(join(root(), 'server', 'tests', 'fixtures', imageName + extension)) - const minLength = body.length - ((30 * body.length) / 100) - const maxLength = body.length + ((30 * body.length) / 100) + const minLength = data.length - ((40 * data.length) / 100) + const maxLength = data.length + ((40 * data.length) / 100) - expect(data.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture') - expect(data.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture') + expect(body.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture') + expect(body.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture') } async function testFileExistsOrNot (server: PeerTubeServer, directory: string, filePath: string, exist: boolean) { diff --git a/support/doc/development/tests.md b/support/doc/development/tests.md index 02fc41147..47602156c 100644 --- a/support/doc/development/tests.md +++ b/support/doc/development/tests.md @@ -31,6 +31,12 @@ $ sudo docker run -p 9444:9000 chocobozzz/s3-ninja $ sudo docker run -p 10389:10389 chocobozzz/docker-test-openldap ``` +Ensure you also have these commands: + +``` +$ exiftool --help +``` + ### Test To run all test suites: @@ -39,7 +45,7 @@ To run all test suites: $ npm run test # See scripts/test.sh to run a particular suite ``` -Most of tests can be runned using: +Most of tests can be run using: ```bash TS_NODE_TRANSPILE_ONLY=true npm run mocha -- --timeout 30000 --exit -r ts-node/register -r tsconfig-paths/register --bail server/tests/api/videos/video-transcoder.ts diff --git a/support/docker/production/Dockerfile.bullseye b/support/docker/production/Dockerfile.bullseye index e55da3307..c57b878ee 100644 --- a/support/docker/production/Dockerfile.bullseye +++ b/support/docker/production/Dockerfile.bullseye @@ -2,7 +2,7 @@ FROM node:14-bullseye-slim # Install dependencies RUN apt update \ - && apt install -y --no-install-recommends openssl ffmpeg python3 ca-certificates gnupg gosu build-essential curl \ + && apt install -y --no-install-recommends openssl ffmpeg python3 ca-certificates gnupg gosu build-essential curl git \ && gosu nobody true \ && rm /var/lib/apt/lists/* -fR