From 5439a591e0eb24dc8cb62ca95d464e9e95602447 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 13 Aug 2024 10:49:31 +0200 Subject: [PATCH] Add ability to set proxies to youtube-dl --- .github/workflows/test.yml | 1 + config/default.yaml | 6 ++++++ config/production.yaml.example | 6 ++++++ packages/node-utils/src/crypto.ts | 15 ++------------- packages/server-commands/src/server/server.ts | 13 ++++++++++--- .../core/helpers/youtube-dl/youtube-dl-cli.ts | 18 +++++++++++++----- .../core/initializers/checker-before-init.ts | 1 + server/core/initializers/config.ts | 4 +++- support/doc/development/tests.md | 1 + 9 files changed, 43 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e90a20f1..10f5c194a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,6 +50,7 @@ jobs: OBJECT_STORAGE_SCALEWAY_KEY_ID: ${{ secrets.OBJECT_STORAGE_SCALEWAY_KEY_ID }} OBJECT_STORAGE_SCALEWAY_ACCESS_KEY: ${{ secrets.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY }} YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + YOUTUBE_DL_PROXY: ${{ secrets.YOUTUBE_DL_PROXY }} steps: - uses: actions/checkout@v4 diff --git a/config/default.yaml b/config/default.yaml index 92c95550e..5fcb03613 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -803,6 +803,12 @@ import: # IPv6 is very strongly rate-limited on most sites supported by youtube-dl force_ipv4: false + # By default PeerTube uses HTTP_PROXY and HTTPS_PROXY environment variables + # But you can specify custom proxies for youtube-dl because remote websites (like YouTube) may block your server IP address + # PeerTube will randomly select a proxy from the following list + proxies: +# - "https://username:password@example.com:8888" + # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) torrent: # We recommend to only enable magnet URI/torrent import if you trust your users diff --git a/config/production.yaml.example b/config/production.yaml.example index 75599a88e..0b186ab65 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -813,6 +813,12 @@ import: # IPv6 is very strongly rate-limited on most sites supported by youtube-dl force_ipv4: false + # By default PeerTube uses HTTP_PROXY and HTTPS_PROXY environment variables + # But you can specify custom proxies for youtube-dl because remote websites (like YouTube) may block your server IP address + # PeerTube will randomly select a proxy from the following list + proxies: +# - "https://username:password@example.com:8888" + # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) torrent: # We recommend to only enable magnet URI/torrent import if you trust your users diff --git a/packages/node-utils/src/crypto.ts b/packages/node-utils/src/crypto.ts index 1a583f1a0..388d75574 100644 --- a/packages/node-utils/src/crypto.ts +++ b/packages/node-utils/src/crypto.ts @@ -1,20 +1,9 @@ import { BinaryToTextEncoding, createHash } from 'crypto' -function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { +export function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { return createHash('sha256').update(str).digest(encoding) } -function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { +export function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { return createHash('sha1').update(str).digest(encoding) } - -// high excluded -function randomInt (low: number, high: number) { - return Math.floor(Math.random() * (high - low) + low) -} - -export { - randomInt, - sha256, - sha1 -} diff --git a/packages/server-commands/src/server/server.ts b/packages/server-commands/src/server/server.ts index 8e27897e2..2cea695f6 100644 --- a/packages/server-commands/src/server/server.ts +++ b/packages/server-commands/src/server/server.ts @@ -58,6 +58,7 @@ import { PluginsCommand } from './plugins-command.js' import { RedundancyCommand } from './redundancy-command.js' import { ServersCommand } from './servers-command.js' import { StatsCommand } from './stats-command.js' +import merge from 'lodash-es/merge.js' export type RunServerOptions = { hideLogs?: boolean @@ -240,10 +241,10 @@ export class PeerTubeServer { await this.assignCustomConfigFile() - const configOverride = this.buildConfigOverride() + let configOverride = this.buildConfigOverride() if (configOverrideArg !== undefined) { - Object.assign(configOverride, configOverrideArg) + configOverride = merge(configOverride, configOverrideArg) } // Share the environment @@ -363,9 +364,15 @@ export class PeerTubeServer { } private buildConfigOverride () { - if (!this.parallel) return {} + const base = process.env.YOUTUBE_DL_PROXY + ? { import: { videos: { http: { proxies: [ process.env.YOUTUBE_DL_PROXY ] } } } } + : {} + + if (!this.parallel) return base return { + ...base, + listen: { port: this.port }, diff --git a/server/core/helpers/youtube-dl/youtube-dl-cli.ts b/server/core/helpers/youtube-dl/youtube-dl-cli.ts index 4b17dec8c..c07c7c011 100644 --- a/server/core/helpers/youtube-dl/youtube-dl-cli.ts +++ b/server/core/helpers/youtube-dl/youtube-dl-cli.ts @@ -1,10 +1,11 @@ +import { randomInt } from '@peertube/peertube-core-utils' +import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models' +import { CONFIG } from '@server/initializers/config.js' import { execa, Options as ExecaNodeOptions } from 'execa' import { ensureDir, pathExists } from 'fs-extra/esm' import { writeFile } from 'fs/promises' import { OptionsOfBufferResponseBody } from 'got' import { dirname, join } from 'path' -import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models' -import { CONFIG } from '@server/initializers/config.js' import { logger, loggerTagsFactory } from '../logger.js' import { getProxy, isProxyEnabled } from '../proxy.js' import { isBinaryResponse, peertubeGot } from '../requests.js' @@ -231,10 +232,17 @@ export class YoutubeDLCLI { } private wrapWithProxyOptions (args: string[]) { - if (isProxyEnabled()) { - logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags()) + const config = CONFIG.IMPORT.VIDEOS.HTTP.PROXIES + const configProxyEnabled = Array.isArray(config) && config.length !== 0 - return [ '--proxy', getProxy() ].concat(args) + if (configProxyEnabled || isProxyEnabled()) { + const proxy = configProxyEnabled + ? config[randomInt(0, config.length)] + : getProxy() + + logger.debug('Using proxy %s for YoutubeDL', proxy, lTags()) + + return [ '--proxy', proxy ].concat(args) } return args diff --git a/server/core/initializers/checker-before-init.ts b/server/core/initializers/checker-before-init.ts index 713e114dc..8397b1517 100644 --- a/server/core/initializers/checker-before-init.ts +++ b/server/core/initializers/checker-before-init.ts @@ -43,6 +43,7 @@ function checkMissedConfig () { 'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live', 'thumbnails.generation_from_video.frames_to_analyze', 'thumbnails.sizes', 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', + 'import.videos.http.force_ipv4', 'import.videos.http.proxies', 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', 'import.video_channel_synchronization.full_sync_videos_limit', diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts index e365366d8..79367784d 100644 --- a/server/core/initializers/config.ts +++ b/server/core/initializers/config.ts @@ -554,7 +554,9 @@ const CONFIG = { get PYTHON_PATH () { return config.get('import.videos.http.youtube_dl_release.python_path') } }, - get FORCE_IPV4 () { return config.get('import.videos.http.force_ipv4') } + get FORCE_IPV4 () { return config.get('import.videos.http.force_ipv4') }, + + get PROXIES () { return config.get('import.videos.http.proxies') } }, TORRENT: { get ENABLED () { return config.get('import.videos.torrent.enabled') } diff --git a/support/doc/development/tests.md b/support/doc/development/tests.md index cc1ba09c3..f3acbb01d 100644 --- a/support/doc/development/tests.md +++ b/support/doc/development/tests.md @@ -76,6 +76,7 @@ Some env variables can be defined to disable/enable some tests: * `OBJECT_STORAGE_SCALEWAY_KEY_ID` and `OBJECT_STORAGE_SCALEWAY_ACCESS_KEY`: specify Scaleway API keys to test object storage ACL (not supported by our `chocobozzz/s3-ninja` container) * `ENABLE_FFMPEG_THUMBNAIL_PIXEL_COMPARISON_TESTS=true`: enable pixel comparison on images generated by ffmpeg. Disabled by default because a custom ffmpeg version may fails the tests * `YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN`: Bearer token to download youtube-dl binary + * `YOUTUBE_DL_PROXY`: Custom proxy URL for youtube-dl HTTP video import ### Debug server logs