Add ability to set proxies to youtube-dl

.github/workflows/test.yml

@@ -50,6 +50,7 @@ jobs:
       OBJECT_STORAGE_SCALEWAY_KEY_ID: ${{ secrets.OBJECT_STORAGE_SCALEWAY_KEY_ID }}
       OBJECT_STORAGE_SCALEWAY_ACCESS_KEY: ${{ secrets.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY }}
       YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      YOUTUBE_DL_PROXY: ${{ secrets.YOUTUBE_DL_PROXY }}
 
     steps:
       - uses: actions/checkout@v4

config/default.yaml

@@ -803,6 +803,12 @@ import:
       # IPv6 is very strongly rate-limited on most sites supported by youtube-dl
       force_ipv4: false
 
+      # By default PeerTube uses HTTP_PROXY and HTTPS_PROXY environment variables
+      # But you can specify custom proxies for youtube-dl because remote websites (like YouTube) may block your server IP address
+      # PeerTube will randomly select a proxy from the following list
+      proxies:
+#        - "https://username:password@example.com:8888"
+
     # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
     torrent:
       # We recommend to only enable magnet URI/torrent import if you trust your users

config/production.yaml.example

@@ -813,6 +813,12 @@ import:
       # IPv6 is very strongly rate-limited on most sites supported by youtube-dl
       force_ipv4: false
 
+      # By default PeerTube uses HTTP_PROXY and HTTPS_PROXY environment variables
+      # But you can specify custom proxies for youtube-dl because remote websites (like YouTube) may block your server IP address
+      # PeerTube will randomly select a proxy from the following list
+      proxies:
+#        - "https://username:password@example.com:8888"
+
     # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
     torrent:
       # We recommend to only enable magnet URI/torrent import if you trust your users

packages/node-utils/src/crypto.ts

@@ -1,20 +1,9 @@
 import { BinaryToTextEncoding, createHash } from 'crypto'
 
-function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
+export function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
   return createHash('sha256').update(str).digest(encoding)
 }
 
-function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
+export function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
   return createHash('sha1').update(str).digest(encoding)
 }
-
-// high excluded
-function randomInt (low: number, high: number) {
-  return Math.floor(Math.random() * (high - low) + low)
-}
-
-export {
-  randomInt,
-  sha256,
-  sha1
-}

packages/server-commands/src/server/server.ts

@@ -58,6 +58,7 @@ import { PluginsCommand } from './plugins-command.js'
 import { RedundancyCommand } from './redundancy-command.js'
 import { ServersCommand } from './servers-command.js'
 import { StatsCommand } from './stats-command.js'
+import merge from 'lodash-es/merge.js'
 
 export type RunServerOptions = {
   hideLogs?: boolean
@@ -240,10 +241,10 @@ export class PeerTubeServer {
 
     await this.assignCustomConfigFile()
 
-    const configOverride = this.buildConfigOverride()
+    let configOverride = this.buildConfigOverride()
 
     if (configOverrideArg !== undefined) {
-      Object.assign(configOverride, configOverrideArg)
+      configOverride = merge(configOverride, configOverrideArg)
     }
 
     // Share the environment
@@ -363,9 +364,15 @@ export class PeerTubeServer {
   }
 
   private buildConfigOverride () {
-    if (!this.parallel) return {}
+    const base = process.env.YOUTUBE_DL_PROXY
+      ? { import: { videos: { http: { proxies: [ process.env.YOUTUBE_DL_PROXY ] } } } }
+      : {}
+
+    if (!this.parallel) return base
 
     return {
+      ...base,
+
       listen: {
         port: this.port
       },

server/core/helpers/youtube-dl/youtube-dl-cli.ts

@@ -1,10 +1,11 @@
+import { randomInt } from '@peertube/peertube-core-utils'
+import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
 import { execa, Options as ExecaNodeOptions } from 'execa'
 import { ensureDir, pathExists } from 'fs-extra/esm'
 import { writeFile } from 'fs/promises'
 import { OptionsOfBufferResponseBody } from 'got'
 import { dirname, join } from 'path'
-import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models'
-import { CONFIG } from '@server/initializers/config.js'
 import { logger, loggerTagsFactory } from '../logger.js'
 import { getProxy, isProxyEnabled } from '../proxy.js'
 import { isBinaryResponse, peertubeGot } from '../requests.js'
@@ -231,10 +232,17 @@ export class YoutubeDLCLI {
   }
 
   private wrapWithProxyOptions (args: string[]) {
-    if (isProxyEnabled()) {
+    const config = CONFIG.IMPORT.VIDEOS.HTTP.PROXIES
-      logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags())
+    const configProxyEnabled = Array.isArray(config) && config.length !== 0
 
-      return [ '--proxy', getProxy() ].concat(args)
+    if (configProxyEnabled || isProxyEnabled()) {
+      const proxy = configProxyEnabled
+        ? config[randomInt(0, config.length)]
+        : getProxy()
+
+      logger.debug('Using proxy %s for YoutubeDL', proxy, lTags())
+
+      return [ '--proxy', proxy ].concat(args)
     }
 
     return args

server/core/initializers/checker-before-init.ts

@@ -43,6 +43,7 @@ function checkMissedConfig () {
     'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live',
     'thumbnails.generation_from_video.frames_to_analyze', 'thumbnails.sizes',
     'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
+    'import.videos.http.force_ipv4', 'import.videos.http.proxies',
     'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
     'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
     'import.video_channel_synchronization.full_sync_videos_limit',

server/core/initializers/config.ts

@@ -554,7 +554,9 @@ const CONFIG = {
           get PYTHON_PATH () { return config.get<string>('import.videos.http.youtube_dl_release.python_path') }
         },
 
-        get FORCE_IPV4 () { return config.get<boolean>('import.videos.http.force_ipv4') }
+        get FORCE_IPV4 () { return config.get<boolean>('import.videos.http.force_ipv4') },
+
+        get PROXIES () { return config.get<string[]>('import.videos.http.proxies') }
       },
       TORRENT: {
         get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }

support/doc/development/tests.md

@@ -76,6 +76,7 @@ Some env variables can be defined to disable/enable some tests:
  * `OBJECT_STORAGE_SCALEWAY_KEY_ID` and `OBJECT_STORAGE_SCALEWAY_ACCESS_KEY`: specify Scaleway API keys to test object storage ACL (not supported by our `chocobozzz/s3-ninja` container)
  * `ENABLE_FFMPEG_THUMBNAIL_PIXEL_COMPARISON_TESTS=true`: enable pixel comparison on images generated by ffmpeg. Disabled by default because a custom ffmpeg version may fails the tests
  * `YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN`: Bearer token to download youtube-dl binary
+ * `YOUTUBE_DL_PROXY`: Custom proxy URL for youtube-dl HTTP video import
 
 
 ### Debug server logs