Add house-keeping script

packages/server-commands/src/users/accounts.ts

@@ -1,15 +1,10 @@
+import { arrayify } from '@peertube/peertube-core-utils'
 import { PeerTubeServer } from '../server/server.js'
 
-async function setDefaultAccountAvatar (serversArg: PeerTubeServer | PeerTubeServer[], token?: string) {
-  const servers = Array.isArray(serversArg)
-    ? serversArg
-    : [ serversArg ]
+export async function setDefaultAccountAvatar (serversArg: PeerTubeServer | PeerTubeServer[], token?: string) {
+  const servers = arrayify(serversArg)
 
   for (const server of servers) {
     await server.users.updateMyAvatar({ fixture: 'avatar.png', token })
   }
 }
-
-export {
-  setDefaultAccountAvatar
-}

packages/server-commands/src/videos/channels.ts

@@ -1,6 +1,7 @@
+import { arrayify } from '@peertube/peertube-core-utils'
 import { PeerTubeServer } from '../server/server.js'
 
-function setDefaultVideoChannel (servers: PeerTubeServer[]) {
+export function setDefaultVideoChannel (servers: PeerTubeServer[]) {
   const tasks: Promise<any>[] = []
 
   for (const server of servers) {
@@ -13,17 +14,10 @@ function setDefaultVideoChannel (servers: PeerTubeServer[]) {
   return Promise.all(tasks)
 }
 
-async function setDefaultChannelAvatar (serversArg: PeerTubeServer | PeerTubeServer[], channelName: string = 'root_channel') {
-  const servers = Array.isArray(serversArg)
-    ? serversArg
-    : [ serversArg ]
+export async function setDefaultChannelAvatar (serversArg: PeerTubeServer | PeerTubeServer[], channelName: string = 'root_channel') {
+  const servers = arrayify(serversArg)
 
   for (const server of servers) {
     await server.channels.updateImage({ channelName, fixture: 'avatar.png', type: 'avatar' })
   }
 }
-
-export {
-  setDefaultVideoChannel,
-  setDefaultChannelAvatar
-}

packages/tests/src/cli/create-generate-storyboard-job.ts

@@ -22,7 +22,7 @@ function listStoryboardFiles (server: PeerTubeServer) {
   return readdir(storage)
 }
 
-describe('Test create generate storyboard job', function () {
+describe('Test create generate storyboard job CLI', function () {
   let servers: PeerTubeServer[] = []
   const uuids: string[] = []
   let sql: SQLCommand

packages/tests/src/cli/create-import-video-file-job.ts

@@ -154,7 +154,7 @@ function runTests (enableObjectStorage: boolean) {
   })
 }
 
-describe('Test create import video jobs', function () {
+describe('Test create import video jobs CLI', function () {
 
   describe('On filesystem', function () {
     runTests(false)

packages/tests/src/cli/create-move-video-storage-job.ts

@@ -64,7 +64,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectSt
   }
 }
 
-describe('Test create move video storage job', function () {
+describe('Test create move video storage job CLI', function () {
   if (areMockObjectStorageTestsDisabled()) return
 
   let servers: PeerTubeServer[] = []

packages/tests/src/cli/house-keeping.ts

@@ -0,0 +1,96 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { HttpStatusCode } from '@peertube/peertube-models'
+import {
+  PeerTubeServer,
+  cleanupTests,
+  createMultipleServers,
+  doubleFollow,
+  makeGetRequest,
+  setAccessTokensToServers,
+  setDefaultAccountAvatar,
+  setDefaultChannelAvatar,
+  waitJobs
+} from '@peertube/peertube-server-commands'
+import { expect } from 'chai'
+
+describe('House keeping CLI', function () {
+  let servers: PeerTubeServer[]
+
+  function runHouseKeeping (option: string) {
+    const env = servers[0].cli.getEnv()
+    const command = `echo y | ${env} npm run house-keeping -- ${option}`
+
+    return servers[0].cli.execWithEnv(command)
+  }
+
+  async function fetchRemoteData () {
+    {
+      const { data } = await servers[0].videos.list()
+      for (const video of data) {
+        await makeGetRequest({ url: servers[0].url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
+        await makeGetRequest({ url: servers[0].url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 })
+      }
+    }
+
+    {
+      const { data: accounts } = await servers[0].accounts.list()
+      const { data: channels } = await servers[0].channels.list()
+
+      for (const { avatars } of [ ...accounts, ...channels ]) {
+        for (const avatar of avatars) {
+          await makeGetRequest({ url: servers[0].url, path: avatar.path, expectedStatus: HttpStatusCode.OK_200 })
+        }
+      }
+    }
+  }
+
+  before(async function () {
+    this.timeout(360000)
+
+    servers = await createMultipleServers(2)
+    await setAccessTokensToServers(servers)
+
+    await setDefaultAccountAvatar(servers)
+    await setDefaultChannelAvatar(servers)
+
+    await servers[1].config.enableMinimumTranscoding()
+
+    for (const server of servers) {
+      await server.videos.quickUpload({ name: 'video' })
+    }
+
+    await waitJobs(servers)
+
+    await doubleFollow(servers[0], servers[1])
+  })
+
+  it('Should have remote files locally', async function () {
+    this.timeout(120000)
+
+    await fetchRemoteData()
+
+    expect(await servers[0].servers.countFiles('thumbnails')).to.equal(2)
+    expect(await servers[0].servers.countFiles('avatars')).to.equal((2 + 2) * 4) // 2 accounts and 2 channels in 4 versions
+  })
+
+  it('Should remove remote files', async function () {
+    this.timeout(60000)
+
+    await servers[0].kill()
+    await runHouseKeeping('--delete-remote-files')
+    await servers[0].run()
+
+    expect(await servers[0].servers.countFiles('thumbnails')).to.equal(1)
+    expect(await servers[0].servers.countFiles('avatars')).to.equal((1 + 1) * 4) // 1 account and 1 channel in 4 versions
+
+    await fetchRemoteData()
+
+    expect(await servers[0].servers.countFiles('thumbnails')).to.equal(2)
+    expect(await servers[0].servers.countFiles('avatars')).to.equal((2 + 2) * 4) // 2 accounts and 2 channels in 4 versions
+  })
+
+  after(async function () {
+    await cleanupTests(servers)
+  })
+})

packages/tests/src/cli/index.ts

@@ -2,6 +2,7 @@
 import './create-import-video-file-job'
 import './create-generate-storyboard-job'
 import './create-move-video-storage-job'
+import './house-keeping.js'
 import './peertube'
 import './plugins'
 import './prune-storage'

packages/tests/src/cli/plugins.ts

@@ -10,7 +10,7 @@ import {
   setAccessTokensToServers
 } from '@peertube/peertube-server-commands'
 
-describe('Test plugin scripts', function () {
+describe('Test plugin CLI', function () {
   let server: PeerTubeServer
 
   before(async function () {

packages/tests/src/cli/prune-storage.ts

@@ -23,7 +23,7 @@ import { createFile } from 'fs-extra/esm'
 import { readdir } from 'fs/promises'
 import { join } from 'path'
 
-describe('Test prune storage scripts', function () {
+describe('Test prune storage CLI', function () {
   let servers: PeerTubeServer[]
 
   before(async function () {

packages/tests/src/cli/regenerate-thumbnails.ts

@@ -26,7 +26,7 @@ async function testThumbnail (server: PeerTubeServer, videoId: number | string)
   }
 }
 
-describe('Test regenerate thumbnails script', function () {
+describe('Test regenerate thumbnails CLI', function () {
   let servers: PeerTubeServer[]
 
   let video1: Video

packages/tests/src/cli/reset-password.ts

@@ -1,6 +1,6 @@
 import { cleanupTests, CLICommand, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands'
 
-describe('Test reset password scripts', function () {
+describe('Test reset password CLI', function () {
   let server: PeerTubeServer
 
   before(async function () {

packages/tests/src/cli/update-host.ts

@@ -15,7 +15,7 @@ import {
 import { parseTorrentVideo } from '@tests/shared/webtorrent.js'
 import { VideoPlaylistPrivacy } from '@peertube/peertube-models'
 
-describe('Test update host scripts', function () {
+describe('Test update host CLI', function () {
   let server: PeerTubeServer
 
   before(async function () {

packages/tests/src/cli/update-object-storage-url.ts

@@ -14,7 +14,7 @@ import {
 import { expectStartWith } from '@tests/shared/checks.js'
 import { expect } from 'chai'
 
-describe('Update object storage URL', function () {
+describe('Update object storage URL CLI', function () {
   if (areMockObjectStorageTestsDisabled()) return
 
   let server: PeerTubeServer