Feat: video download stats (#7437)

* register job

* track downloads & display total count on stats page

* display download timeseries graph

* automated tests

* diff cleanup

* add downloads to platform stats

* remove extra semicolons

* fix lint issues

* set DB migration target

* move download stats to VideoViewModel

* rename videoView to videoStats

* lint fixes

* fix failing tests

* track remote downloads

* lint fixes

* Rename view directory to stat

* Simplify metric typing

* Do not display downloads in table

* Simplify downloads process

* Create stats tests directory

* Simplify download stats

* Fix stats fallback

* Fix redundancy

* Prevent useless log

---------

Co-authored-by: Chocobozzz <me@florianbigard.com>

packages/models/src/activitypub/activity.ts

@@ -44,6 +44,7 @@ export type Activity =
   | ActivityFlag
   | ActivityApproveReply
   | ActivityRejectReply
+  | ActivityDownload
 
 export type ActivityType =
   | 'Create'
@@ -60,6 +61,7 @@ export type ActivityType =
   | 'Flag'
   | 'ApproveReply'
   | 'RejectReply'
+  | 'Download'
 
 export interface ActivityAudience {
   to: string[]
@@ -162,3 +164,9 @@ export interface ActivityFlag extends BaseActivity {
   startAt?: number
   endAt?: number
 }
+
+export interface ActivityDownload extends BaseActivity {
+  type: 'Download'
+  actor: string
+  object: APObjectId
+}

packages/models/src/activitypub/context.ts

@@ -18,3 +18,4 @@ export type ContextType =
   | 'ApproveReply'
   | 'RejectReply'
   | 'PlayerSettings'
+  | 'Download'

packages/models/src/activitypub/objects/video-object.ts

@@ -22,6 +22,7 @@ export interface VideoObject {
   subtitleLanguage: VideoCaptionObject[]
 
   views: number
+  downloads: number
 
   sensitive: boolean
   summary: string

packages/models/src/server/debug.model.ts

@@ -6,12 +6,12 @@ export interface Debug {
 export type SendDebugCommand = {
   command:
     | 'remove-dandling-resumable-uploads'
-    | 'process-video-views-buffer'
+    | 'process-video-stats-buffer'
     | 'process-video-viewers'
     | 'process-video-channel-sync-latest'
     | 'process-update-videos-scheduler'
     | 'remove-expired-user-exports'
-    | 'process-remove-old-views'
+    | 'process-remove-old-stats'
 } | SendDebugTestEmails
 
 export type SendDebugTestEmails = {

packages/models/src/server/job.model.ts

@@ -29,7 +29,7 @@ export type JobType =
   | 'video-redundancy'
   | 'video-studio-edition'
   | 'video-transcoding'
-  | 'videos-views-stats'
+  | 'videos-stats'
   | 'generate-video-storyboard'
   | 'create-user-export'
   | 'import-user-archive'

packages/models/src/server/server-stats.model.ts

@@ -15,6 +15,7 @@ export interface ServerStats extends ActivityPubMessagesSuccess, ActivityPubMess
 
   totalLocalVideos: number
   totalLocalVideoViews: number
+  totalLocalVideoDownloads: number
   totalLocalVideoComments: number
   totalLocalVideoFilesSize: number
 

packages/models/src/videos/stats/video-stats-timeserie-metric.type.ts

@@ -1 +1 @@
-export type VideoStatsTimeserieMetric = 'viewers' | 'aggregateWatchTime'
+export type VideoStatsTimeserieMetric = 'viewers' | 'aggregateWatchTime' | 'downloads'

packages/models/src/videos/video.model.ts

@@ -69,6 +69,8 @@ export interface Video extends Partial<VideoAdditionalAttributes> {
   views: number
   viewers: number
 
+  downloads: number
+
   likes: number
   dislikes: number
   comments: number

packages/server-commands/src/server/jobs.ts

@@ -22,7 +22,7 @@ async function waitJobs (
   const states: JobState[] = [ 'waiting', 'active' ]
   if (!skipDelayed) states.push('delayed')
 
-  const repeatableJobs: JobType[] = [ 'videos-views-stats', 'activitypub-cleaner' ]
+  const repeatableJobs: JobType[] = [ 'videos-stats', 'activitypub-cleaner' ]
   let pendingRequests: boolean
 
   function tasksBuilder () {

packages/tests/src/api/server/jobs.ts

@@ -54,7 +54,7 @@ describe('Test jobs', function () {
 
       let job = body.data[0]
       // Skip repeat jobs
-      if (job.type === 'videos-views-stats') job = body.data[1]
+      if (job.type === 'videos-stats') job = body.data[1]
 
       expect(job.state).to.equal('completed')
       expect(dateIsValid(job.createdAt as string)).to.be.true

packages/tests/src/api/stats/index.ts

@@ -1,3 +1,4 @@
+export * from './video-downloads-counter.js'
 export * from './video-views-counter.js'
 export * from './video-views-overall-stats.js'
 export * from './video-views-retention-stats.js'

packages/tests/src/api/stats/video-downloads-counter.ts

@@ -0,0 +1,121 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { wait } from '@peertube/peertube-core-utils'
+import { HttpStatusCode } from '@peertube/peertube-models'
+import {
+  createMultipleServers,
+  doubleFollow,
+  makeRawRequest,
+  PeerTubeServer,
+  setAccessTokensToServers,
+  waitJobs
+} from '@peertube/peertube-server-commands'
+import { expect } from 'chai'
+
+describe('Test video downloads stats', function () {
+  let servers: PeerTubeServer[]
+  let videoId: string
+  let remoteVideoId: string
+
+  async function processStats () {
+    for (const server of servers) {
+      await server.debug.sendCommand({ body: { command: 'process-video-stats-buffer' } })
+    }
+
+    await waitJobs(servers)
+  }
+
+  before(async function () {
+    this.timeout(120000)
+
+    servers = await createMultipleServers(2)
+    await setAccessTokensToServers(servers)
+
+    await doubleFollow(servers[0], servers[1])
+
+    await servers[0].config.enableMinimumTranscoding()
+
+    {
+      const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
+      videoId = uuid
+    }
+
+    {
+      const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
+      remoteVideoId = uuid
+    }
+
+    await waitJobs(servers)
+  })
+
+  it('Should count web video downloads', async function () {
+    const video = await servers[0].videos.get({ id: remoteVideoId })
+    await makeRawRequest({ url: video.files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
+
+    await processStats()
+
+    for (const server of servers) {
+      const video = await server.videos.get({ id: remoteVideoId })
+      expect(video.downloads).to.equal(1)
+    }
+  })
+
+  it('Should count hls downloads', async function () {
+    const video = await servers[0].videos.get({ id: videoId })
+    await makeRawRequest({ url: video.streamingPlaylists[0].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
+
+    await processStats()
+
+    for (const server of servers) {
+      const video = await server.videos.get({ id: videoId })
+      expect(video.downloads).to.equal(1)
+    }
+  })
+
+  it('Should count generated download', async function () {
+    const video = await servers[0].videos.get({ id: videoId })
+    const videoFileIds = [ video.files[0].id ]
+
+    await servers[0].videos.generateDownload({ videoId, videoFileIds })
+
+    await processStats()
+
+    for (const server of servers) {
+      const video = await server.videos.get({ id: videoId })
+      expect(video.downloads).to.equal(2)
+    }
+  })
+
+  it('Should count remote download', async function () {
+    const video = await servers[1].videos.get({ id: videoId })
+    const videoFileIds = [ video.files[0].id ]
+
+    await servers[1].videos.generateDownload({ videoId, videoFileIds })
+    await waitJobs(servers)
+
+    await servers[0].debug.sendCommand({ body: { command: 'process-video-stats-buffer' } })
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      const video = await server.videos.get({ id: videoId })
+      expect(video.downloads).to.equal(3)
+    }
+  })
+
+  it('Should return time-series for downloads stats', async function () {
+    await wait(6000)
+
+    const now = new Date()
+    const twoHoursAgo = new Date()
+    twoHoursAgo.setHours(twoHoursAgo.getHours() - 4)
+
+    const { data } = await servers[0].videoStats.getTimeserieStats({
+      videoId,
+      metric: 'downloads',
+      startDate: twoHoursAgo,
+      endDate: now
+    })
+
+    expect(data.reduce((sum, point) => sum + point.value, 0)).to.equal(3)
+  })
+})

packages/tests/src/api/stats/videos-views-cleaner.ts

@@ -53,7 +53,7 @@ describe('Test video views cleaner', function () {
 
       await killallServers([ servers[0] ])
       await servers[0].run({ views: { videos: { remote: { max_age: '10 days' } } } })
-      await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-views' } })
+      await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-stats' } })
 
       for (let i = 0; i < servers.length; i++) {
         const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1)
@@ -75,7 +75,7 @@ describe('Test video views cleaner', function () {
 
       await killallServers([ servers[0] ])
       await servers[0].run({ views: { videos: { remote: { max_age: '5 seconds' } } } })
-      await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-views' } })
+      await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-stats' } })
 
       for (let i = 0; i < servers.length; i++) {
         const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1)
@@ -100,7 +100,7 @@ describe('Test video views cleaner', function () {
 
       await killallServers([ servers[0] ])
       await servers[0].run({ views: { videos: { local: { max_age: '5 hours' } } } })
-      await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-views' } })
+      await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-stats' } })
 
       for (let i = 0; i < servers.length; i++) {
         const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1)
@@ -123,7 +123,7 @@ describe('Test video views cleaner', function () {
 
       await killallServers([ servers[0] ])
       await servers[0].run({ views: { videos: { local: { max_age: '5 seconds' } } } })
-      await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-views' } })
+      await servers[0].debug.sendCommand({ body: { command: 'process-remove-old-stats' } })
 
       {
         const totalServer1 = await sqlCommands[0].countVideoViewsOf(videoIdServer1)

packages/tests/src/api/videos/multiple-servers.ts

@@ -514,7 +514,7 @@ describe('Test multiple servers', function () {
       await waitJobs(servers)
 
       for (const server of servers) {
-        await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
+        await server.debug.sendCommand({ body: { command: 'process-video-stats-buffer' } })
       }
 
       await waitJobs(servers)
@@ -550,7 +550,7 @@ describe('Test multiple servers', function () {
       await waitJobs(servers)
 
       for (const server of servers) {
-        await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
+        await server.debug.sendCommand({ body: { command: 'process-video-stats-buffer' } })
       }
 
       await waitJobs(servers)

packages/tests/src/api/videos/single-server.ts

@@ -189,7 +189,7 @@ describe('Test a single server', function () {
       await server.views.simulateView({ id: videoId })
       await server.views.simulateView({ id: videoId })
 
-      await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
+      await server.debug.sendCommand({ body: { command: 'process-video-stats-buffer' } })
 
       const video = await server.videos.get({ id: videoId })
       expect(video.views).to.equal(3)

packages/tests/src/client/jsonld.ts

@@ -66,7 +66,7 @@ describe('Test JSONLD HTML tags', function () {
       await servers[0].videos.rate({ id: publicVideo.id, rating: 'like' })
       await servers[0].views.simulateView({ id: publicVideo.id, xForwardedFor: '0.0.0.1,127.0.0.1' })
       await servers[0].views.simulateView({ id: publicVideo.id, xForwardedFor: '0.0.0.2,127.0.0.1' })
-      await servers[0].debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
+      await servers[0].debug.sendCommand({ body: { command: 'process-video-stats-buffer' } })
     }
 
     {

packages/tests/src/shared/sql-command.ts

@@ -42,8 +42,8 @@ export class SQLCommand {
   }
 
   async countVideoViewsOf (uuid: string) {
-    const query = 'SELECT SUM("videoView"."views") AS "total" FROM "videoView" ' +
-      `INNER JOIN "video" ON "video"."id" = "videoView"."videoId" WHERE "video"."uuid" = :uuid`
+    const query = 'SELECT SUM("videoStat"."views") AS "total" FROM "videoStat" ' +
+      `INNER JOIN "video" ON "video"."id" = "videoStat"."videoId" WHERE "video"."uuid" = :uuid`
 
     const [ { total } ] = await this.selectQuery<{ total: number }>(query, { uuid })
     if (!total) return 0

packages/tests/src/shared/views.ts

@@ -16,7 +16,7 @@ async function processViewersStats (servers: PeerTubeServer[]) {
   await wait(6000)
 
   for (const server of servers) {
-    await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
+    await server.debug.sendCommand({ body: { command: 'process-video-stats-buffer' } })
     await server.debug.sendCommand({ body: { command: 'process-video-viewers' } })
   }
 
@@ -25,7 +25,7 @@ async function processViewersStats (servers: PeerTubeServer[]) {
 
 async function processViewsBuffer (servers: PeerTubeServer[]) {
   for (const server of servers) {
-    await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
+    await server.debug.sendCommand({ body: { command: 'process-video-stats-buffer' } })
   }
 
   await waitJobs(servers)