diff --git a/client/src/app/+admin/follows/followers-list/followers-list.component.html b/client/src/app/+admin/follows/followers-list/followers-list.component.html index 9499a0433..d5b1b789d 100644 --- a/client/src/app/+admin/follows/followers-list/followers-list.component.html +++ b/client/src/app/+admin/follows/followers-list/followers-list.component.html @@ -3,8 +3,8 @@ sortField="createdAt" (onLazyLoad)="loadLazy($event)" > + - diff --git a/client/src/app/shared/misc/utils.ts b/client/src/app/shared/misc/utils.ts index 2739ff81a..23b46812b 100644 --- a/client/src/app/shared/misc/utils.ts +++ b/client/src/app/shared/misc/utils.ts @@ -31,7 +31,7 @@ function populateAsyncUserVideoChannels (authService: AuthService, channel: any[ const videoChannels = user.videoChannels if (Array.isArray(videoChannels) === false) return - videoChannels.forEach(c => channel.push({ id: c.id, label: c.name })) + videoChannels.forEach(c => channel.push({ id: c.id, label: c.displayName })) return res() } diff --git a/client/src/app/videos/+video-edit/video-add.component.html b/client/src/app/videos/+video-edit/video-add.component.html index 2040ff9d4..34291c6c6 100644 --- a/client/src/app/videos/+video-edit/video-add.component.html +++ b/client/src/app/videos/+video-edit/video-add.component.html @@ -44,7 +44,6 @@ [validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels" > -
Publish will be available when upload is finished
diff --git a/client/src/app/videos/+video-watch/video-watch.component.html b/client/src/app/videos/+video-watch/video-watch.component.html index a5c387638..5921b4b72 100644 --- a/client/src/app/videos/+video-watch/video-watch.component.html +++ b/client/src/app/videos/+video-watch/video-watch.component.html @@ -85,7 +85,7 @@
- {{ video.channel.name }} + {{ video.channel.displayName }}
diff --git a/server.ts b/server.ts index a52c47083..99077a173 100644 --- a/server.ts +++ b/server.ts @@ -56,6 +56,7 @@ import { installApplication } from './server/initializers' import { activitypubHttpJobScheduler, transcodingJobScheduler } from './server/lib/jobs' import { VideosPreviewCache } from './server/lib/cache' import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers' +import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follow-scheduler' // ----------- Command line ----------- @@ -168,6 +169,8 @@ function onDatabaseInitDone () { // ----------- Make the server listening ----------- server.listen(port, () => { VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE) + BadActorFollowScheduler.Instance.enable() + activitypubHttpJobScheduler.activate() transcodingJobScheduler.activate() diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index c735e6daf..0c139912c 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -9,7 +9,7 @@ import { isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 165 +const LAST_MIGRATION_VERSION = 170 // --------------------------------------------------------------------------- @@ -40,12 +40,12 @@ const OAUTH_LIFETIME = { // --------------------------------------------------------------------------- -// Number of points we add/remove from a friend after a successful/bad request -const SERVERS_SCORE = { +// Number of points we add/remove after a successful/bad request +const ACTOR_FOLLOW_SCORE = { PENALTY: -10, BONUS: 10, - BASE: 100, - MAX: 1000 + BASE: 1000, + MAX: 10000 } const FOLLOW_STATES: { [ id: string ]: FollowState } = { @@ -76,6 +76,9 @@ const JOBS_FETCH_LIMIT_PER_CYCLE = { // 1 minutes let JOBS_FETCHING_INTERVAL = 60000 +// 1 hour +let SCHEDULER_INTERVAL = 60000 * 60 + // --------------------------------------------------------------------------- const CONFIG = { @@ -346,7 +349,7 @@ const OPENGRAPH_AND_OEMBED_COMMENT = '' // Special constants for a test instance if (isTestInstance() === true) { - SERVERS_SCORE.BASE = 20 + ACTOR_FOLLOW_SCORE.BASE = 20 JOBS_FETCHING_INTERVAL = 1000 REMOTE_SCHEME.HTTP = 'http' REMOTE_SCHEME.WS = 'ws' @@ -354,6 +357,7 @@ if (isTestInstance() === true) { ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 60 // 1 minute CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB + SCHEDULER_INTERVAL = 10000 } CONFIG.WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT) @@ -378,7 +382,7 @@ export { OAUTH_LIFETIME, OPENGRAPH_AND_OEMBED_COMMENT, PAGINATION_COUNT_DEFAULT, - SERVERS_SCORE, + ACTOR_FOLLOW_SCORE, PREVIEWS_SIZE, REMOTE_SCHEME, FOLLOW_STATES, @@ -396,5 +400,6 @@ export { VIDEO_LICENCES, VIDEO_RATE_TYPES, VIDEO_MIMETYPE_EXT, - AVATAR_MIMETYPE_EXT + AVATAR_MIMETYPE_EXT, + SCHEDULER_INTERVAL } diff --git a/server/initializers/migrations/0170-actor-follow-score.ts b/server/initializers/migrations/0170-actor-follow-score.ts new file mode 100644 index 000000000..2deabaf98 --- /dev/null +++ b/server/initializers/migrations/0170-actor-follow-score.ts @@ -0,0 +1,28 @@ +import * as Sequelize from 'sequelize' +import { ACTOR_FOLLOW_SCORE } from '../index' + +async function up (utils: { + transaction: Sequelize.Transaction, + queryInterface: Sequelize.QueryInterface, + sequelize: Sequelize.Sequelize +}): Promise { + await utils.queryInterface.removeColumn('server', 'score') + + const data = { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: ACTOR_FOLLOW_SCORE.BASE + } + + await utils.queryInterface.addColumn('actorFollow', 'score', data) + +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts index c20a48a4e..3f780e319 100644 --- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts +++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-broadcast-handler.ts @@ -1,5 +1,6 @@ import { logger } from '../../../helpers/logger' import { doRequest } from '../../../helpers/requests' +import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler' async function process (payload: ActivityPubHttpPayload, jobId: number) { @@ -15,15 +16,22 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) { httpSignature: httpSignatureOptions } + const badUrls: string[] = [] + const goodUrls: string[] = [] + for (const uri of payload.uris) { options.uri = uri try { await doRequest(options) + goodUrls.push(uri) } catch (err) { - await maybeRetryRequestLater(err, payload, uri) + const isRetryingLater = await maybeRetryRequestLater(err, payload, uri) + if (isRetryingLater === false) badUrls.push(uri) } } + + return ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes(goodUrls, badUrls, undefined) } function onError (err: Error, jobId: number) { diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts index d576cd42e..884ede5a3 100644 --- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts +++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts @@ -4,6 +4,7 @@ import { logger } from '../../../helpers/logger' import { getServerActor } from '../../../helpers/utils' import { ACTIVITY_PUB } from '../../../initializers' import { ActorModel } from '../../../models/activitypub/actor' +import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { JobHandler, JobScheduler } from '../job-scheduler' import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler' @@ -26,7 +27,7 @@ const jobCategory: JobCategory = 'activitypub-http' const activitypubHttpJobScheduler = new JobScheduler(jobCategory, jobHandlers) -function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) { +async function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, uri: string) { logger.warn('Cannot make request to %s.', uri, err) let attemptNumber = payload.attemptNumber || 1 @@ -39,8 +40,12 @@ function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPayload, ur uris: [ uri ], attemptNumber }) - return activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload) + await activitypubHttpJobScheduler.createJob(undefined, 'activitypubHttpUnicastHandler', newPayload) + + return true } + + return false } async function computeBody (payload: ActivityPubHttpPayload) { diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts index 175ec6642..e02bd698e 100644 --- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts +++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts @@ -1,5 +1,6 @@ import { logger } from '../../../helpers/logger' import { doRequest } from '../../../helpers/requests' +import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRetryRequestLater } from './activitypub-http-job-scheduler' async function process (payload: ActivityPubHttpPayload, jobId: number) { @@ -18,8 +19,13 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) { try { await doRequest(options) + await ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([ uri ], [], undefined) } catch (err) { - await maybeRetryRequestLater(err, payload, uri) + const isRetryingLater = await maybeRetryRequestLater(err, payload, uri) + if (isRetryingLater === false) { + await ActorFollowModel.updateActorFollowsScoreAndRemoveBadOnes([], [ uri ], undefined) + } + throw err } } diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts new file mode 100644 index 000000000..473544ddf --- /dev/null +++ b/server/lib/schedulers/abstract-scheduler.ts @@ -0,0 +1,16 @@ +import { SCHEDULER_INTERVAL } from '../../initializers' + +export abstract class AbstractScheduler { + + private interval: NodeJS.Timer + + enable () { + this.interval = setInterval(() => this.execute(), SCHEDULER_INTERVAL) + } + + disable () { + clearInterval(this.interval) + } + + protected abstract execute () +} diff --git a/server/lib/schedulers/bad-actor-follow-scheduler.ts b/server/lib/schedulers/bad-actor-follow-scheduler.ts new file mode 100644 index 000000000..c6c285ece --- /dev/null +++ b/server/lib/schedulers/bad-actor-follow-scheduler.ts @@ -0,0 +1,24 @@ +import { logger } from '../../helpers/logger' +import { ActorFollowModel } from '../../models/activitypub/actor-follow' +import { AbstractScheduler } from './abstract-scheduler' + +export class BadActorFollowScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + private constructor () { + super() + } + + async execute () { + try { + await ActorFollowModel.removeBadActorFollows() + } catch (err) { + logger.error('Error in bad actor follows scheduler.', err) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/models/account/account.ts b/server/models/account/account.ts index 47336d1e0..f81c50180 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -179,7 +179,6 @@ export class AccountModel extends Model { const actor = this.Actor.toFormattedJSON() const account = { id: this.id, - name: this.Actor.preferredUsername, displayName: this.name, createdAt: this.createdAt, updatedAt: this.updatedAt diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index 5fcc3449d..78a65a0ff 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts @@ -1,8 +1,14 @@ import * as Bluebird from 'bluebird' import { values } from 'lodash' import * as Sequelize from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { + AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, IsInt, Max, Model, Table, + UpdatedAt +} from 'sequelize-typescript' import { FollowState } from '../../../shared/models/actors' +import { AccountFollow } from '../../../shared/models/actors/follow.model' +import { logger } from '../../helpers/logger' +import { ACTOR_FOLLOW_SCORE } from '../../initializers' import { FOLLOW_STATES } from '../../initializers/constants' import { ServerModel } from '../server/server' import { getSort } from '../utils' @@ -20,6 +26,9 @@ import { ActorModel } from './actor' { fields: [ 'actorId', 'targetActorId' ], unique: true + }, + { + fields: [ 'score' ] } ] }) @@ -29,6 +38,13 @@ export class ActorFollowModel extends Model { @Column(DataType.ENUM(values(FOLLOW_STATES))) state: FollowState + @AllowNull(false) + @Default(ACTOR_FOLLOW_SCORE.BASE) + @IsInt + @Max(ACTOR_FOLLOW_SCORE.MAX) + @Column + score: number + @CreatedAt createdAt: Date @@ -63,6 +79,34 @@ export class ActorFollowModel extends Model { }) ActorFollowing: ActorModel + // Remove actor follows with a score of 0 (too many requests where they were unreachable) + static async removeBadActorFollows () { + const actorFollows = await ActorFollowModel.listBadActorFollows() + + const actorFollowsRemovePromises = actorFollows.map(actorFollow => actorFollow.destroy()) + await Promise.all(actorFollowsRemovePromises) + + const numberOfActorFollowsRemoved = actorFollows.length + + if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved) + } + + static updateActorFollowsScoreAndRemoveBadOnes (goodInboxes: string[], badInboxes: string[], t: Sequelize.Transaction) { + if (goodInboxes.length === 0 && badInboxes.length === 0) return + + logger.info('Updating %d good actor follows and %d bad actor follows scores.', goodInboxes.length, badInboxes.length) + + if (goodInboxes.length !== 0) { + ActorFollowModel.incrementScores(goodInboxes, ACTOR_FOLLOW_SCORE.BONUS, t) + .catch(err => logger.error('Cannot increment scores of good actor follows.', err)) + } + + if (badInboxes.length !== 0) { + ActorFollowModel.incrementScores(badInboxes, ACTOR_FOLLOW_SCORE.PENALTY, t) + .catch(err => logger.error('Cannot decrement scores of bad actor follows.', err)) + } + } + static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Sequelize.Transaction) { const query = { where: { @@ -260,7 +304,37 @@ export class ActorFollowModel extends Model { } } - toFormattedJSON () { + private static incrementScores (inboxUrls: string[], value: number, t: Sequelize.Transaction) { + const inboxUrlsString = inboxUrls.map(url => `'${url}'`).join(',') + + const query = 'UPDATE "actorFollow" SET "score" = "score" +' + value + ' ' + + 'WHERE id IN (' + + 'SELECT "actorFollow"."id" FROM "actorFollow" ' + + 'INNER JOIN "actor" ON "actor"."id" = "actorFollow"."actorId" ' + + 'WHERE "actor"."inboxUrl" IN (' + inboxUrlsString + ') OR "actor"."sharedInboxUrl" IN (' + inboxUrlsString + ')' + + ')' + + const options = { + type: Sequelize.QueryTypes.BULKUPDATE, + transaction: t + } + + return ActorFollowModel.sequelize.query(query, options) + } + + private static listBadActorFollows () { + const query = { + where: { + score: { + [Sequelize.Op.lte]: 0 + } + } + } + + return ActorFollowModel.findAll(query) + } + + toFormattedJSON (): AccountFollow { const follower = this.ActorFollower.toFormattedJSON() const following = this.ActorFollowing.toFormattedJSON() @@ -268,6 +342,7 @@ export class ActorFollowModel extends Model { id: this.id, follower, following, + score: this.score, state: this.state, createdAt: this.createdAt, updatedAt: this.updatedAt diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index b88e06b41..912d8d748 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -204,7 +204,7 @@ export class ActorModel extends Model { VideoChannel: VideoChannelModel static load (id: number) { - return ActorModel.scope(ScopeNames.FULL).findById(id) + return ActorModel.unscoped().findById(id) } static listByFollowersUrls (followersUrls: string[], transaction?: Sequelize.Transaction) { @@ -267,20 +267,17 @@ export class ActorModel extends Model { avatar = this.Avatar.toFormattedJSON() } - let score: number - if (this.Server) { - score = this.Server.score - } - return { id: this.id, url: this.url, uuid: this.uuid, + name: this.preferredUsername, host: this.getHost(), - score, followingCount: this.followingCount, followersCount: this.followersCount, - avatar + avatar, + createdAt: this.createdAt, + updatedAt: this.updatedAt } } diff --git a/server/models/server/server.ts b/server/models/server/server.ts index d35aa0ca4..c43146156 100644 --- a/server/models/server/server.ts +++ b/server/models/server/server.ts @@ -1,8 +1,5 @@ -import * as Sequelize from 'sequelize' -import { AllowNull, Column, CreatedAt, Default, Is, IsInt, Max, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' import { isHostValid } from '../../helpers/custom-validators/servers' -import { logger } from '../../helpers/logger' -import { SERVERS_SCORE } from '../../initializers' import { throwIfNotValid } from '../utils' @Table({ @@ -11,9 +8,6 @@ import { throwIfNotValid } from '../utils' { fields: [ 'host' ], unique: true - }, - { - fields: [ 'score' ] } ] }) @@ -24,86 +18,9 @@ export class ServerModel extends Model { @Column host: string - @AllowNull(false) - @Default(SERVERS_SCORE.BASE) - @IsInt - @Max(SERVERS_SCORE.MAX) - @Column - score: number - @CreatedAt createdAt: Date @UpdatedAt updatedAt: Date - - static updateServersScoreAndRemoveBadOnes (goodServers: number[], badServers: number[]) { - logger.info('Updating %d good servers and %d bad servers scores.', goodServers.length, badServers.length) - - if (goodServers.length !== 0) { - ServerModel.incrementScores(goodServers, SERVERS_SCORE.BONUS) - .catch(err => { - logger.error('Cannot increment scores of good servers.', err) - }) - } - - if (badServers.length !== 0) { - ServerModel.incrementScores(badServers, SERVERS_SCORE.PENALTY) - .then(() => ServerModel.removeBadServers()) - .catch(err => { - if (err) logger.error('Cannot decrement scores of bad servers.', err) - }) - - } - } - - // Remove servers with a score of 0 (too many requests where they were unreachable) - private static async removeBadServers () { - try { - const servers = await ServerModel.listBadServers() - - const serversRemovePromises = servers.map(server => server.destroy()) - await Promise.all(serversRemovePromises) - - const numberOfServersRemoved = servers.length - - if (numberOfServersRemoved) { - logger.info('Removed %d servers.', numberOfServersRemoved) - } else { - logger.info('No need to remove bad servers.') - } - } catch (err) { - logger.error('Cannot remove bad servers.', err) - } - } - - private static incrementScores (ids: number[], value: number) { - const update = { - score: Sequelize.literal('score +' + value) - } - - const options = { - where: { - id: { - [Sequelize.Op.in]: ids - } - }, - // In this case score is a literal and not an integer so we do not validate it - validate: false - } - - return ServerModel.update(update, options) - } - - private static listBadServers () { - const query = { - where: { - score: { - [Sequelize.Op.lte]: 0 - } - } - } - - return ServerModel.findAll(query) - } } diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index acc2486b3..e2cbf0422 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -228,7 +228,7 @@ export class VideoChannelModel extends Model { const actor = this.Actor.toFormattedJSON() const account = { id: this.id, - name: this.name, + displayName: this.name, description: this.description, isLocal: this.Actor.isOwned(), createdAt: this.createdAt, diff --git a/shared/models/actors/account.model.ts b/shared/models/actors/account.model.ts index e4dbc81e5..5cc12c18f 100644 --- a/shared/models/actors/account.model.ts +++ b/shared/models/actors/account.model.ts @@ -1,15 +1,5 @@ -import { Avatar } from '../avatars/avatar.model' +import { Actor } from './actor.model' -export interface Account { - id: number - uuid: string - url: string - name: string +export interface Account extends Actor { displayName: string - host: string - followingCount: number - followersCount: number - createdAt: Date - updatedAt: Date - avatar: Avatar } diff --git a/shared/models/actors/actor.model.ts b/shared/models/actors/actor.model.ts new file mode 100644 index 000000000..f91616519 --- /dev/null +++ b/shared/models/actors/actor.model.ts @@ -0,0 +1,14 @@ +import { Avatar } from '../avatars/avatar.model' + +export interface Actor { + id: number + uuid: string + url: string + name: string + host: string + followingCount: number + followersCount: number + createdAt: Date + updatedAt: Date + avatar: Avatar +} diff --git a/shared/models/actors/follow.model.ts b/shared/models/actors/follow.model.ts index cdc3da560..70562bfc7 100644 --- a/shared/models/actors/follow.model.ts +++ b/shared/models/actors/follow.model.ts @@ -1,11 +1,12 @@ -import { Account } from './account.model' +import { Actor } from './actor.model' export type FollowState = 'pending' | 'accepted' export interface AccountFollow { id: number - follower: Account - following: Account + follower: Actor + following: Actor + score: number state: FollowState createdAt: Date updatedAt: Date diff --git a/shared/models/videos/video-channel.model.ts b/shared/models/videos/video-channel.model.ts index d1a952826..b164fb555 100644 --- a/shared/models/videos/video-channel.model.ts +++ b/shared/models/videos/video-channel.model.ts @@ -1,13 +1,10 @@ +import { Actor } from '../actors/actor.model' import { Video } from './video.model' -export interface VideoChannel { - id: number - name: string - url: string +export interface VideoChannel extends Actor { + displayName: string description: string isLocal: boolean - createdAt: Date | string - updatedAt: Date | string owner?: { name: string uuid: string