diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index 17c86a381..b33ae27b1 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -81,7 +81,6 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
     // Speed up things and resolve directly the thread
     if (commentFromDatabase.InReplyToVideoComment) {
       const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC')
-      console.log(data)
 
       parentComments = parentComments.concat(data)
     }
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
index 10423a7df..4459152db 100644
--- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
+++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
@@ -4,6 +4,7 @@ import { logger } from '../../../helpers/logger'
 import { getServerActor } from '../../../helpers/utils'
 import { ACTIVITY_PUB } from '../../../initializers'
 import { ActorModel } from '../../../models/activitypub/actor'
+import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { JobHandler, JobScheduler } from '../job-scheduler'
 
 import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler'
@@ -35,6 +36,12 @@ async function maybeRetryRequestLater (err: Error, payload: ActivityPubHttpPaylo
   if (attemptNumber < ACTIVITY_PUB.MAX_HTTP_ATTEMPT) {
     logger.debug('Retrying request to %s (attempt %d/%d).', uri, attemptNumber, ACTIVITY_PUB.MAX_HTTP_ATTEMPT, err)
 
+    const actor = await ActorFollowModel.loadByFollowerInbox(uri, undefined)
+    if (!actor) {
+      logger.debug('Actor %s is not a follower, do not retry the request.', uri)
+      return false
+    }
+
     const newPayload = Object.assign(payload, {
       uris: [ uri ],
       attemptNumber
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts
index deedf8402..54a7504e8 100644
--- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts
+++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-unicast-handler.ts
@@ -6,10 +6,11 @@ import { ActivityPubHttpPayload, buildSignedRequestOptions, computeBody, maybeRe
 async function process (payload: ActivityPubHttpPayload, jobId: number) {
   logger.info('Processing ActivityPub unicast in job %d.', jobId)
 
+  const uri = payload.uris[0]
+
   const body = await computeBody(payload)
   const httpSignatureOptions = await buildSignedRequestOptions(payload)
 
-  const uri = payload.uris[0]
   const options = {
     method: 'POST',
     uri,
diff --git a/server/middlewares/activitypub.ts b/server/middlewares/activitypub.ts
index c7102b6bf..1488b42ab 100644
--- a/server/middlewares/activitypub.ts
+++ b/server/middlewares/activitypub.ts
@@ -35,7 +35,6 @@ async function checkSignature (req: Request, res: Response, next: NextFunction)
 function executeIfActivityPub (fun: RequestHandler | RequestHandler[]) {
   return (req: Request, res: Response, next: NextFunction) => {
     const accepted = req.accepts(ACCEPT_HEADERS)
-    console.log(accepted)
     if (accepted === false || ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS.indexOf(accepted) === -1) {
       return next()
     }
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts
index 920c83d88..de5bb6f74 100644
--- a/server/models/activitypub/actor-follow.ts
+++ b/server/models/activitypub/actor-follow.ts
@@ -163,6 +163,34 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
     return ActorFollowModel.findOne(query)
   }
 
+  static loadByFollowerInbox (url: string, t?: Sequelize.Transaction) {
+    const query = {
+      where: {
+        state: 'accepted'
+      },
+      include: [
+        {
+          model: ActorModel,
+          required: true,
+          as: 'ActorFollower',
+          where: {
+            [Sequelize.Op.or]: [
+              {
+                inboxUrl: url
+              },
+              {
+                sharedInboxUrl: url
+              }
+            ]
+          }
+        }
+      ],
+      transaction: t
+    } as any // FIXME: typings does not work
+
+    return ActorFollowModel.findOne(query)
+  }
+
   static listFollowingForApi (id: number, start: number, count: number, sort: string) {
     const query = {
       distinct: true,
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index 6c74aa61b..17f69f7a7 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -68,6 +68,9 @@ enum ScopeNames {
     {
       fields: [ 'preferredUsername', 'serverId' ],
       unique: true
+    },
+    {
+      fields: [ 'inboxUrl', 'sharedInboxUrl' ]
     }
   ]
 })
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts
index cc1ff9a9f..6ca8cfb64 100644
--- a/server/tests/api/server/handle-down.ts
+++ b/server/tests/api/server/handle-down.ts
@@ -113,6 +113,10 @@ describe('Test handle downs', function () {
       videos.push(resVideo.body.video)
     }
 
+    await wait(2000)
+
+    await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, videoAttributes)
+
     // Add comments to video 2
     {
       const text = 'thread 1'