diff --git a/src/rpc/virnetserver.c b/src/rpc/virnetserver.c
index 204f425264..28afe54e49 100644
--- a/src/rpc/virnetserver.c
+++ b/src/rpc/virnetserver.c
@@ -217,9 +217,11 @@ static int virNetServerDispatchNewMessage(virNetServerClientPtr client,
             priority = virNetServerProgramGetPriority(prog, msg->header.proc);
         }
 
+        virObjectRef(client);
         ret = virThreadPoolSendJob(srv->workers, priority, job);
 
         if (ret < 0) {
+            virObjectUnref(client);
             VIR_FREE(job);
             virObjectUnref(prog);
         }
diff --git a/src/rpc/virnetserverclient.c b/src/rpc/virnetserverclient.c
index 388514946b..00459d17ba 100644
--- a/src/rpc/virnetserverclient.c
+++ b/src/rpc/virnetserverclient.c
@@ -1315,12 +1315,10 @@ static void virNetServerClientDispatchRead(virNetServerClientPtr client)
 
         /* Send off to for normal dispatch to workers */
         if (msg) {
-            virObjectRef(client);
             if (!client->dispatchFunc ||
                 client->dispatchFunc(client, msg, client->dispatchOpaque) < 0) {
                 virNetMessageFree(msg);
                 client->wantClose = true;
-                virObjectUnref(client);
                 return;
             }
         }