diff --git a/src/controller/artifact/controller.go b/src/controller/artifact/controller.go
index 34ea29077..6db99fccb 100644
--- a/src/controller/artifact/controller.go
+++ b/src/controller/artifact/controller.go
@@ -326,12 +326,6 @@ func (c *controller) deleteDeeply(ctx context.Context, id int64, isRoot, isAcces
 		return err
 	}
 
-	if isAccessory {
-		if err := c.accessoryMgr.DeleteAccessories(ctx, q.New(q.KeyWords{"ArtifactID": art.ID, "Digest": art.Digest})); err != nil && !errors.IsErr(err, errors.NotFoundCode) {
-			return err
-		}
-	}
-
 	// the child artifact is referenced by some tags, skip
 	if !isRoot && len(art.Tags) > 0 {
 		return nil
@@ -354,6 +348,12 @@ func (c *controller) deleteDeeply(ctx context.Context, id int64, isRoot, isAcces
 		return nil
 	}
 
+	if isAccessory {
+		if err := c.accessoryMgr.DeleteAccessories(ctx, q.New(q.KeyWords{"ArtifactID": art.ID, "Digest": art.Digest})); err != nil && !errors.IsErr(err, errors.NotFoundCode) {
+			return err
+		}
+	}
+
 	// delete accessories if contains any
 	for _, acc := range art.Accessories {
 		// only hard ref accessory should be removed