diff --git a/src/pkg/scan/sbom/sbom.go b/src/pkg/scan/sbom/sbom.go
index 67949a796..0ea4a9379 100644
--- a/src/pkg/scan/sbom/sbom.go
+++ b/src/pkg/scan/sbom/sbom.go
@@ -32,6 +32,7 @@ import (
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/lib/orm"
+	accessoryModel "github.com/goharbor/harbor/src/pkg/accessory/model"
 	"github.com/goharbor/harbor/src/pkg/permission/types"
 	"github.com/goharbor/harbor/src/pkg/robot/model"
 	"github.com/goharbor/harbor/src/pkg/scan"
@@ -247,7 +248,7 @@ func (h *scanHandler) deleteSBOMAccessories(ctx context.Context, reports []*sbom
 		if rpt.MimeType != v1.MimeTypeSBOMReport {
 			continue
 		}
-		if err := h.deleteSBOMAccessory(ctx, rpt.ReportSummary); err != nil {
+		if err := h.deleteSBOMAccessory(ctx, rpt.ArtifactID); err != nil {
 			return err
 		}
 		if err := mgr.Delete(ctx, rpt.UUID); err != nil {
@@ -258,36 +259,25 @@ func (h *scanHandler) deleteSBOMAccessories(ctx context.Context, reports []*sbom
 }
 
 // deleteSBOMAccessory check if current report has sbom accessory info, if there is, delete it
-func (h *scanHandler) deleteSBOMAccessory(ctx context.Context, report string) error {
-	if len(report) == 0 {
-		return nil
-	}
-	sbomSummary := sbom.Summary{}
-	if err := json.Unmarshal([]byte(report), &sbomSummary); err != nil {
-		// it could be a non sbom report, just skip
-		log.Debugf("fail to unmarshal %v, skip to delete sbom report", err)
-		return nil
-	}
-	repo, dgst := sbomSummary.SBOMAccArt()
-	if len(repo) == 0 || len(dgst) == 0 {
-		return nil
-	}
+func (h *scanHandler) deleteSBOMAccessory(ctx context.Context, artID int64) error {
 	artifactCtl := h.ArtifactControllerFunc()
-	art, err := artifactCtl.GetByReference(ctx, repo, dgst, nil)
-	if errors.IsNotFoundErr(err) {
-		return nil
-	}
+	art, err := artifactCtl.Get(ctx, artID, &artifact.Option{
+		WithAccessory: true,
+	})
 	if err != nil {
 		return err
 	}
 	if art == nil {
 		return nil
 	}
-	err = artifactCtl.Delete(ctx, art.ID)
-	if errors.IsNotFoundErr(err) {
-		return nil
+	for _, acc := range art.Accessories {
+		if acc.GetData().Type == accessoryModel.TypeHarborSBOM {
+			if err := artifactCtl.Delete(ctx, acc.GetData().ArtifactID); err != nil {
+				return err
+			}
+		}
 	}
-	return err
+	return nil
 }
 
 func (h *scanHandler) GetPlaceHolder(ctx context.Context, artRepo string, artDigest, scannerUUID string, mimeType string) (rp *scanModel.Report, err error) {
diff --git a/src/pkg/scan/sbom/sbom_test.go b/src/pkg/scan/sbom/sbom_test.go
index 53c5a5948..4b4c3645b 100644
--- a/src/pkg/scan/sbom/sbom_test.go
+++ b/src/pkg/scan/sbom/sbom_test.go
@@ -12,6 +12,8 @@ import (
 	sc "github.com/goharbor/harbor/src/controller/scan"
 	"github.com/goharbor/harbor/src/controller/scanner"
 	"github.com/goharbor/harbor/src/lib/orm"
+	accessoryModel "github.com/goharbor/harbor/src/pkg/accessory/model"
+	basemodel "github.com/goharbor/harbor/src/pkg/accessory/model/base"
 	art "github.com/goharbor/harbor/src/pkg/artifact"
 	sbomModel "github.com/goharbor/harbor/src/pkg/scan/sbom/model"
 	htesting "github.com/goharbor/harbor/src/testing"
@@ -194,7 +196,16 @@ func (suite *SBOMTestSuite) TestPostScan() {
 
 func (suite *SBOMTestSuite) TestMakeReportPlaceHolder() {
 	ctx := orm.NewContext(nil, &ormtesting.FakeOrmer{})
-	art := &artifact.Artifact{Artifact: art.Artifact{ID: 1, Digest: "digest", ManifestMediaType: v1.MimeTypeDockerArtifact}}
+	acc := &basemodel.Default{
+		Data: accessoryModel.AccessoryData{
+			ID:                1,
+			ArtifactID:        2,
+			SubArtifactDigest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
+			Type:              accessoryModel.TypeHarborSBOM,
+		},
+	}
+	art := &artifact.Artifact{Artifact: art.Artifact{ID: 1, Digest: "digest", ManifestMediaType: v1.MimeTypeDockerArtifact},
+		Accessories: []accessoryModel.Accessory{acc}}
 	r := &scanner.Registration{
 		UUID: "uuid",
 		Metadata: &v1.ScannerAdapterMetadata{
@@ -208,6 +219,8 @@ func (suite *SBOMTestSuite) TestMakeReportPlaceHolder() {
 	mock.OnAnything(suite.sbomManager, "Create").Return("uuid", nil).Once()
 	mock.OnAnything(suite.sbomManager, "Delete").Return(nil).Once()
 	mock.OnAnything(suite.taskMgr, "ListScanTasksByReportUUID").Return([]*task.Task{{Status: "Success"}}, nil)
+	mock.OnAnything(suite.artifactCtl, "Get").Return(art, nil)
+	mock.OnAnything(suite.artifactCtl, "Delete").Return(nil)
 	rps, err := suite.handler.MakePlaceHolder(ctx, art, r)
 	require.NoError(suite.T(), err)
 	suite.Equal(1, len(rps))