add get GC candidate (#12314)

* add get GC candidate

select non referenced blobs from table blob and exclude the ones in the time windows.

Signed-off-by: wang yan <wangyan@vmware.com>

src/pkg/blob/dao/dao.go

@@ -75,6 +75,9 @@ type DAO interface {
 
 	// DeleteBlob delete blob
 	DeleteBlob(ctx context.Context, id int64) (err error)
+
+	// GetBlobsNotRefedByProjectBlob get the blobs that are not referenced by the table project_blob and also not in the reserve window(in hours)
+	GetBlobsNotRefedByProjectBlob(ctx context.Context, timeWindowHours int64) ([]*models.Blob, error)
 }
 
 // New returns an instance of the default DAO
@@ -389,3 +392,19 @@ func (d *dao) DeleteBlob(ctx context.Context, id int64) error {
 	}
 	return nil
 }
+
+func (d *dao) GetBlobsNotRefedByProjectBlob(ctx context.Context, timeWindowHours int64) ([]*models.Blob, error) {
+	var noneRefed []*models.Blob
+	ormer, err := orm.FromContext(ctx)
+	if err != nil {
+		return noneRefed, err
+	}
+
+	sql := fmt.Sprintf(`SELECT b.id, b.digest, b.content_type, b.status FROM blob AS b LEFT JOIN project_blob pb ON b.id = pb.blob_id WHERE pb.id IS NULL AND b.update_time <= now() - interval '%d hours';`, timeWindowHours)
+	_, err = ormer.Raw(sql).QueryRows(&noneRefed)
+	if err != nil {
+		return noneRefed, err
+	}
+
+	return noneRefed, nil
+}

src/pkg/blob/dao/dao_test.go

@@ -376,6 +376,34 @@ func (suite *DaoTestSuite) TestDelete() {
 	suite.Require().Nil(err)
 }
 
+func (suite *DaoTestSuite) TestGetBlobsNotRefedByProjectBlob() {
+	ctx := suite.Context()
+
+	blobs, err := suite.dao.GetBlobsNotRefedByProjectBlob(ctx, 0)
+	suite.Require().Nil(err)
+	beforeAdd := len(blobs)
+
+	suite.dao.CreateBlob(ctx, &models.Blob{Digest: suite.DigestString()})
+	suite.dao.CreateBlob(ctx, &models.Blob{Digest: suite.DigestString()})
+	digest := suite.DigestString()
+	suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
+
+	blob, err := suite.dao.GetBlobByDigest(ctx, digest)
+	suite.Nil(err)
+
+	projectID := int64(1)
+	_, err = suite.dao.CreateProjectBlob(ctx, projectID, blob.ID)
+	suite.Nil(err)
+
+	blobs, err = suite.dao.GetBlobsNotRefedByProjectBlob(ctx, 0)
+	suite.Require().Nil(err)
+	suite.Require().Equal(2+beforeAdd, len(blobs))
+
+	blobs, err = suite.dao.GetBlobsNotRefedByProjectBlob(ctx, 2)
+	suite.Require().Nil(err)
+	suite.Require().Equal(0, len(blobs))
+}
+
 func TestDaoTestSuite(t *testing.T) {
 	suite.Run(t, &DaoTestSuite{})
 }

src/pkg/blob/manager.go

@@ -65,6 +65,9 @@ type Manager interface {
 
 	// DeleteBlob delete blob
 	Delete(ctx context.Context, id int64) (err error)
+
+	// UselessBlobs useless blob is the blob that is not used in any of projects.
+	UselessBlobs(ctx context.Context, timeWindowHours int64) ([]*models.Blob, error)
 }
 
 type manager struct {
@@ -129,6 +132,10 @@ func (m *manager) Delete(ctx context.Context, id int64) error {
 	return m.dao.DeleteBlob(ctx, id)
 }
 
+func (m *manager) UselessBlobs(ctx context.Context, timeWindowHours int64) ([]*models.Blob, error) {
+	return m.dao.GetBlobsNotRefedByProjectBlob(ctx, timeWindowHours)
+}
+
 // NewManager returns blob manager
 func NewManager() Manager {
 	return &manager{dao: dao.New()}

src/pkg/blob/manager_test.go

@@ -290,6 +290,32 @@ func (suite *ManagerTestSuite) TestUpdateStatus() {
 	}
 }
 
+func (suite *ManagerTestSuite) TestUselessBlobs() {
+	ctx := suite.Context()
+
+	blobs, err := Mgr.UselessBlobs(ctx, 0)
+	suite.Require().Nil(err)
+	beforeAdd := len(blobs)
+
+	Mgr.Create(ctx, suite.DigestString(), "media type", 100)
+	Mgr.Create(ctx, suite.DigestString(), "media type", 100)
+	digest := suite.DigestString()
+	blobID, err := Mgr.Create(ctx, digest, "media type", 100)
+	suite.Nil(err)
+
+	projectID := int64(1)
+	_, err = Mgr.AssociateWithProject(ctx, blobID, projectID)
+	suite.Nil(err)
+
+	blobs, err = Mgr.UselessBlobs(ctx, 0)
+	suite.Require().Nil(err)
+	suite.Require().Equal(2+beforeAdd, len(blobs))
+
+	blobs, err = Mgr.UselessBlobs(ctx, 2)
+	suite.Require().Nil(err)
+	suite.Require().Equal(0, len(blobs))
+}
+
 func TestManagerTestSuite(t *testing.T) {
 	suite.Run(t, &ManagerTestSuite{})
 }

src/testing/pkg/blob/manager.go

@@ -220,3 +220,26 @@ func (_m *Manager) UpdateBlobStatus(ctx context.Context, _a1 *models.Blob) (int6
 
 	return r0, r1
 }
+
+// UselessBlobs provides a mock function with given fields: ctx, timeWindow
+func (_m *Manager) UselessBlobs(ctx context.Context, timeWindow int64) ([]*models.Blob, error) {
+	ret := _m.Called(ctx, timeWindow)
+
+	var r0 []*models.Blob
+	if rf, ok := ret.Get(0).(func(context.Context, int64) []*models.Blob); ok {
+		r0 = rf(ctx, timeWindow)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*models.Blob)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok {
+		r1 = rf(ctx, timeWindow)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}