update blob controller & manager

1, add two more attributes, update_time and status
2, add delete and fresh update time method in blob mgr & ctr.

Signed-off-by: wang yan <wangyan@vmware.com>

make/migrations/postgresql/0035_2_1_0_schema.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE blob ADD COLUMN IF NOT EXISTS update_time timestamp default CURRENT_TIMESTAMP;
+ALTER TABLE blob ADD COLUMN IF NOT EXISTS status varchar(255);

src/common/models/base.go

@@ -34,7 +34,6 @@ func init() {
 		new(OIDCUser),
 		new(NotificationPolicy),
 		new(NotificationJob),
-		new(Blob),
 		new(ProjectBlob),
 		new(ArtifactAndBlob),
 		new(CVEWhitelist),

src/common/models/blob.go

@@ -1,34 +0,0 @@
-package models
-
-import (
-	"time"
-
-	"github.com/docker/distribution/manifest/schema2"
-)
-
-// Blob holds the details of a blob.
-type Blob struct {
-	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
-	Digest       string    `orm:"column(digest)" json:"digest"`
-	ContentType  string    `orm:"column(content_type)" json:"content_type"`
-	Size         int64     `orm:"column(size)" json:"size"`
-	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
-}
-
-// TableName ...
-func (b *Blob) TableName() string {
-	return "blob"
-}
-
-// IsForeignLayer returns true if the blob is foreign layer
-func (b *Blob) IsForeignLayer() bool {
-	return b.ContentType == schema2.MediaTypeForeignLayer
-}
-
-// BlobQuery ...
-type BlobQuery struct {
-	Digest      string
-	ContentType string
-	Digests     []string
-	Pagination
-}

src/controller/blob/controller.go

@@ -20,12 +20,12 @@ import (
 
 	"github.com/docker/distribution"
 	"github.com/garyburd/redigo/redis"
-	"github.com/goharbor/harbor/src/common/models"
 	util "github.com/goharbor/harbor/src/common/utils/redis"
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/pkg/blob"
+	"time"
 )
 
 var (
@@ -75,6 +75,9 @@ type Controller interface {
 
 	// GetAcceptedBlobSize returns the accepted size of stream upload blob.
 	GetAcceptedBlobSize(sessionID string) (int64, error)
+
+	// ReFreshUpdateTime updates the update time for the blob.
+	ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) (err error)
 }
 
 // NewController creates an instance of the default repository controller
@@ -184,7 +187,7 @@ func (c *controller) FindMissingAssociationsForProject(ctx context.Context, proj
 		associated[blob.Digest] = true
 	}
 
-	var results []*models.Blob
+	var results []*blob.Blob
 	for _, blob := range blobs {
 		if !associated[blob.Digest] {
 			results = append(results, blob)
@@ -314,3 +317,7 @@ func (c *controller) GetAcceptedBlobSize(sessionID string) (int64, error) {
 
 	return size, nil
 }
+
+func (c *controller) ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) (err error) {
+	return c.blobMgr.ReFreshUpdateTime(ctx, digest, time)
+}

src/controller/blob/controller_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/goharbor/harbor/src/pkg/blob"
@@ -267,6 +268,21 @@ func (suite *ControllerTestSuite) TestGetSetAcceptedBlobSize() {
 	suite.Equal(int64(100), size)
 }
 
+func (suite *ControllerTestSuite) TestReFreshUpdateTime() {
+	ctx := suite.Context()
+
+	digest := suite.prepareBlob()
+	blob, err := Ctl.Get(ctx, digest)
+	suite.Nil(err)
+
+	now := time.Now()
+	suite.NotEqual(blob.UpdateTime, now)
+
+	err = Ctl.ReFreshUpdateTime(ctx, blob.Digest, now)
+	suite.Nil(err)
+	suite.Equal(blob.UpdateTime.Unix(), now.Unix())
+}
+
 func TestControllerTestSuite(t *testing.T) {
 	suite.Run(t, &ControllerTestSuite{})
 }

src/jobservice/job/impl/gc/garbage_collection_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/pkg/artifact"
 	"github.com/goharbor/harbor/src/pkg/artifactrash/model"
+	pkg_blob "github.com/goharbor/harbor/src/pkg/blob/models"
 	artifacttesting "github.com/goharbor/harbor/src/testing/controller/artifact"
 	projecttesting "github.com/goharbor/harbor/src/testing/controller/project"
 	mockjobservice "github.com/goharbor/harbor/src/testing/jobservice"
@@ -93,7 +94,7 @@ func (suite *gcTestSuite) TestRemoveUntaggedBlobs() {
 		},
 	}, nil)
 
-	mock.OnAnything(suite.blobMgr, "List").Return([]*models.Blob{
+	mock.OnAnything(suite.blobMgr, "List").Return([]*pkg_blob.Blob{
 		{
 			ID:     1234,
 			Digest: "sha256:1234",
@@ -203,7 +204,7 @@ func (suite *gcTestSuite) TestRun() {
 		},
 	}, nil)
 
-	mock.OnAnything(suite.blobMgr, "List").Return([]*models.Blob{
+	mock.OnAnything(suite.blobMgr, "List").Return([]*pkg_blob.Blob{
 		{
 			ID:     12345,
 			Digest: "sha256:12345",

src/pkg/blob/dao/dao.go

@@ -17,6 +17,7 @@ package dao
 import (
 	"context"
 	"fmt"
+	"github.com/goharbor/harbor/src/lib/errors"
 	"strings"
 	"time"
 
@@ -24,6 +25,8 @@ import (
 	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/lib/q"
 	"github.com/goharbor/harbor/src/pkg/blob/models"
+
+	beego_orm "github.com/astaxie/beego/orm"
 )
 
 // DAO the dao for Blob, ArtifactAndBlob and ProjectBlob
@@ -66,6 +69,12 @@ type DAO interface {
 
 	// ExistProjectBlob returns true when ProjectBlob exist
 	ExistProjectBlob(ctx context.Context, projectID int64, blobDigest string) (bool, error)
+
+	// DeleteBlob delete blob
+	DeleteBlob(ctx context.Context, id int64) (err error)
+
+	// ReFreshUpdateTime updates the blob update time
+	ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) error
 }
 
 // New returns an instance of the default DAO
@@ -318,3 +327,30 @@ func (d *dao) DeleteProjectBlob(ctx context.Context, projectID int64, blobIDs ..
 	_, err = qs.Delete()
 	return err
 }
+
+func (d *dao) DeleteBlob(ctx context.Context, id int64) error {
+	ormer, err := orm.FromContext(ctx)
+	if err != nil {
+		return err
+	}
+	n, err := ormer.Delete(&models.Blob{
+		ID: id,
+	})
+	if err != nil {
+		return err
+	}
+	if n == 0 {
+		return errors.NotFoundError(nil).WithMessage("blob %d not found", id)
+	}
+	return nil
+}
+
+func (d *dao) ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) error {
+	qs, err := orm.QuerySetter(ctx, &models.Blob{}, &q.Query{
+		Keywords: map[string]interface{}{
+			"digest": digest,
+		},
+	})
+	_, err = qs.Update(beego_orm.Params{"update_time": time})
+	return err
+}

src/pkg/blob/dao/dao_test.go

@@ -15,7 +15,9 @@
 package dao
 
 import (
+	"github.com/goharbor/harbor/src/lib/errors"
 	"testing"
+	"time"
 
 	"github.com/goharbor/harbor/src/pkg/blob/models"
 	htesting "github.com/goharbor/harbor/src/testing"
@@ -322,6 +324,39 @@ func (suite *DaoTestSuite) TestDeleteProjectBlob() {
 	}
 }
 
+func (suite *DaoTestSuite) TestDelete() {
+	ctx := suite.Context()
+
+	err := suite.dao.DeleteBlob(ctx, 100021)
+	suite.Require().NotNil(err)
+	suite.True(errors.IsErr(err, errors.NotFoundCode))
+
+	digest := suite.DigestString()
+	id, err := suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
+	suite.Nil(err)
+	err = suite.dao.DeleteBlob(ctx, id)
+	suite.Require().Nil(err)
+}
+
+func (suite *DaoTestSuite) TestReFreshUpdateTime() {
+	ctx := suite.Context()
+	digest := suite.DigestString()
+	suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
+	blob, err := suite.dao.GetBlobByDigest(ctx, digest)
+	suite.Require().Nil(err)
+
+	time.Sleep(1 * time.Second)
+	now := time.Now()
+	suite.NotEqual(blob.UpdateTime, now)
+
+	if suite.Nil(suite.dao.ReFreshUpdateTime(ctx, blob.Digest, now)) {
+		blob, err := suite.dao.GetBlobByDigest(ctx, digest)
+		if suite.Nil(err) {
+			suite.Equal(now.Unix(), blob.UpdateTime.Unix())
+		}
+	}
+}
+
 func TestDaoTestSuite(t *testing.T) {
 	suite.Run(t, &DaoTestSuite{})
 }

src/pkg/blob/manager.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/goharbor/harbor/src/pkg/blob/dao"
 	"github.com/goharbor/harbor/src/pkg/blob/models"
+	"time"
 )
 
 // Blob alias `models.Blob` to make it natural to use the Manager
@@ -60,6 +61,12 @@ type Manager interface {
 
 	// List returns blobs by params
 	List(ctx context.Context, params ListParams) ([]*Blob, error)
+
+	// DeleteBlob delete blob
+	Delete(ctx context.Context, id int64) (err error)
+
+	// ReFreshUpdateTime updates the blob update time
+	ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) error
 }
 
 type manager struct {
@@ -116,6 +123,14 @@ func (m *manager) List(ctx context.Context, params ListParams) ([]*Blob, error)
 	return m.dao.ListBlobs(ctx, params)
 }
 
+func (m *manager) Delete(ctx context.Context, id int64) error {
+	return m.dao.DeleteBlob(ctx, id)
+}
+
+func (m *manager) ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) error {
+	return m.dao.ReFreshUpdateTime(ctx, digest, time)
+}
+
 // NewManager returns blob manager
 func NewManager() Manager {
 	return &manager{dao: dao.New()}

src/pkg/blob/manager_test.go

@@ -17,6 +17,7 @@ package blob
 import (
 	"context"
 	"testing"
+	"time"
 
 	htesting "github.com/goharbor/harbor/src/testing"
 	"github.com/stretchr/testify/suite"
@@ -256,6 +257,40 @@ func (suite *ManagerTestSuite) TestListByArtifact() {
 	suite.Len(blobs, 3)
 }
 
+func (suite *ManagerTestSuite) TestDelete() {
+	ctx := suite.Context()
+	digest := suite.DigestString()
+	blobID, err := Mgr.Create(ctx, digest, "media type", 100)
+	suite.Nil(err)
+
+	err = Mgr.Delete(ctx, blobID)
+	suite.Nil(err)
+}
+
+func (suite *ManagerTestSuite) TestReFreshUpdateTime() {
+	ctx := suite.Context()
+
+	digest := suite.DigestString()
+	_, err := Mgr.Create(ctx, digest, "media type", 100)
+	suite.Nil(err)
+
+	time.Sleep(1 * time.Second)
+	now := time.Now()
+
+	blob, err := Mgr.Get(ctx, digest)
+	if suite.Nil(err) {
+		blob.UpdateTime = now
+		suite.Nil(Mgr.Update(ctx, blob))
+
+		{
+			blob, err := Mgr.Get(ctx, digest)
+			suite.Nil(err)
+			suite.Equal(digest, blob.Digest)
+			suite.Equal(now.Unix(), blob.UpdateTime.Unix())
+		}
+	}
+}
+
 func TestManagerTestSuite(t *testing.T) {
 	suite.Run(t, &ManagerTestSuite{})
 }

src/pkg/blob/models/blob.go

@@ -15,16 +15,41 @@
 package models
 
 import (
+	"github.com/astaxie/beego/orm"
+	"github.com/docker/distribution/manifest/schema2"
 	"github.com/goharbor/harbor/src/common/models"
+	"time"
 )
 
-// TODO: move ArtifactAndBlob, Blob and ProjectBlob to here
+func init() {
+	orm.RegisterModel(&Blob{})
+}
+
+// TODO: move ArtifactAndBlob, ProjectBlob to here
 
 // ArtifactAndBlob alias ArtifactAndBlob model
 type ArtifactAndBlob = models.ArtifactAndBlob
 
-// Blob alias Blob model
+// Blob holds the details of a blob.
-type Blob = models.Blob
+type Blob struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	Digest       string    `orm:"column(digest)" json:"digest"`
+	ContentType  string    `orm:"column(content_type)" json:"content_type"`
+	Size         int64     `orm:"column(size)" json:"size"`
+	Status       string    `orm:"column(status)" json:"status"`
+	UpdateTime   time.Time `orm:"column(update_time);auto_now_add" json:"update_time"`
+	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+}
+
+// TableName ...
+func (b *Blob) TableName() string {
+	return "blob"
+}
+
+// IsForeignLayer returns true if the blob is foreign layer
+func (b *Blob) IsForeignLayer() bool {
+	return b.ContentType == schema2.MediaTypeForeignLayer
+}
 
 // ProjectBlob alias ProjectBlob model
 type ProjectBlob = models.ProjectBlob

src/testing/controller/blob/controller.go

@@ -3,16 +3,17 @@
 package blob
 
 import (
-	blob "github.com/goharbor/harbor/src/controller/blob"
-	blobmodels "github.com/goharbor/harbor/src/pkg/blob/models"
-
 	context "context"
 
+	blob "github.com/goharbor/harbor/src/controller/blob"
+
 	distribution "github.com/docker/distribution"
 
 	mock "github.com/stretchr/testify/mock"
 
-	models "github.com/goharbor/harbor/src/common/models"
+	models "github.com/goharbor/harbor/src/pkg/blob/models"
+
+	time "time"
 )
 
 // Controller is an autogenerated mock type for the Controller type
@@ -207,11 +208,11 @@ func (_m *Controller) GetAcceptedBlobSize(sessionID string) (int64, error) {
 }
 
 // List provides a mock function with given fields: ctx, params
-func (_m *Controller) List(ctx context.Context, params blobmodels.ListParams) ([]*models.Blob, error) {
+func (_m *Controller) List(ctx context.Context, params models.ListParams) ([]*models.Blob, error) {
 	ret := _m.Called(ctx, params)
 
 	var r0 []*models.Blob
-	if rf, ok := ret.Get(0).(func(context.Context, blobmodels.ListParams) []*models.Blob); ok {
+	if rf, ok := ret.Get(0).(func(context.Context, models.ListParams) []*models.Blob); ok {
 		r0 = rf(ctx, params)
 	} else {
 		if ret.Get(0) != nil {
@@ -220,7 +221,7 @@ func (_m *Controller) List(ctx context.Context, params blobmodels.ListParams) ([
 	}
 
 	var r1 error
-	if rf, ok := ret.Get(1).(func(context.Context, blobmodels.ListParams) error); ok {
+	if rf, ok := ret.Get(1).(func(context.Context, models.ListParams) error); ok {
 		r1 = rf(ctx, params)
 	} else {
 		r1 = ret.Error(1)
@@ -229,6 +230,20 @@ func (_m *Controller) List(ctx context.Context, params blobmodels.ListParams) ([
 	return r0, r1
 }
 
+// ReFreshUpdateTime provides a mock function with given fields: ctx, digest, _a2
+func (_m *Controller) ReFreshUpdateTime(ctx context.Context, digest string, _a2 time.Time) error {
+	ret := _m.Called(ctx, digest, _a2)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, string, time.Time) error); ok {
+		r0 = rf(ctx, digest, _a2)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
 // SetAcceptedBlobSize provides a mock function with given fields: sessionID, size
 func (_m *Controller) SetAcceptedBlobSize(sessionID string, size int64) error {
 	ret := _m.Called(sessionID, size)

src/testing/pkg/blob/manager.go

@@ -5,11 +5,10 @@ package blob
 import (
 	context "context"
 
-	blobmodels "github.com/goharbor/harbor/src/pkg/blob/models"
+	models "github.com/goharbor/harbor/src/pkg/blob/models"
-
 	mock "github.com/stretchr/testify/mock"
 
-	models "github.com/goharbor/harbor/src/common/models"
+	time "time"
 )
 
 // Manager is an autogenerated mock type for the Manager type
@@ -129,6 +128,20 @@ func (_m *Manager) Create(ctx context.Context, digest string, contentType string
 	return r0, r1
 }
 
+// Delete provides a mock function with given fields: ctx, id
+func (_m *Manager) Delete(ctx context.Context, id int64) error {
+	ret := _m.Called(ctx, id)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok {
+		r0 = rf(ctx, id)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
 // Get provides a mock function with given fields: ctx, digest
 func (_m *Manager) Get(ctx context.Context, digest string) (*models.Blob, error) {
 	ret := _m.Called(ctx, digest)
@@ -153,11 +166,11 @@ func (_m *Manager) Get(ctx context.Context, digest string) (*models.Blob, error)
 }
 
 // List provides a mock function with given fields: ctx, params
-func (_m *Manager) List(ctx context.Context, params blobmodels.ListParams) ([]*models.Blob, error) {
+func (_m *Manager) List(ctx context.Context, params models.ListParams) ([]*models.Blob, error) {
 	ret := _m.Called(ctx, params)
 
 	var r0 []*models.Blob
-	if rf, ok := ret.Get(0).(func(context.Context, blobmodels.ListParams) []*models.Blob); ok {
+	if rf, ok := ret.Get(0).(func(context.Context, models.ListParams) []*models.Blob); ok {
 		r0 = rf(ctx, params)
 	} else {
 		if ret.Get(0) != nil {
@@ -166,7 +179,7 @@ func (_m *Manager) List(ctx context.Context, params blobmodels.ListParams) ([]*m
 	}
 
 	var r1 error
-	if rf, ok := ret.Get(1).(func(context.Context, blobmodels.ListParams) error); ok {
+	if rf, ok := ret.Get(1).(func(context.Context, models.ListParams) error); ok {
 		r1 = rf(ctx, params)
 	} else {
 		r1 = ret.Error(1)
@@ -175,6 +188,20 @@ func (_m *Manager) List(ctx context.Context, params blobmodels.ListParams) ([]*m
 	return r0, r1
 }
 
+// ReFreshUpdateTime provides a mock function with given fields: ctx, digest, _a2
+func (_m *Manager) ReFreshUpdateTime(ctx context.Context, digest string, _a2 time.Time) error {
+	ret := _m.Called(ctx, digest, _a2)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, string, time.Time) error); ok {
+		r0 = rf(ctx, digest, _a2)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
 // Update provides a mock function with given fields: ctx, _a1
 func (_m *Manager) Update(ctx context.Context, _a1 *models.Blob) error {
 	ret := _m.Called(ctx, _a1)