Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-22 02:05:41 +01:00
Code Issues Projects Releases Wiki Activity

update blob controller & manager

Browse Source 
1, add two more attributes, update_time and status
2, add delete and fresh update time method in blob mgr & ctr.

Signed-off-by: wang yan <wangyan@vmware.com>
This commit is contained in:
Wang Yan 2020-05-28 18:48:36 +08:00 committed by wang yan
parent b983961260
commit de504993ad
13 changed files with 234 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
make/migrations/postgresql/0035_2_1_0_schema.up.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE blob ADD COLUMN IF NOT EXISTS update_time timestamp default CURRENT_TIMESTAMP;
ALTER TABLE blob ADD COLUMN IF NOT EXISTS status varchar(255);

1
src/common/models/base.go
View File

@ -34,7 +34,6 @@ func init() {
new(OIDCUser),
new(NotificationPolicy),
new(NotificationJob),
new(Blob),
new(ProjectBlob),
new(ArtifactAndBlob),
new(CVEWhitelist),

34
src/common/models/blob.go
View File

@ -1,34 +0,0 @@
package models
import (
"time"
"github.com/docker/distribution/manifest/schema2"
)
// Blob holds the details of a blob.
type Blob struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Digest string `orm:"column(digest)" json:"digest"`
ContentType string `orm:"column(content_type)" json:"content_type"`
Size int64 `orm:"column(size)" json:"size"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
}
// TableName ...
func (b *Blob) TableName() string {
return "blob"
}
// IsForeignLayer returns true if the blob is foreign layer
func (b *Blob) IsForeignLayer() bool {
return b.ContentType == schema2.MediaTypeForeignLayer
}
// BlobQuery ...
type BlobQuery struct {
Digest string
ContentType string
Digests []string
Pagination
}

11
src/controller/blob/controller.go
View File

@ -20,12 +20,12 @@ import (
"github.com/docker/distribution"
"github.com/garyburd/redigo/redis"
"github.com/goharbor/harbor/src/common/models"
util "github.com/goharbor/harbor/src/common/utils/redis"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/blob"
"time"
)
var (
@ -75,6 +75,9 @@ type Controller interface {
// GetAcceptedBlobSize returns the accepted size of stream upload blob.
GetAcceptedBlobSize(sessionID string) (int64, error)
// ReFreshUpdateTime updates the update time for the blob.
ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) (err error)
}
// NewController creates an instance of the default repository controller
@ -184,7 +187,7 @@ func (c *controller) FindMissingAssociationsForProject(ctx context.Context, proj
associated[blob.Digest] = true
}
var results []*models.Blob
var results []*blob.Blob
for _, blob := range blobs {
if !associated[blob.Digest] {
results = append(results, blob)
@ -314,3 +317,7 @@ func (c *controller) GetAcceptedBlobSize(sessionID string) (int64, error) {
return size, nil
}
func (c *controller) ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) (err error) {
return c.blobMgr.ReFreshUpdateTime(ctx, digest, time)
}

16
src/controller/blob/controller_test.go
View File

@ -18,6 +18,7 @@ import (
"context"
"fmt"
"testing"
"time"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/pkg/blob"
@ -267,6 +268,21 @@ func (suite *ControllerTestSuite) TestGetSetAcceptedBlobSize() {
suite.Equal(int64(100), size)
}
func (suite *ControllerTestSuite) TestReFreshUpdateTime() {
ctx := suite.Context()
digest := suite.prepareBlob()
blob, err := Ctl.Get(ctx, digest)
suite.Nil(err)
now := time.Now()
suite.NotEqual(blob.UpdateTime, now)
err = Ctl.ReFreshUpdateTime(ctx, blob.Digest, now)
suite.Nil(err)
suite.Equal(blob.UpdateTime.Unix(), now.Unix())
}
func TestControllerTestSuite(t *testing.T) {
suite.Run(t, &ControllerTestSuite{})
}

5
src/jobservice/job/impl/gc/garbage_collection_test.go
View File

@ -7,6 +7,7 @@ import (
"github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/artifactrash/model"
pkg_blob "github.com/goharbor/harbor/src/pkg/blob/models"
artifacttesting "github.com/goharbor/harbor/src/testing/controller/artifact"
projecttesting "github.com/goharbor/harbor/src/testing/controller/project"
mockjobservice "github.com/goharbor/harbor/src/testing/jobservice"
@ -93,7 +94,7 @@ func (suite *gcTestSuite) TestRemoveUntaggedBlobs() {
},
}, nil)
mock.OnAnything(suite.blobMgr, "List").Return([]*models.Blob{
mock.OnAnything(suite.blobMgr, "List").Return([]*pkg_blob.Blob{
{
ID: 1234,
Digest: "sha256:1234",
@ -203,7 +204,7 @@ func (suite *gcTestSuite) TestRun() {
},
}, nil)
mock.OnAnything(suite.blobMgr, "List").Return([]*models.Blob{
mock.OnAnything(suite.blobMgr, "List").Return([]*pkg_blob.Blob{
{
ID: 12345,
Digest: "sha256:12345",

36
src/pkg/blob/dao/dao.go
View File

@ -17,6 +17,7 @@ package dao
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"strings"
"time"
@ -24,6 +25,8 @@ import (
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/blob/models"
beego_orm "github.com/astaxie/beego/orm"
)
// DAO the dao for Blob, ArtifactAndBlob and ProjectBlob
@ -66,6 +69,12 @@ type DAO interface {
// ExistProjectBlob returns true when ProjectBlob exist
ExistProjectBlob(ctx context.Context, projectID int64, blobDigest string) (bool, error)
// DeleteBlob delete blob
DeleteBlob(ctx context.Context, id int64) (err error)
// ReFreshUpdateTime updates the blob update time
ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) error
}
// New returns an instance of the default DAO
@ -318,3 +327,30 @@ func (d *dao) DeleteProjectBlob(ctx context.Context, projectID int64, blobIDs ..
_, err = qs.Delete()
return err
}
func (d *dao) DeleteBlob(ctx context.Context, id int64) error {
ormer, err := orm.FromContext(ctx)
if err != nil {
return err
}
n, err := ormer.Delete(&models.Blob{
ID: id,
})
if err != nil {
return err
}
if n == 0 {
return errors.NotFoundError(nil).WithMessage("blob %d not found", id)
}
return nil
}
func (d *dao) ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) error {
qs, err := orm.QuerySetter(ctx, &models.Blob{}, &q.Query{
Keywords: map[string]interface{}{
"digest": digest,
},
})
_, err = qs.Update(beego_orm.Params{"update_time": time})
return err
}

35
src/pkg/blob/dao/dao_test.go
View File

@ -15,7 +15,9 @@
package dao
import (
"github.com/goharbor/harbor/src/lib/errors"
"testing"
"time"
"github.com/goharbor/harbor/src/pkg/blob/models"
htesting "github.com/goharbor/harbor/src/testing"
@ -322,6 +324,39 @@ func (suite *DaoTestSuite) TestDeleteProjectBlob() {
}
}
func (suite *DaoTestSuite) TestDelete() {
ctx := suite.Context()
err := suite.dao.DeleteBlob(ctx, 100021)
suite.Require().NotNil(err)
suite.True(errors.IsErr(err, errors.NotFoundCode))
digest := suite.DigestString()
id, err := suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
suite.Nil(err)
err = suite.dao.DeleteBlob(ctx, id)
suite.Require().Nil(err)
}
func (suite *DaoTestSuite) TestReFreshUpdateTime() {
ctx := suite.Context()
digest := suite.DigestString()
suite.dao.CreateBlob(ctx, &models.Blob{Digest: digest})
blob, err := suite.dao.GetBlobByDigest(ctx, digest)
suite.Require().Nil(err)
time.Sleep(1 * time.Second)
now := time.Now()
suite.NotEqual(blob.UpdateTime, now)
if suite.Nil(suite.dao.ReFreshUpdateTime(ctx, blob.Digest, now)) {
blob, err := suite.dao.GetBlobByDigest(ctx, digest)
if suite.Nil(err) {
suite.Equal(now.Unix(), blob.UpdateTime.Unix())
}
}
}
func TestDaoTestSuite(t *testing.T) {
suite.Run(t, &DaoTestSuite{})
}

15
src/pkg/blob/manager.go
View File

@ -19,6 +19,7 @@ import (
"github.com/goharbor/harbor/src/pkg/blob/dao"
"github.com/goharbor/harbor/src/pkg/blob/models"
"time"
)
// Blob alias `models.Blob` to make it natural to use the Manager
@ -60,6 +61,12 @@ type Manager interface {
// List returns blobs by params
List(ctx context.Context, params ListParams) ([]*Blob, error)
// DeleteBlob delete blob
Delete(ctx context.Context, id int64) (err error)
// ReFreshUpdateTime updates the blob update time
ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) error
}
type manager struct {
@ -116,6 +123,14 @@ func (m *manager) List(ctx context.Context, params ListParams) ([]*Blob, error)
return m.dao.ListBlobs(ctx, params)
}
func (m *manager) Delete(ctx context.Context, id int64) error {
return m.dao.DeleteBlob(ctx, id)
}
func (m *manager) ReFreshUpdateTime(ctx context.Context, digest string, time time.Time) error {
return m.dao.ReFreshUpdateTime(ctx, digest, time)
}
// NewManager returns blob manager
func NewManager() Manager {
return &manager{dao: dao.New()}

35
src/pkg/blob/manager_test.go
View File

@ -17,6 +17,7 @@ package blob
import (
"context"
"testing"
"time"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/suite"
@ -256,6 +257,40 @@ func (suite *ManagerTestSuite) TestListByArtifact() {
suite.Len(blobs, 3)
}
func (suite *ManagerTestSuite) TestDelete() {
ctx := suite.Context()
digest := suite.DigestString()
blobID, err := Mgr.Create(ctx, digest, "media type", 100)
suite.Nil(err)
err = Mgr.Delete(ctx, blobID)
suite.Nil(err)
}
func (suite *ManagerTestSuite) TestReFreshUpdateTime() {
ctx := suite.Context()
digest := suite.DigestString()
_, err := Mgr.Create(ctx, digest, "media type", 100)
suite.Nil(err)
time.Sleep(1 * time.Second)
now := time.Now()
blob, err := Mgr.Get(ctx, digest)
if suite.Nil(err) {
blob.UpdateTime = now
suite.Nil(Mgr.Update(ctx, blob))
{
blob, err := Mgr.Get(ctx, digest)
suite.Nil(err)
suite.Equal(digest, blob.Digest)
suite.Equal(now.Unix(), blob.UpdateTime.Unix())
}
}
}
func TestManagerTestSuite(t *testing.T) {
suite.Run(t, &ManagerTestSuite{})
}

31
src/pkg/blob/models/blob.go
View File

@ -15,16 +15,41 @@
package models
import (
"github.com/astaxie/beego/orm"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/models"
"time"
)
// TODO: move ArtifactAndBlob, Blob and ProjectBlob to here
func init() {
orm.RegisterModel(&Blob{})
}
// TODO: move ArtifactAndBlob, ProjectBlob to here
// ArtifactAndBlob alias ArtifactAndBlob model
type ArtifactAndBlob = models.ArtifactAndBlob
// Blob alias Blob model
type Blob = models.Blob
// Blob holds the details of a blob.
type Blob struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Digest string `orm:"column(digest)" json:"digest"`
ContentType string `orm:"column(content_type)" json:"content_type"`
Size int64 `orm:"column(size)" json:"size"`
Status string `orm:"column(status)" json:"status"`
UpdateTime time.Time `orm:"column(update_time);auto_now_add" json:"update_time"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
}
// TableName ...
func (b *Blob) TableName() string {
return "blob"
}
// IsForeignLayer returns true if the blob is foreign layer
func (b *Blob) IsForeignLayer() bool {
return b.ContentType == schema2.MediaTypeForeignLayer
}
// ProjectBlob alias ProjectBlob model
type ProjectBlob = models.ProjectBlob

29
src/testing/controller/blob/controller.go
View File

@ -3,16 +3,17 @@
package blob
import (
blob "github.com/goharbor/harbor/src/controller/blob"
blobmodels "github.com/goharbor/harbor/src/pkg/blob/models"
context "context"
blob "github.com/goharbor/harbor/src/controller/blob"
distribution "github.com/docker/distribution"
mock "github.com/stretchr/testify/mock"
models "github.com/goharbor/harbor/src/common/models"
models "github.com/goharbor/harbor/src/pkg/blob/models"
time "time"
)
// Controller is an autogenerated mock type for the Controller type
@ -207,11 +208,11 @@ func (_m *Controller) GetAcceptedBlobSize(sessionID string) (int64, error) {
}
// List provides a mock function with given fields: ctx, params
func (_m *Controller) List(ctx context.Context, params blobmodels.ListParams) ([]*models.Blob, error) {
func (_m *Controller) List(ctx context.Context, params models.ListParams) ([]*models.Blob, error) {
ret := _m.Called(ctx, params)
var r0 []*models.Blob
if rf, ok := ret.Get(0).(func(context.Context, blobmodels.ListParams) []*models.Blob); ok {
if rf, ok := ret.Get(0).(func(context.Context, models.ListParams) []*models.Blob); ok {
r0 = rf(ctx, params)
} else {
if ret.Get(0) != nil {
@ -220,7 +221,7 @@ func (_m *Controller) List(ctx context.Context, params blobmodels.ListParams) ([
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, blobmodels.ListParams) error); ok {
if rf, ok := ret.Get(1).(func(context.Context, models.ListParams) error); ok {
r1 = rf(ctx, params)
} else {
r1 = ret.Error(1)
@ -229,6 +230,20 @@ func (_m *Controller) List(ctx context.Context, params blobmodels.ListParams) ([
return r0, r1
}
// ReFreshUpdateTime provides a mock function with given fields: ctx, digest, _a2
func (_m *Controller) ReFreshUpdateTime(ctx context.Context, digest string, _a2 time.Time) error {
ret := _m.Called(ctx, digest, _a2)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string, time.Time) error); ok {
r0 = rf(ctx, digest, _a2)
} else {
r0 = ret.Error(0)
}
return r0
}
// SetAcceptedBlobSize provides a mock function with given fields: sessionID, size
func (_m *Controller) SetAcceptedBlobSize(sessionID string, size int64) error {
ret := _m.Called(sessionID, size)

39
src/testing/pkg/blob/manager.go
View File

@ -5,11 +5,10 @@ package blob
import (
context "context"
blobmodels "github.com/goharbor/harbor/src/pkg/blob/models"
models "github.com/goharbor/harbor/src/pkg/blob/models"
mock "github.com/stretchr/testify/mock"
models "github.com/goharbor/harbor/src/common/models"
time "time"
)
// Manager is an autogenerated mock type for the Manager type
@ -129,6 +128,20 @@ func (_m *Manager) Create(ctx context.Context, digest string, contentType string
return r0, r1
}
// Delete provides a mock function with given fields: ctx, id
func (_m *Manager) Delete(ctx context.Context, id int64) error {
ret := _m.Called(ctx, id)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok {
r0 = rf(ctx, id)
} else {
r0 = ret.Error(0)
}
return r0
}
// Get provides a mock function with given fields: ctx, digest
func (_m *Manager) Get(ctx context.Context, digest string) (*models.Blob, error) {
ret := _m.Called(ctx, digest)
@ -153,11 +166,11 @@ func (_m *Manager) Get(ctx context.Context, digest string) (*models.Blob, error)
}
// List provides a mock function with given fields: ctx, params
func (_m *Manager) List(ctx context.Context, params blobmodels.ListParams) ([]*models.Blob, error) {
func (_m *Manager) List(ctx context.Context, params models.ListParams) ([]*models.Blob, error) {
ret := _m.Called(ctx, params)
var r0 []*models.Blob
if rf, ok := ret.Get(0).(func(context.Context, blobmodels.ListParams) []*models.Blob); ok {
if rf, ok := ret.Get(0).(func(context.Context, models.ListParams) []*models.Blob); ok {
r0 = rf(ctx, params)
} else {
if ret.Get(0) != nil {
@ -166,7 +179,7 @@ func (_m *Manager) List(ctx context.Context, params blobmodels.ListParams) ([]*m
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, blobmodels.ListParams) error); ok {
if rf, ok := ret.Get(1).(func(context.Context, models.ListParams) error); ok {
r1 = rf(ctx, params)
} else {
r1 = ret.Error(1)
@ -175,6 +188,20 @@ func (_m *Manager) List(ctx context.Context, params blobmodels.ListParams) ([]*m
return r0, r1
}
// ReFreshUpdateTime provides a mock function with given fields: ctx, digest, _a2
func (_m *Manager) ReFreshUpdateTime(ctx context.Context, digest string, _a2 time.Time) error {
ret := _m.Called(ctx, digest, _a2)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string, time.Time) error); ok {
r0 = rf(ctx, digest, _a2)
} else {
r0 = ret.Error(0)
}
return r0
}
// Update provides a mock function with given fields: ctx, _a1
func (_m *Manager) Update(ctx context.Context, _a1 *models.Blob) error {
ret := _m.Called(ctx, _a1)