Define the controller/manager interface for artifact and tag

1. Define the controller/manager interface for artifact and tag
    2. Provide a null implementation for artifact manager

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2019-12-13 08:49:45 +08:00
parent 26905baca2
commit ac605db5da
11 changed files with 348 additions and 166 deletions

View File

@ -3,19 +3,18 @@ CREATE TABLE artifact_2
(
id SERIAL PRIMARY KEY NOT NULL,
/* image, chart, etc */
type varchar(255),
media_type varchar(255),
type varchar(255) NOT NULL,
media_type varchar(255) NOT NULL,
/* the media type of some classical image manifest can be null, so don't add the "NOT NULL" constraint*/
manifest_media_type varchar(255),
project_id int NOT NULL,
repository_id int NOT NULL,
digest varchar(255) NOT NULL,
size bigint,
push_time timestamp default CURRENT_TIMESTAMP,
platform varchar(255),
pull_time timestamp,
extra_attrs text,
annotations jsonb,
/* when updating the data the revision MUST be checked and updated */
revision varchar(64) NOT NULL,
CONSTRAINT unique_artifact_2 UNIQUE (repository_id, digest)
);
@ -24,19 +23,18 @@ CREATE TABLE tag
id SERIAL PRIMARY KEY NOT NULL,
repository_id int NOT NULL,
artifact_id int NOT NULL,
name varchar(255),
name varchar(255) NOT NULL,
push_time timestamp default CURRENT_TIMESTAMP,
pull_time timestamp,
/* when updating the data the revision MUST be checked and updated */
revision varchar(64) NOT NULL,
CONSTRAINT unique_tag UNIQUE (repository_id, name)
);
/* artifact_reference records the child artifact referenced by parent artifact */
CREATE TABLE artifact_reference
(
id SERIAL PRIMARY KEY NOT NULL,
id SERIAL PRIMARY KEY NOT NULL,
parent_id int NOT NULL,
child_id int NOT NULL,
CONSTRAINT unique_reference UNIQUE (parent_id, child_id)
child_id int NOT NULL,
platform varchar(255),
CONSTRAINT unique_reference UNIQUE (parent_id, child_id)
);

View File

@ -0,0 +1,63 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package artifact
import (
"context"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/q"
"time"
)
var (
// Ctl is a global artifact controller instance
Ctl = NewController()
)
// Controller defines the operations related with artifacts and tags
type Controller interface {
// Ensure the artifact specified by the digest exists under the repository,
// creates it if it doesn't exist. If tags are provided, ensure they exist
// and are attached to the artifact. If the tags don't exist, create them first
Ensure(ctx context.Context, repository *models.RepoRecord, digest string, tags ...string) (err error)
// List artifacts according to the query and option
List(ctx context.Context, query *q.Query, option *Option) (total int64, artifacts []*Artifact, err error)
// Get the artifact specified by ID
Get(ctx context.Context, id int64, option *Option) (artifact *Artifact, err error)
// Delete the artifact specified by ID
Delete(ctx context.Context, id int64) (err error)
// DeleteTag deletes the tag specified by ID
DeleteTag(ctx context.Context, id int64) (err error)
// UpdatePullTime updates the pull time for the artifact. If the tag is provides, update the pull
// time of the tag as well
UpdatePullTime(ctx context.Context, artifactID int64, tag string, time time.Time) (err error)
// GetSubResource returns the sub resource of the artifact
// The sub resource is different according to the artifact type:
// build history for image; values.yaml, readme and dependencies for chart, etc
GetSubResource(ctx context.Context, artifactID int64, resource string) (*Resource, error)
// TODO move this to GC controller?
// Prune removes the useless artifact records. The underlying registry data will
// be removed during garbage collection
// Prune(ctx context.Context, option *Option) error
}
// NewController creates an instance of the default artifact controller
func NewController() Controller {
// TODO implement
return nil
}
// As a redis lock is applied during the artifact pushing, we do not to handle the concurrent issues
// for artifacts and tags

55
src/api/artifact/model.go Normal file
View File

@ -0,0 +1,55 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package artifact
import (
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
)
// Artifact is the overall view of artifact
// TODO reuse the model generated by swagger
type Artifact struct {
artifact.Artifact
Tags []*tag.Tag // the list of tags that attached to the artifact
SubResourceLinks map[string][]*ResourceLink // the resource link for build history(image), values.yaml(chart), dependency(chart), etc
// TODO add other attrs: signature, scan result, label, etc
}
// Resource defines the specific resource of different artifacts: build history for image, values.yaml for chart, etc
type Resource struct {
Content []byte // the content of the resource
ContentType string // the content type of the resource, returned as "Content-Type" header in API
}
// ResourceLink is a link via that a resource can be fetched
type ResourceLink struct {
HREF string
Absolute bool // specify the href is an absolute URL or not
}
// Option is used to specify the properties returned when listing/getting artifacts
type Option struct {
WithTag bool
WithLabel bool
WithScanResult bool
WithSignature bool
}
// TODO move this to GC controller?
// Option for pruning artifact records
// type Option struct {
// KeepUntagged bool // keep the untagged artifacts or not
// }

View File

@ -22,7 +22,6 @@ import (
func init() {
orm.RegisterModel(&Artifact{})
orm.RegisterModel(&Tag{})
orm.RegisterModel(&ArtifactReference{})
}
@ -37,10 +36,9 @@ type Artifact struct {
Digest string `orm:"column(digest)"`
Size int64 `orm:"column(size)"`
PushTime time.Time `orm:"column(push_time)"`
Platform string `orm:"column(platform)"` // json string
PullTime time.Time `orm:"column(pull_time)"`
ExtraAttrs string `orm:"column(extra_attrs)"` // json string
Annotations string `orm:"column(annotations);type(jsonb)"` // json string
Revision string `orm:"column(revision)"` // record data revision, when updating the data the revision MUST be checked and updated
}
// TableName for artifact
@ -49,27 +47,12 @@ func (a *Artifact) TableName() string {
return "artifact_2"
}
// Tag model in database
type Tag struct {
ID int64 `orm:"pk;auto;column(id)"`
RepositoryID int64 `orm:"column(repository_id)"` // tags are the resources of repository, one repository only contains one same name tag
ArtifactID int64 `orm:"column(artifact_id)"` // the artifact ID that the tag attaches to, it changes when pushing a same name but different digest artifact
Name string `orm:"column(name)"`
PushTime time.Time `orm:"column(push_time)"`
PullTime time.Time `orm:"column(pull_time)"`
Revision string `orm:"column(revision)"` // record data revision, when updating the data the revision MUST be checked and updated
}
// TableName for tag
func (t *Tag) TableName() string {
return "tag"
}
// ArtifactReference records the child artifact referenced by parent artifact
type ArtifactReference struct {
ID int64 `orm:"pk;auto;column(id)"`
ParentID int64 `orm:"column(parent_id)"`
ChildID int64 `orm:"column(child_id)"`
ID int64 `orm:"pk;auto;column(id)"`
ParentID int64 `orm:"column(parent_id)"`
ChildID int64 `orm:"column(child_id)"`
Platform string `orm:"column(platform)"` // json string
}
// TableName for artifact reference

View File

@ -0,0 +1,73 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package artifact
import (
"context"
"github.com/goharbor/harbor/src/pkg/q"
"time"
)
var (
// Mgr is a global artifact manager instance
Mgr = NewManager()
)
// Manager is the only interface of artifact module to provide the management functions for artifacts
type Manager interface {
// List artifacts according to the query, returns all artifacts if query is nil
List(ctx context.Context, query *q.Query) (total int64, artifacts []*Artifact, err error)
// Get the artifact specified by the ID
Get(ctx context.Context, id int64) (*Artifact, error)
// Create the artifact. If the artifact is an index, make sure all the artifacts it references
// already exist
Create(ctx context.Context, artifact *Artifact) (id int64, err error)
// Delete just deletes the artifact record. The underlying data of registry will be
// removed during garbage collection
Delete(ctx context.Context, id int64) error
// UpdatePullTime updates the pull time of the artifact
UpdatePullTime(ctx context.Context, artifactID int64, time time.Time) error
}
// NewManager returns an instance of the default manager
func NewManager() Manager {
return &manager{}
}
var _ Manager = &manager{}
type manager struct {
}
func (m *manager) List(ctx context.Context, query *q.Query) (total int64, artifacts []*Artifact, err error) {
// TODO implement
return 0, nil, nil
}
func (m *manager) Get(ctx context.Context, id int64) (*Artifact, error) {
// TODO implement
return nil, nil
}
func (m *manager) Create(ctx context.Context, artifact *Artifact) (id int64, err error) {
// TODO implement
return 0, nil
}
func (m *manager) Delete(ctx context.Context, id int64) error {
// TODO implement
return nil
}
func (m *manager) UpdatePullTime(ctx context.Context, artifactID int64, time time.Time) error {
// TODO implement
return nil
}

View File

@ -13,3 +13,5 @@
// limitations under the License.
package artifact
// TODO add test cases after implementing the manager

View File

@ -12,15 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package model
package artifact
import (
"encoding/json"
"time"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/artifact/manager/dao"
"github.com/goharbor/harbor/src/pkg/artifact/dao"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
)
@ -32,22 +31,15 @@ type Artifact struct {
Type string // image, chart, etc
MediaType string // the media type of artifact. Mostly, it's the value of `manifest.config.mediatype`
ManifestMediaType string // the media type of manifest/index
Repository *models.RepoRecord
Tags []*Tag // the list of tags that attached to the artifact
ProjectID int64
RepositoryID int64
Digest string
Size int64
PushTime time.Time
Platform *v1.Platform // when the parent of the artifact is an index, populate the platform information here
ExtraAttrs map[string]interface{} // only contains the simple attributes specific for the different artifact type, most of them should come from the config layer
SubResourceLinks map[string][]*ResourceLink // the resource link for build history(image), values.yaml(chart), dependency(chart), etc
PullTime time.Time
ExtraAttrs map[string]interface{} // only contains the simple attributes specific for the different artifact type, most of them should come from the config layer
Annotations map[string]string
References []int64 // child artifacts referenced by the parent artifact if the artifact is an index
Revision string // record data revision
// TODO: As the labels and signature aren't handled inside the artifact module,
// we should move it to the API level artifact model rather than
// keeping it here. The same to scan information
// Labels []*models.Label
// Signature *Signature // add the signature in the artifact level rather than tag level as we cannot make sure the signature always apply to tag
References []*Reference // child artifacts referenced by the parent artifact if the artifact is an index
}
// From converts the database level artifact to the business level object
@ -56,21 +48,14 @@ func (a *Artifact) From(art *dao.Artifact) {
a.Type = art.Type
a.MediaType = art.MediaType
a.ManifestMediaType = art.ManifestMediaType
a.Repository = &models.RepoRecord{
ProjectID: art.ProjectID,
RepositoryID: art.RepositoryID,
}
a.ProjectID = art.ProjectID
a.RepositoryID = art.RepositoryID
a.Digest = art.Digest
a.Size = art.Size
a.PushTime = art.PushTime
a.PullTime = art.PullTime
a.ExtraAttrs = map[string]interface{}{}
a.Annotations = map[string]string{}
a.Revision = art.Revision
if len(art.Platform) > 0 {
if err := json.Unmarshal([]byte(art.Platform), &a.Platform); err != nil {
log.Errorf("failed to unmarshal the platform of artifact %d: %v", art.ID, err)
}
}
if len(art.ExtraAttrs) > 0 {
if err := json.Unmarshal([]byte(art.ExtraAttrs), &a.ExtraAttrs); err != nil {
log.Errorf("failed to unmarshal the extra attrs of artifact %d: %v", art.ID, err)
@ -90,20 +75,12 @@ func (a *Artifact) To() *dao.Artifact {
Type: a.Type,
MediaType: a.MediaType,
ManifestMediaType: a.ManifestMediaType,
ProjectID: a.Repository.ProjectID,
RepositoryID: a.Repository.RepositoryID,
ProjectID: a.ProjectID,
RepositoryID: a.RepositoryID,
Digest: a.Digest,
Size: a.Size,
PushTime: a.PushTime,
Revision: a.Revision,
}
if a.Platform != nil {
platform, err := json.Marshal(a.Platform)
if err != nil {
log.Errorf("failed to marshal the platform of artifact %d: %v", a.ID, err)
}
art.Platform = string(platform)
PullTime: a.PullTime,
}
if len(a.ExtraAttrs) > 0 {
attrs, err := json.Marshal(a.ExtraAttrs)
@ -122,44 +99,36 @@ func (a *Artifact) To() *dao.Artifact {
return art
}
// ResourceLink is a link via that a resource can be fetched
type ResourceLink struct {
HREF string
Absolute bool // specify the href is an absolute URL or not
// Reference records the child artifact referenced by parent artifact
type Reference struct {
ParentID int64
ChildID int64
Platform *v1.Platform
}
// TODO: move it to the API level artifact model
// Signature information
// type Signature struct {
// Signatures map[string]bool // tag: signed or not
// }
// Tag belongs to one repository and can only be attached to a single
// one artifact under the repository
type Tag struct {
ID int64
Name string
PushTime time.Time
PullTime time.Time
Revision string // record data revision
}
// From converts the database level tag to the business level object
func (t *Tag) From(tag *dao.Tag) {
t.ID = tag.ID
t.Name = tag.Name
t.PushTime = tag.PushTime
t.PullTime = tag.PullTime
t.Revision = tag.Revision
}
// To converts the tag to the database level model
func (t *Tag) To() *dao.Tag {
return &dao.Tag{
ID: t.ID,
Name: t.Name,
PushTime: t.PushTime,
PullTime: t.PullTime,
Revision: t.Revision,
// From converts the data level reference to business level
func (r *Reference) From(ref *dao.ArtifactReference) {
r.ParentID = ref.ParentID
r.ChildID = ref.ChildID
if len(ref.Platform) > 0 {
if err := json.Unmarshal([]byte(ref.Platform), r); err != nil {
log.Errorf("failed to unmarshal the platform of reference: %v", err)
}
}
}
// To converts the reference to data level object
func (r *Reference) To() *dao.ArtifactReference {
ref := &dao.ArtifactReference{
ParentID: r.ParentID,
ChildID: r.ChildID,
}
if r.Platform != nil {
platform, err := json.Marshal(r.Platform)
if err != nil {
log.Errorf("failed to marshal the platform of reference: %v", err)
}
ref.Platform = string(platform)
}
return ref
}

View File

@ -12,15 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package model
package artifact
import (
"testing"
"time"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/artifact/manager/dao"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/goharbor/harbor/src/pkg/artifact/dao"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
@ -41,10 +39,9 @@ func (m *modelTestSuite) TestArtifactFrom() {
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
Size: 1024,
PushTime: time.Now(),
Platform: `{"architecture":"amd64"}`,
PullTime: time.Now(),
ExtraAttrs: `{"attr1":"value1"}`,
Annotations: `{"anno1":"value1"}`,
Revision: "1",
}
art := &Artifact{}
art.From(dbArt)
@ -52,91 +49,49 @@ func (m *modelTestSuite) TestArtifactFrom() {
assert.Equal(t, dbArt.Type, art.Type)
assert.Equal(t, dbArt.MediaType, art.MediaType)
assert.Equal(t, dbArt.ManifestMediaType, art.ManifestMediaType)
assert.Equal(t, dbArt.ProjectID, art.Repository.ProjectID)
assert.Equal(t, dbArt.RepositoryID, art.Repository.RepositoryID)
assert.Equal(t, dbArt.ProjectID, art.ProjectID)
assert.Equal(t, dbArt.RepositoryID, art.RepositoryID)
assert.Equal(t, dbArt.Digest, art.Digest)
assert.Equal(t, dbArt.Size, art.Size)
assert.Equal(t, dbArt.PushTime, art.PushTime)
assert.Equal(t, "amd64", art.Platform.Architecture)
assert.Equal(t, dbArt.PullTime, art.PullTime)
assert.Equal(t, "value1", art.ExtraAttrs["attr1"].(string))
assert.Equal(t, "value1", art.Annotations["anno1"])
assert.Equal(t, dbArt.Revision, art.Revision)
}
func (m *modelTestSuite) TestArtifactTo() {
t := m.T()
art := &Artifact{
ID: 1,
Type: "IMAGE",
Repository: &models.RepoRecord{
ProjectID: 1,
RepositoryID: 1,
},
ID: 1,
Type: "IMAGE",
ProjectID: 1,
RepositoryID: 1,
MediaType: "application/vnd.oci.image.config.v1+json",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
Size: 1024,
PushTime: time.Now(),
Platform: &v1.Platform{
Architecture: "amd64",
},
PullTime: time.Now(),
ExtraAttrs: map[string]interface{}{
"attr1": "value1",
},
Annotations: map[string]string{
"anno1": "value1",
},
Revision: "1",
}
dbArt := art.To()
assert.Equal(t, art.ID, dbArt.ID)
assert.Equal(t, art.Type, dbArt.Type)
assert.Equal(t, art.MediaType, dbArt.MediaType)
assert.Equal(t, art.ManifestMediaType, dbArt.ManifestMediaType)
assert.Equal(t, art.Repository.ProjectID, dbArt.ProjectID)
assert.Equal(t, art.Repository.RepositoryID, dbArt.RepositoryID)
assert.Equal(t, art.ProjectID, dbArt.ProjectID)
assert.Equal(t, art.RepositoryID, dbArt.RepositoryID)
assert.Equal(t, art.Digest, dbArt.Digest)
assert.Equal(t, art.Size, dbArt.Size)
assert.Equal(t, art.PushTime, dbArt.PushTime)
assert.Equal(t, `{"architecture":"amd64","os":""}`, dbArt.Platform)
assert.Equal(t, art.PullTime, dbArt.PullTime)
assert.Equal(t, `{"attr1":"value1"}`, dbArt.ExtraAttrs)
assert.Equal(t, `{"anno1":"value1"}`, dbArt.Annotations)
assert.Equal(t, art.Revision, dbArt.Revision)
}
func (m *modelTestSuite) TestTagFrom() {
t := m.T()
dbTag := &dao.Tag{
ID: 1,
Name: "1.0",
PushTime: time.Now(),
PullTime: time.Now(),
Revision: "1",
}
tag := &Tag{}
tag.From(dbTag)
assert.Equal(t, dbTag.ID, tag.ID)
assert.Equal(t, dbTag.Name, tag.Name)
assert.Equal(t, dbTag.PushTime, tag.PushTime)
assert.Equal(t, dbTag.PullTime, tag.PullTime)
assert.Equal(t, dbTag.Revision, tag.Revision)
}
func (m *modelTestSuite) TestTagTo() {
t := m.T()
tag := &Tag{
ID: 1,
Name: "1.0",
PushTime: time.Now(),
PullTime: time.Now(),
Revision: "1",
}
dbTag := tag.To()
assert.Equal(t, tag.ID, dbTag.ID)
assert.Equal(t, tag.Name, dbTag.Name)
assert.Equal(t, tag.PushTime, dbTag.PushTime)
assert.Equal(t, tag.PullTime, dbTag.PullTime)
assert.Equal(t, tag.Revision, dbTag.Revision)
}
func TestModel(t *testing.T) {

View File

@ -12,4 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package manager
package dao
import (
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
)
func init() {
orm.RegisterModel(&tag.Tag{})
}

46
src/pkg/tag/manager.go Normal file
View File

@ -0,0 +1,46 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tag
import (
"context"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/pkg/tag/model/tag"
)
var (
// Mgr is a global instance of tag manager
Mgr = NewManager()
)
// Manager manages the tags
type Manager interface {
// List tags according to the query
List(ctx context.Context, query *q.Query) (total int64, tags []*tag.Tag, err error)
// Get the tag specified by ID
Get(ctx context.Context, id int64) (tag *tag.Tag, err error)
// Create the tag and returns the ID
Create(ctx context.Context, tag *tag.Tag) (id int64, err error)
// Update the tag
Update(ctx context.Context, tag *tag.Tag) (err error)
// Delete the tag specified by ID
Delete(ctx context.Context, id int64) (err error)
}
// NewManager creates an instance of the default tag manager
func NewManager() Manager {
// TODO implement
return nil
}

View File

@ -0,0 +1,29 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tag
import (
"time"
)
// Tag model in database
type Tag struct {
ID int64 `orm:"pk;auto;column(id)"`
RepositoryID int64 `orm:"column(repository_id)"` // tags are the resources of repository, one repository only contains one same name tag
ArtifactID int64 `orm:"column(artifact_id)"` // the artifact ID that the tag attaches to, it changes when pushing a same name but different digest artifact
Name string `orm:"column(name)"`
PushTime time.Time `orm:"column(push_time)"`
PullTime time.Time `orm:"column(pull_time)"`
}