Create the models for OCI supporting

This commits does some basic init work for supporting OCI:
1. Create the artifact and tag model
2. Create database tables

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2019-12-04 16:55:41 +08:00
parent 9f359b991f
commit 7b41c900a7
7 changed files with 460 additions and 1 deletions

View File

@ -0,0 +1,42 @@
/* TODO remove the table artifact_2 and use the artifact instead after finishing the upgrade work */
CREATE TABLE artifact_2
(
id SERIAL PRIMARY KEY NOT NULL,
/* image, chart, etc */
type varchar(255),
media_type varchar(255),
manifest_media_type varchar(255),
project_id int NOT NULL,
repository_id int NOT NULL,
digest varchar(255) NOT NULL,
size bigint,
push_time timestamp default CURRENT_TIMESTAMP,
platform varchar(255),
extra_attrs text,
annotations jsonb,
/* when updating the data the revision MUST be checked and updated */
revision varchar(64) NOT NULL,
CONSTRAINT unique_artifact_2 UNIQUE (repository_id, digest)
);
CREATE TABLE tag
(
id SERIAL PRIMARY KEY NOT NULL,
repository_id int NOT NULL,
artifact_id int NOT NULL,
name varchar(255),
push_time timestamp default CURRENT_TIMESTAMP,
pull_time timestamp,
/* when updating the data the revision MUST be checked and updated */
revision varchar(64) NOT NULL,
CONSTRAINT unique_tag UNIQUE (repository_id, name)
);
/* artifact_reference records the child artifact referenced by parent artifact */
CREATE TABLE artifact_reference
(
id SERIAL PRIMARY KEY NOT NULL,
parent_id int NOT NULL,
child_id int NOT NULL,
CONSTRAINT unique_reference UNIQUE (parent_id, child_id)
);

View File

@ -59,7 +59,7 @@ require (
github.com/miekg/pkcs11 v0.0.0-20170220202408-7283ca79f35e // indirect
github.com/olekukonko/tablewriter v0.0.1
github.com/opencontainers/go-digest v1.0.0-rc0
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/opencontainers/image-spec v1.0.1
github.com/opentracing/opentracing-go v1.1.0 // indirect
github.com/pkg/errors v0.8.1
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect

View File

@ -0,0 +1,15 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package artifact

View File

@ -0,0 +1,78 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"time"
"github.com/astaxie/beego/orm"
)
func init() {
orm.RegisterModel(&Artifact{})
orm.RegisterModel(&Tag{})
orm.RegisterModel(&ArtifactReference{})
}
// Artifact model in database
type Artifact struct {
ID int64 `orm:"pk;auto;column(id)"`
Type string `orm:"column(type)"` // image or chart
MediaType string `orm:"column(media_type)"` // the media type of artifact
ManifestMediaType string `orm:"column(manifest_media_type)"` // the media type of manifest/index
ProjectID int64 `orm:"column(project_id)"` // needed for quota
RepositoryID int64 `orm:"column(repository_id)"`
Digest string `orm:"column(digest)"`
Size int64 `orm:"column(size)"`
PushTime time.Time `orm:"column(push_time)"`
Platform string `orm:"column(platform)"` // json string
ExtraAttrs string `orm:"column(extra_attrs)"` // json string
Annotations string `orm:"column(annotations);type(jsonb)"` // json string
Revision string `orm:"column(revision)"` // record data revision, when updating the data the revision MUST be checked and updated
}
// TableName for artifact
func (a *Artifact) TableName() string {
// TODO use "artifact" after finishing the upgrade/migration work
return "artifact_2"
}
// Tag model in database
type Tag struct {
ID int64 `orm:"pk;auto;column(id)"`
RepositoryID int64 `orm:"column(repository_id)"` // tags are the resources of repository, one repository only contains one same name tag
ArtifactID int64 `orm:"column(artifact_id)"` // the artifact ID that the tag attaches to, it changes when pushing a same name but different digest artifact
Name string `orm:"column(name)"`
PushTime time.Time `orm:"column(push_time)"`
PullTime time.Time `orm:"column(pull_time)"`
Revision string `orm:"column(revision)"` // record data revision, when updating the data the revision MUST be checked and updated
}
// TableName for tag
func (t *Tag) TableName() string {
return "tag"
}
// ArtifactReference records the child artifact referenced by parent artifact
type ArtifactReference struct {
ID int64 `orm:"pk;auto;column(id)"`
ParentID int64 `orm:"column(parent_id)"`
ChildID int64 `orm:"column(child_id)"`
}
// TableName for artifact reference
func (a *ArtifactReference) TableName() string {
return "artifact_reference"
}

View File

@ -0,0 +1,15 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package manager

View File

@ -0,0 +1,165 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"encoding/json"
"time"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/artifact/manager/dao"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
)
// Artifact is the abstract object managed by Harbor. It hides the
// underlying concrete detail and provides an unified artifact view
// for all users.
type Artifact struct {
ID int64
Type string // image, chart, etc
MediaType string // the media type of artifact. Mostly, it's the value of `manifest.config.mediatype`
ManifestMediaType string // the media type of manifest/index
Repository *models.RepoRecord
Tags []*Tag // the list of tags that attached to the artifact
Digest string
Size int64
PushTime time.Time
Platform *v1.Platform // when the parent of the artifact is an index, populate the platform information here
ExtraAttrs map[string]interface{} // only contains the simple attributes specific for the different artifact type, most of them should come from the config layer
SubResourceLinks map[string][]*ResourceLink // the resource link for build history(image), values.yaml(chart), dependency(chart), etc
Annotations map[string]string
References []int64 // child artifacts referenced by the parent artifact if the artifact is an index
Revision string // record data revision
// TODO: As the labels and signature aren't handled inside the artifact module,
// we should move it to the API level artifact model rather than
// keeping it here. The same to scan information
// Labels []*models.Label
// Signature *Signature // add the signature in the artifact level rather than tag level as we cannot make sure the signature always apply to tag
}
// From converts the database level artifact to the business level object
func (a *Artifact) From(art *dao.Artifact) {
a.ID = art.ID
a.Type = art.Type
a.MediaType = art.MediaType
a.ManifestMediaType = art.ManifestMediaType
a.Repository = &models.RepoRecord{
ProjectID: art.ProjectID,
RepositoryID: art.RepositoryID,
}
a.Digest = art.Digest
a.Size = art.Size
a.PushTime = art.PushTime
a.ExtraAttrs = map[string]interface{}{}
a.Annotations = map[string]string{}
a.Revision = art.Revision
if len(art.Platform) > 0 {
if err := json.Unmarshal([]byte(art.Platform), &a.Platform); err != nil {
log.Errorf("failed to unmarshal the platform of artifact %d: %v", art.ID, err)
}
}
if len(art.ExtraAttrs) > 0 {
if err := json.Unmarshal([]byte(art.ExtraAttrs), &a.ExtraAttrs); err != nil {
log.Errorf("failed to unmarshal the extra attrs of artifact %d: %v", art.ID, err)
}
}
if len(art.Annotations) > 0 {
if err := json.Unmarshal([]byte(art.Annotations), &a.Annotations); err != nil {
log.Errorf("failed to unmarshal the annotations of artifact %d: %v", art.ID, err)
}
}
}
// To converts the artifact to the database level object
func (a *Artifact) To() *dao.Artifact {
art := &dao.Artifact{
ID: a.ID,
Type: a.Type,
MediaType: a.MediaType,
ManifestMediaType: a.ManifestMediaType,
ProjectID: a.Repository.ProjectID,
RepositoryID: a.Repository.RepositoryID,
Digest: a.Digest,
Size: a.Size,
PushTime: a.PushTime,
Revision: a.Revision,
}
if a.Platform != nil {
platform, err := json.Marshal(a.Platform)
if err != nil {
log.Errorf("failed to marshal the platform of artifact %d: %v", a.ID, err)
}
art.Platform = string(platform)
}
if len(a.ExtraAttrs) > 0 {
attrs, err := json.Marshal(a.ExtraAttrs)
if err != nil {
log.Errorf("failed to marshal the extra attrs of artifact %d: %v", a.ID, err)
}
art.ExtraAttrs = string(attrs)
}
if len(a.Annotations) > 0 {
annotations, err := json.Marshal(a.Annotations)
if err != nil {
log.Errorf("failed to marshal the annotations of artifact %d: %v", a.ID, err)
}
art.Annotations = string(annotations)
}
return art
}
// ResourceLink is a link via that a resource can be fetched
type ResourceLink struct {
HREF string
Absolute bool // specify the href is an absolute URL or not
}
// TODO: move it to the API level artifact model
// Signature information
// type Signature struct {
// Signatures map[string]bool // tag: signed or not
// }
// Tag belongs to one repository and can only be attached to a single
// one artifact under the repository
type Tag struct {
ID int64
Name string
PushTime time.Time
PullTime time.Time
Revision string // record data revision
}
// From converts the database level tag to the business level object
func (t *Tag) From(tag *dao.Tag) {
t.ID = tag.ID
t.Name = tag.Name
t.PushTime = tag.PushTime
t.PullTime = tag.PullTime
t.Revision = tag.Revision
}
// To converts the tag to the database level model
func (t *Tag) To() *dao.Tag {
return &dao.Tag{
ID: t.ID,
Name: t.Name,
PushTime: t.PushTime,
PullTime: t.PullTime,
Revision: t.Revision,
}
}

View File

@ -0,0 +1,144 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
import (
"testing"
"time"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/pkg/artifact/manager/dao"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type modelTestSuite struct {
suite.Suite
}
func (m *modelTestSuite) TestArtifactFrom() {
t := m.T()
dbArt := &dao.Artifact{
ID: 1,
Type: "IMAGE",
MediaType: "application/vnd.oci.image.config.v1+json",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
ProjectID: 1,
RepositoryID: 1,
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
Size: 1024,
PushTime: time.Now(),
Platform: `{"architecture":"amd64"}`,
ExtraAttrs: `{"attr1":"value1"}`,
Annotations: `{"anno1":"value1"}`,
Revision: "1",
}
art := &Artifact{}
art.From(dbArt)
assert.Equal(t, dbArt.ID, art.ID)
assert.Equal(t, dbArt.Type, art.Type)
assert.Equal(t, dbArt.MediaType, art.MediaType)
assert.Equal(t, dbArt.ManifestMediaType, art.ManifestMediaType)
assert.Equal(t, dbArt.ProjectID, art.Repository.ProjectID)
assert.Equal(t, dbArt.RepositoryID, art.Repository.RepositoryID)
assert.Equal(t, dbArt.Digest, art.Digest)
assert.Equal(t, dbArt.Size, art.Size)
assert.Equal(t, dbArt.PushTime, art.PushTime)
assert.Equal(t, "amd64", art.Platform.Architecture)
assert.Equal(t, "value1", art.ExtraAttrs["attr1"].(string))
assert.Equal(t, "value1", art.Annotations["anno1"])
assert.Equal(t, dbArt.Revision, art.Revision)
}
func (m *modelTestSuite) TestArtifactTo() {
t := m.T()
art := &Artifact{
ID: 1,
Type: "IMAGE",
Repository: &models.RepoRecord{
ProjectID: 1,
RepositoryID: 1,
},
MediaType: "application/vnd.oci.image.config.v1+json",
ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
Size: 1024,
PushTime: time.Now(),
Platform: &v1.Platform{
Architecture: "amd64",
},
ExtraAttrs: map[string]interface{}{
"attr1": "value1",
},
Annotations: map[string]string{
"anno1": "value1",
},
Revision: "1",
}
dbArt := art.To()
assert.Equal(t, art.ID, dbArt.ID)
assert.Equal(t, art.Type, dbArt.Type)
assert.Equal(t, art.MediaType, dbArt.MediaType)
assert.Equal(t, art.ManifestMediaType, dbArt.ManifestMediaType)
assert.Equal(t, art.Repository.ProjectID, dbArt.ProjectID)
assert.Equal(t, art.Repository.RepositoryID, dbArt.RepositoryID)
assert.Equal(t, art.Digest, dbArt.Digest)
assert.Equal(t, art.Size, dbArt.Size)
assert.Equal(t, art.PushTime, dbArt.PushTime)
assert.Equal(t, `{"architecture":"amd64","os":""}`, dbArt.Platform)
assert.Equal(t, `{"attr1":"value1"}`, dbArt.ExtraAttrs)
assert.Equal(t, `{"anno1":"value1"}`, dbArt.Annotations)
assert.Equal(t, art.Revision, dbArt.Revision)
}
func (m *modelTestSuite) TestTagFrom() {
t := m.T()
dbTag := &dao.Tag{
ID: 1,
Name: "1.0",
PushTime: time.Now(),
PullTime: time.Now(),
Revision: "1",
}
tag := &Tag{}
tag.From(dbTag)
assert.Equal(t, dbTag.ID, tag.ID)
assert.Equal(t, dbTag.Name, tag.Name)
assert.Equal(t, dbTag.PushTime, tag.PushTime)
assert.Equal(t, dbTag.PullTime, tag.PullTime)
assert.Equal(t, dbTag.Revision, tag.Revision)
}
func (m *modelTestSuite) TestTagTo() {
t := m.T()
tag := &Tag{
ID: 1,
Name: "1.0",
PushTime: time.Now(),
PullTime: time.Now(),
Revision: "1",
}
dbTag := tag.To()
assert.Equal(t, tag.ID, dbTag.ID)
assert.Equal(t, tag.Name, dbTag.Name)
assert.Equal(t, tag.PushTime, dbTag.PushTime)
assert.Equal(t, tag.PullTime, dbTag.PullTime)
assert.Equal(t, tag.Revision, dbTag.Revision)
}
func TestModel(t *testing.T) {
suite.Run(t, &modelTestSuite{})
}