Merge pull request #10138 from ywk253100/191204_artifact

Create the models for OCI supporting

make/migrations/postgresql/0030_1.11.0_schema.up.sql

@@ -0,0 +1,42 @@
+/* TODO remove the table artifact_2 and use the artifact instead after finishing the upgrade work */
+CREATE TABLE artifact_2
+(
+  id            SERIAL PRIMARY KEY NOT NULL,
+  /* image, chart, etc */
+  type          varchar(255),
+  media_type    varchar(255),
+  manifest_media_type varchar(255),
+  project_id    int NOT NULL,
+  repository_id int NOT NULL,
+  digest        varchar(255) NOT NULL,
+  size          bigint,  
+  push_time     timestamp default CURRENT_TIMESTAMP,
+  platform      varchar(255),
+  extra_attrs   text,
+  annotations   jsonb,
+  /* when updating the data the revision MUST be checked and updated */
+  revision      varchar(64) NOT NULL,
+  CONSTRAINT unique_artifact_2 UNIQUE (repository_id, digest)
+);
+
+CREATE TABLE tag
+(
+  id            SERIAL PRIMARY KEY NOT NULL,
+  repository_id int NOT NULL,
+  artifact_id   int NOT NULL,
+  name          varchar(255),
+  push_time     timestamp default CURRENT_TIMESTAMP,
+  pull_time     timestamp,
+  /* when updating the data the revision MUST be checked and updated */
+  revision      varchar(64) NOT NULL,
+  CONSTRAINT unique_tag UNIQUE (repository_id, name)
+);
+
+/* artifact_reference records the child artifact referenced by parent artifact */
+CREATE TABLE artifact_reference
+(
+  id            SERIAL PRIMARY KEY NOT NULL,
+  parent_id   int NOT NULL,
+  child_id  int NOT NULL,
+  CONSTRAINT unique_reference UNIQUE (parent_id, child_id)
+);

src/go.mod

@@ -59,7 +59,7 @@ require (
 	github.com/miekg/pkcs11 v0.0.0-20170220202408-7283ca79f35e // indirect
 	github.com/olekukonko/tablewriter v0.0.1
 	github.com/opencontainers/go-digest v1.0.0-rc0
-	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/image-spec v1.0.1
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect

src/pkg/artifact/controller.go

@@ -0,0 +1,15 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package artifact

src/pkg/artifact/manager/dao/model.go

@@ -0,0 +1,78 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"time"
+
+	"github.com/astaxie/beego/orm"
+)
+
+func init() {
+	orm.RegisterModel(&Artifact{})
+	orm.RegisterModel(&Tag{})
+	orm.RegisterModel(&ArtifactReference{})
+}
+
+// Artifact model in database
+type Artifact struct {
+	ID                int64     `orm:"pk;auto;column(id)"`
+	Type              string    `orm:"column(type)"`                // image or chart
+	MediaType         string    `orm:"column(media_type)"`          // the media type of artifact
+	ManifestMediaType string    `orm:"column(manifest_media_type)"` // the media type of manifest/index
+	ProjectID         int64     `orm:"column(project_id)"`          // needed for quota
+	RepositoryID      int64     `orm:"column(repository_id)"`
+	Digest            string    `orm:"column(digest)"`
+	Size              int64     `orm:"column(size)"`
+	PushTime          time.Time `orm:"column(push_time)"`
+	Platform          string    `orm:"column(platform)"`                // json string
+	ExtraAttrs        string    `orm:"column(extra_attrs)"`             // json string
+	Annotations       string    `orm:"column(annotations);type(jsonb)"` // json string
+	Revision          string    `orm:"column(revision)"`                // record data revision, when updating the data the revision MUST be checked and updated
+}
+
+// TableName for artifact
+func (a *Artifact) TableName() string {
+	// TODO use "artifact" after finishing the upgrade/migration work
+	return "artifact_2"
+}
+
+// Tag model in database
+type Tag struct {
+	ID           int64     `orm:"pk;auto;column(id)"`
+	RepositoryID int64     `orm:"column(repository_id)"` // tags are the resources of repository, one repository only contains one same name tag
+	ArtifactID   int64     `orm:"column(artifact_id)"`   // the artifact ID that the tag attaches to, it changes when pushing a same name but different digest artifact
+	Name         string    `orm:"column(name)"`
+	PushTime     time.Time `orm:"column(push_time)"`
+	PullTime     time.Time `orm:"column(pull_time)"`
+	Revision     string    `orm:"column(revision)"` // record data revision, when updating the data the revision MUST be checked and updated
+}
+
+// TableName for tag
+func (t *Tag) TableName() string {
+	return "tag"
+}
+
+// ArtifactReference records the child artifact referenced by parent artifact
+type ArtifactReference struct {
+	ID       int64 `orm:"pk;auto;column(id)"`
+	ParentID int64 `orm:"column(parent_id)"`
+	ChildID  int64 `orm:"column(child_id)"`
+}
+
+// TableName for artifact reference
+func (a *ArtifactReference) TableName() string {
+	return "artifact_reference"
+}

src/pkg/artifact/manager/manager.go

@@ -0,0 +1,15 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package manager

src/pkg/artifact/model/model.go

@@ -0,0 +1,165 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package model
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/pkg/artifact/manager/dao"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Artifact is the abstract object managed by Harbor. It hides the
+// underlying concrete detail and provides an unified artifact view
+// for all users.
+type Artifact struct {
+	ID                int64
+	Type              string // image, chart, etc
+	MediaType         string // the media type of artifact. Mostly, it's the value of `manifest.config.mediatype`
+	ManifestMediaType string // the media type of manifest/index
+	Repository        *models.RepoRecord
+	Tags              []*Tag // the list of tags that attached to the artifact
+	Digest            string
+	Size              int64
+	PushTime          time.Time
+	Platform          *v1.Platform               // when the parent of the artifact is an index, populate the platform information here
+	ExtraAttrs        map[string]interface{}     // only contains the simple attributes specific for the different artifact type, most of them should come from the config layer
+	SubResourceLinks  map[string][]*ResourceLink // the resource link for build history(image), values.yaml(chart), dependency(chart), etc
+	Annotations       map[string]string
+	References        []int64 // child artifacts referenced by the parent artifact if the artifact is an index
+	Revision          string  // record data revision
+	// TODO: As the labels and signature aren't handled inside the artifact module,
+	// we should move it to the API level artifact model rather than
+	// keeping it here. The same to scan information
+	// Labels                  []*models.Label
+	// Signature               *Signature                 // add the signature in the artifact level rather than tag level as we cannot make sure the signature always apply to tag
+}
+
+// From converts the database level artifact to the business level object
+func (a *Artifact) From(art *dao.Artifact) {
+	a.ID = art.ID
+	a.Type = art.Type
+	a.MediaType = art.MediaType
+	a.ManifestMediaType = art.ManifestMediaType
+	a.Repository = &models.RepoRecord{
+		ProjectID:    art.ProjectID,
+		RepositoryID: art.RepositoryID,
+	}
+	a.Digest = art.Digest
+	a.Size = art.Size
+	a.PushTime = art.PushTime
+	a.ExtraAttrs = map[string]interface{}{}
+	a.Annotations = map[string]string{}
+	a.Revision = art.Revision
+	if len(art.Platform) > 0 {
+		if err := json.Unmarshal([]byte(art.Platform), &a.Platform); err != nil {
+			log.Errorf("failed to unmarshal the platform of artifact %d: %v", art.ID, err)
+		}
+	}
+	if len(art.ExtraAttrs) > 0 {
+		if err := json.Unmarshal([]byte(art.ExtraAttrs), &a.ExtraAttrs); err != nil {
+			log.Errorf("failed to unmarshal the extra attrs of artifact %d: %v", art.ID, err)
+		}
+	}
+	if len(art.Annotations) > 0 {
+		if err := json.Unmarshal([]byte(art.Annotations), &a.Annotations); err != nil {
+			log.Errorf("failed to unmarshal the annotations of artifact %d: %v", art.ID, err)
+		}
+	}
+}
+
+// To converts the artifact to the database level object
+func (a *Artifact) To() *dao.Artifact {
+	art := &dao.Artifact{
+		ID:                a.ID,
+		Type:              a.Type,
+		MediaType:         a.MediaType,
+		ManifestMediaType: a.ManifestMediaType,
+		ProjectID:         a.Repository.ProjectID,
+		RepositoryID:      a.Repository.RepositoryID,
+		Digest:            a.Digest,
+		Size:              a.Size,
+		PushTime:          a.PushTime,
+		Revision:          a.Revision,
+	}
+
+	if a.Platform != nil {
+		platform, err := json.Marshal(a.Platform)
+		if err != nil {
+			log.Errorf("failed to marshal the platform of artifact %d: %v", a.ID, err)
+		}
+		art.Platform = string(platform)
+	}
+	if len(a.ExtraAttrs) > 0 {
+		attrs, err := json.Marshal(a.ExtraAttrs)
+		if err != nil {
+			log.Errorf("failed to marshal the extra attrs of artifact %d: %v", a.ID, err)
+		}
+		art.ExtraAttrs = string(attrs)
+	}
+	if len(a.Annotations) > 0 {
+		annotations, err := json.Marshal(a.Annotations)
+		if err != nil {
+			log.Errorf("failed to marshal the annotations of artifact %d: %v", a.ID, err)
+		}
+		art.Annotations = string(annotations)
+	}
+	return art
+}
+
+// ResourceLink is a link via that a resource can be fetched
+type ResourceLink struct {
+	HREF     string
+	Absolute bool // specify the href is an absolute URL or not
+}
+
+// TODO: move it to the API level artifact model
+// Signature information
+// type Signature struct {
+// 	Signatures map[string]bool // tag: signed or not
+// }
+
+// Tag belongs to one repository and can only be attached to a single
+// one artifact under the repository
+type Tag struct {
+	ID       int64
+	Name     string
+	PushTime time.Time
+	PullTime time.Time
+	Revision string // record data revision
+}
+
+// From converts the database level tag to the business level object
+func (t *Tag) From(tag *dao.Tag) {
+	t.ID = tag.ID
+	t.Name = tag.Name
+	t.PushTime = tag.PushTime
+	t.PullTime = tag.PullTime
+	t.Revision = tag.Revision
+}
+
+// To converts the tag to the database level model
+func (t *Tag) To() *dao.Tag {
+	return &dao.Tag{
+		ID:       t.ID,
+		Name:     t.Name,
+		PushTime: t.PushTime,
+		PullTime: t.PullTime,
+		Revision: t.Revision,
+	}
+}

src/pkg/artifact/model/model_test.go

@@ -0,0 +1,144 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package model
+
+import (
+	"testing"
+	"time"
+
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/pkg/artifact/manager/dao"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+type modelTestSuite struct {
+	suite.Suite
+}
+
+func (m *modelTestSuite) TestArtifactFrom() {
+	t := m.T()
+	dbArt := &dao.Artifact{
+		ID:                1,
+		Type:              "IMAGE",
+		MediaType:         "application/vnd.oci.image.config.v1+json",
+		ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
+		ProjectID:         1,
+		RepositoryID:      1,
+		Digest:            "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
+		Size:              1024,
+		PushTime:          time.Now(),
+		Platform:          `{"architecture":"amd64"}`,
+		ExtraAttrs:        `{"attr1":"value1"}`,
+		Annotations:       `{"anno1":"value1"}`,
+		Revision:          "1",
+	}
+	art := &Artifact{}
+	art.From(dbArt)
+	assert.Equal(t, dbArt.ID, art.ID)
+	assert.Equal(t, dbArt.Type, art.Type)
+	assert.Equal(t, dbArt.MediaType, art.MediaType)
+	assert.Equal(t, dbArt.ManifestMediaType, art.ManifestMediaType)
+	assert.Equal(t, dbArt.ProjectID, art.Repository.ProjectID)
+	assert.Equal(t, dbArt.RepositoryID, art.Repository.RepositoryID)
+	assert.Equal(t, dbArt.Digest, art.Digest)
+	assert.Equal(t, dbArt.Size, art.Size)
+	assert.Equal(t, dbArt.PushTime, art.PushTime)
+	assert.Equal(t, "amd64", art.Platform.Architecture)
+	assert.Equal(t, "value1", art.ExtraAttrs["attr1"].(string))
+	assert.Equal(t, "value1", art.Annotations["anno1"])
+	assert.Equal(t, dbArt.Revision, art.Revision)
+}
+
+func (m *modelTestSuite) TestArtifactTo() {
+	t := m.T()
+	art := &Artifact{
+		ID:   1,
+		Type: "IMAGE",
+		Repository: &models.RepoRecord{
+			ProjectID:    1,
+			RepositoryID: 1,
+		},
+		MediaType:         "application/vnd.oci.image.config.v1+json",
+		ManifestMediaType: "application/vnd.oci.image.manifest.v1+json",
+		Digest:            "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180",
+		Size:              1024,
+		PushTime:          time.Now(),
+		Platform: &v1.Platform{
+			Architecture: "amd64",
+		},
+		ExtraAttrs: map[string]interface{}{
+			"attr1": "value1",
+		},
+		Annotations: map[string]string{
+			"anno1": "value1",
+		},
+		Revision: "1",
+	}
+	dbArt := art.To()
+	assert.Equal(t, art.ID, dbArt.ID)
+	assert.Equal(t, art.Type, dbArt.Type)
+	assert.Equal(t, art.MediaType, dbArt.MediaType)
+	assert.Equal(t, art.ManifestMediaType, dbArt.ManifestMediaType)
+	assert.Equal(t, art.Repository.ProjectID, dbArt.ProjectID)
+	assert.Equal(t, art.Repository.RepositoryID, dbArt.RepositoryID)
+	assert.Equal(t, art.Digest, dbArt.Digest)
+	assert.Equal(t, art.Size, dbArt.Size)
+	assert.Equal(t, art.PushTime, dbArt.PushTime)
+	assert.Equal(t, `{"architecture":"amd64","os":""}`, dbArt.Platform)
+	assert.Equal(t, `{"attr1":"value1"}`, dbArt.ExtraAttrs)
+	assert.Equal(t, `{"anno1":"value1"}`, dbArt.Annotations)
+	assert.Equal(t, art.Revision, dbArt.Revision)
+}
+
+func (m *modelTestSuite) TestTagFrom() {
+	t := m.T()
+	dbTag := &dao.Tag{
+		ID:       1,
+		Name:     "1.0",
+		PushTime: time.Now(),
+		PullTime: time.Now(),
+		Revision: "1",
+	}
+	tag := &Tag{}
+	tag.From(dbTag)
+	assert.Equal(t, dbTag.ID, tag.ID)
+	assert.Equal(t, dbTag.Name, tag.Name)
+	assert.Equal(t, dbTag.PushTime, tag.PushTime)
+	assert.Equal(t, dbTag.PullTime, tag.PullTime)
+	assert.Equal(t, dbTag.Revision, tag.Revision)
+}
+
+func (m *modelTestSuite) TestTagTo() {
+	t := m.T()
+	tag := &Tag{
+		ID:       1,
+		Name:     "1.0",
+		PushTime: time.Now(),
+		PullTime: time.Now(),
+		Revision: "1",
+	}
+	dbTag := tag.To()
+	assert.Equal(t, tag.ID, dbTag.ID)
+	assert.Equal(t, tag.Name, dbTag.Name)
+	assert.Equal(t, tag.PushTime, dbTag.PushTime)
+	assert.Equal(t, tag.PullTime, dbTag.PullTime)
+	assert.Equal(t, tag.Revision, dbTag.Revision)
+}
+
+func TestModel(t *testing.T) {
+	suite.Run(t, &modelTestSuite{})
+}