Merge pull request #8230 from wy65701436/dao-artifact

add dao of artifact

make/migrations/postgresql/0010_1.9.0_schema.up.sql

@@ -9,6 +9,46 @@ CREATE TABLE cve_whitelist (
     UNIQUE (project_id)
 );
 
+CREATE TABLE blob (
+ id SERIAL PRIMARY KEY NOT NULL,
+ /*
+    digest of config, layer, manifest
+ */
+ digest varchar(255) NOT NULL,
+ content_type varchar(255) NOT NULL,
+ size int NOT NULL,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ UNIQUE (digest)
+);
+
+CREATE TABLE artifact (
+ id SERIAL PRIMARY KEY NOT NULL,
+ project_id int NOT NULL,
+ repo varchar(255) NOT NULL,
+ tag varchar(255) NOT NULL,
+ /*
+    digest of manifest
+ */
+ digest varchar(255) NOT NULL,
+ /*
+    kind of artifact, image, chart, etc..
+ */
+ kind varchar(255) NOT NULL,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ pull_time timestamp,
+ push_time timestamp,
+ CONSTRAINT unique_artifact UNIQUE (project_id, repo, tag)
+);
+
+/* add the table for relation of artifact and blob */
+CREATE TABLE artifact_blob (
+ id SERIAL PRIMARY KEY NOT NULL,
+ digest_af varchar(255) NOT NULL,
+ digest_blob varchar(255) NOT NULL,
+ creation_time timestamp default CURRENT_TIMESTAMP,
+ CONSTRAINT unique_artifact_blob UNIQUE (digest_af, digest_blob)
+);
+
 /* add quota table */
 CREATE TABLE quota (
  id SERIAL PRIMARY KEY NOT NULL,

src/common/dao/artifact.go

@@ -0,0 +1,60 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"github.com/goharbor/harbor/src/common/models"
+	"strings"
+	"time"
+)
+
+// AddArtifact ...
+func AddArtifact(af *models.Artifact) (int64, error) {
+	now := time.Now()
+	af.CreationTime = now
+	id, err := GetOrmer().Insert(af)
+	if err != nil {
+		if strings.Contains(err.Error(), "duplicate key value violates unique constraint") {
+			return 0, ErrDupRows
+		}
+		return 0, err
+	}
+	return id, nil
+}
+
+// DeleteArtifact ...
+func DeleteArtifact(id int64) error {
+	_, err := GetOrmer().QueryTable(&models.Artifact{}).Filter("ID", id).Delete()
+	return err
+}
+
+// DeleteArtifactByDigest ...
+func DeleteArtifactByDigest(digest string) error {
+	_, err := GetOrmer().Raw(`delete from artifact where digest = ? `, digest).Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// DeleteByTag ...
+func DeleteByTag(projectID int, repo, tag string) error {
+	_, err := GetOrmer().Raw(`delete from artifact where project_id = ? and repo = ? and tag = ? `,
+		projectID, repo, tag).Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}

src/common/dao/artifact_blob.go

@@ -0,0 +1,107 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"fmt"
+	"github.com/astaxie/beego/orm"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/pkg/errors"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// AddArtifactNBlob ...
+func AddArtifactNBlob(afnb *models.ArtifactAndBlob) (int64, error) {
+	now := time.Now()
+	afnb.CreationTime = now
+	id, err := GetOrmer().Insert(afnb)
+	if err != nil {
+		if strings.Contains(err.Error(), "duplicate key value violates unique constraint") {
+			return 0, ErrDupRows
+		}
+		return 0, err
+	}
+	return id, nil
+}
+
+// AddArtifactNBlobs ...
+func AddArtifactNBlobs(afnbs []*models.ArtifactAndBlob) error {
+	o := orm.NewOrm()
+	err := o.Begin()
+	if err != nil {
+		return err
+	}
+
+	var errInsertMultiple error
+	total := len(afnbs)
+	successNums, err := o.InsertMulti(total, afnbs)
+	if err != nil {
+		errInsertMultiple = err
+		err := o.Rollback()
+		if err != nil {
+			log.Errorf("fail to rollback when to insert multiple artifact and blobs, %v", err)
+			errInsertMultiple = errors.Wrap(errInsertMultiple, err.Error())
+		}
+		return errInsertMultiple
+	}
+
+	// part of them cannot be inserted successfully.
+	if successNums != int64(total) {
+		errInsertMultiple = errors.New("Not all of artifact and blobs are inserted successfully")
+		err := o.Rollback()
+		if err != nil {
+			log.Errorf("fail to rollback when to insert multiple artifact and blobs, %v", err)
+			errInsertMultiple = errors.Wrap(errInsertMultiple, err.Error())
+		}
+		return errInsertMultiple
+	}
+
+	err = o.Commit()
+	if err != nil {
+		log.Errorf("fail to commit when to insert multiple artifact and blobs, %v", err)
+		return fmt.Errorf("fail to commit when to insert multiple artifact and blobs, %v", err)
+	}
+
+	return nil
+}
+
+// DeleteArtifactAndBlobByDigest ...
+func DeleteArtifactAndBlobByDigest(digest string) error {
+	_, err := GetOrmer().Raw(`delete from artifact_blob where digest_af = ? `, digest).Exec()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// CountSizeOfArtifact ...
+func CountSizeOfArtifact(digest string) (int64, error) {
+	var res []orm.Params
+	num, err := GetOrmer().Raw(`SELECT sum(bb.size) FROM artifact_blob afnb LEFT JOIN blob bb ON afnb.digest_blob = bb.digest WHERE afnb.digest_af = ? `, digest).Values(&res)
+	if err != nil {
+		return -1, err
+	}
+	if num > 0 {
+		size, err := strconv.ParseInt(res[0]["sum"].(string), 0, 64)
+		if err != nil {
+			return -1, err
+		}
+		return size, nil
+	}
+	return -1, err
+}

src/common/dao/artifact_blob_test.go

@@ -0,0 +1,131 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/common/models"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddArtifactNBlob(t *testing.T) {
+	afnb := &models.ArtifactAndBlob{
+		DigestAF:   "vvvv",
+		DigestBlob: "aaaa",
+	}
+
+	// add
+	id, err := AddArtifactNBlob(afnb)
+	require.Nil(t, err)
+	afnb.ID = id
+	assert.Equal(t, id, int64(1))
+}
+
+func TestAddArtifactNBlobs(t *testing.T) {
+	afnb1 := &models.ArtifactAndBlob{
+		DigestAF:   "zzzz",
+		DigestBlob: "zzza",
+	}
+	afnb2 := &models.ArtifactAndBlob{
+		DigestAF:   "zzzz",
+		DigestBlob: "zzzb",
+	}
+	afnb3 := &models.ArtifactAndBlob{
+		DigestAF:   "zzzz",
+		DigestBlob: "zzzc",
+	}
+
+	var afnbs []*models.ArtifactAndBlob
+	afnbs = append(afnbs, afnb1)
+	afnbs = append(afnbs, afnb2)
+	afnbs = append(afnbs, afnb3)
+
+	// add
+	err := AddArtifactNBlobs(afnbs)
+	require.Nil(t, err)
+}
+
+func TestDeleteArtifactAndBlobByDigest(t *testing.T) {
+	afnb := &models.ArtifactAndBlob{
+		DigestAF:   "vvvv",
+		DigestBlob: "vvva",
+	}
+
+	// add
+	_, err := AddArtifactNBlob(afnb)
+	require.Nil(t, err)
+
+	// delete
+	err = DeleteArtifactAndBlobByDigest(afnb.DigestAF)
+	require.Nil(t, err)
+}
+
+func TestCountSizeOfArtifact(t *testing.T) {
+
+	afnb1 := &models.ArtifactAndBlob{
+		DigestAF:   "xxxx",
+		DigestBlob: "aaaa",
+	}
+	afnb2 := &models.ArtifactAndBlob{
+		DigestAF:   "xxxx",
+		DigestBlob: "aaab",
+	}
+	afnb3 := &models.ArtifactAndBlob{
+		DigestAF:   "xxxx",
+		DigestBlob: "aaac",
+	}
+
+	var afnbs []*models.ArtifactAndBlob
+	afnbs = append(afnbs, afnb1)
+	afnbs = append(afnbs, afnb2)
+	afnbs = append(afnbs, afnb3)
+
+	err := AddArtifactNBlobs(afnbs)
+	require.Nil(t, err)
+
+	blob1 := &models.Blob{
+		Digest:      "aaaa",
+		ContentType: "v2.blob",
+		Size:        100,
+	}
+
+	_, err = AddBlob(blob1)
+	require.Nil(t, err)
+
+	blob2 := &models.Blob{
+		Digest:      "aaab",
+		ContentType: "v2.blob",
+		Size:        200,
+	}
+
+	_, err = AddBlob(blob2)
+	require.Nil(t, err)
+
+	blob3 := &models.Blob{
+		Digest:      "aaac",
+		ContentType: "v2.blob",
+		Size:        300,
+	}
+
+	_, err = AddBlob(blob3)
+	require.Nil(t, err)
+
+	imageSize, err := CountSizeOfArtifact("xxxx")
+	require.Nil(t, err)
+	require.Equal(t, imageSize, int64(600))
+}

src/common/dao/artifact_test.go

@@ -0,0 +1,92 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"testing"
+
+	"github.com/goharbor/harbor/src/common/models"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddArtifact(t *testing.T) {
+	af := &models.Artifact{
+		PID:    1,
+		Repo:   "hello-world",
+		Tag:    "latest",
+		Digest: "1234abcd",
+		Kind:   "image",
+	}
+
+	// add
+	id, err := AddArtifact(af)
+	require.Nil(t, err)
+	af.ID = id
+	assert.Equal(t, id, int64(1))
+
+}
+
+func TestDeleteArtifact(t *testing.T) {
+	af := &models.Artifact{
+		PID:    1,
+		Repo:   "hello-world",
+		Tag:    "v1.0",
+		Digest: "1234abcd",
+		Kind:   "image",
+	}
+	// add
+	id, err := AddArtifact(af)
+	require.Nil(t, err)
+
+	// delete
+	err = DeleteArtifact(id)
+	require.Nil(t, err)
+}
+
+func TestDeleteArtifactByDigest(t *testing.T) {
+	af := &models.Artifact{
+		PID:    1,
+		Repo:   "hello-world",
+		Tag:    "v1.1",
+		Digest: "TestDeleteArtifactByDigest",
+		Kind:   "image",
+	}
+	// add
+	_, err := AddArtifact(af)
+	require.Nil(t, err)
+
+	// delete
+	err = DeleteArtifactByDigest(af.Digest)
+	require.Nil(t, err)
+}
+
+func TestDeleteArtifactByTag(t *testing.T) {
+	af := &models.Artifact{
+		PID:    1,
+		Repo:   "hello-world",
+		Tag:    "v1.2",
+		Digest: "TestDeleteArtifactByTag",
+		Kind:   "image",
+	}
+	// add
+	_, err := AddArtifact(af)
+	require.Nil(t, err)
+
+	// delete
+	err = DeleteByTag(1, "hello-world", "v1.2")
+	require.Nil(t, err)
+}

src/common/dao/blob.go

@@ -0,0 +1,50 @@
+package dao
+
+import (
+	"fmt"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"strings"
+	"time"
+)
+
+// AddBlob ...
+func AddBlob(blob *models.Blob) (int64, error) {
+	now := time.Now()
+	blob.CreationTime = now
+	id, err := GetOrmer().Insert(blob)
+	if err != nil {
+		if strings.Contains(err.Error(), "duplicate key value violates unique constraint") {
+			return 0, ErrDupRows
+		}
+		return 0, err
+	}
+	return id, nil
+}
+
+// GetBlob ...
+func GetBlob(digest string) (*models.Blob, error) {
+	o := GetOrmer()
+	qs := o.QueryTable(&models.Blob{})
+	qs = qs.Filter("Digest", digest)
+	b := []*models.Blob{}
+	_, err := qs.All(&b)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get blob for digest %s, error: %v", digest, err)
+	}
+	if len(b) == 0 {
+		log.Infof("No blob found for digest %s, returning empty.", digest)
+		return &models.Blob{}, nil
+	} else if len(b) > 1 {
+		log.Infof("Multiple blob found for digest %s", digest)
+		return &models.Blob{}, fmt.Errorf("Multiple blob found for digest %s", digest)
+	}
+	return b[0], nil
+}
+
+// DeleteBlob ...
+func DeleteBlob(digest string) error {
+	o := GetOrmer()
+	_, err := o.QueryTable("blob").Filter("digest", digest).Delete()
+	return err
+}

src/common/dao/blob_test.go

@@ -0,0 +1,65 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestAddBlob(t *testing.T) {
+	blob := &models.Blob{
+		Digest:      "1234abcd",
+		ContentType: "v2.blob",
+		Size:        1523,
+	}
+
+	// add
+	_, err := AddBlob(blob)
+	require.Nil(t, err)
+}
+
+func TestGetBlob(t *testing.T) {
+	blob := &models.Blob{
+		Digest:      "12345abcde",
+		ContentType: "v2.blob",
+		Size:        453,
+	}
+
+	// add
+	id, err := AddBlob(blob)
+	require.Nil(t, err)
+	blob.ID = id
+
+	blob2, err := GetBlob("12345abcde")
+	require.Nil(t, err)
+	assert.Equal(t, blob.Digest, blob2.Digest)
+
+}
+
+func TestDeleteBlob(t *testing.T) {
+	blob := &models.Blob{
+		Digest:      "123456abcdef",
+		ContentType: "v2.blob",
+		Size:        4543,
+	}
+	id, err := AddBlob(blob)
+	require.Nil(t, err)
+	blob.ID = id
+	err = DeleteBlob(blob.Digest)
+	require.Nil(t, err)
+}

src/common/models/artifact.go

@@ -0,0 +1,23 @@
+package models
+
+import (
+	"time"
+)
+
+// Artifact holds the details of a artifact.
+type Artifact struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	PID          int64     `orm:"column(project_id)" json:"project_id"`
+	Repo         string    `orm:"column(repo)" json:"repo"`
+	Tag          string    `orm:"column(tag)" json:"tag"`
+	Digest       string    `orm:"column(digest)" json:"digest"`
+	Kind         string    `orm:"column(kind)" json:"kind"`
+	PushTime     time.Time `orm:"column(push_time)" json:"push_time"`
+	PullTime     time.Time `orm:"column(pull_time)" json:"pull_time"`
+	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+}
+
+// TableName ...
+func (af *Artifact) TableName() string {
+	return "artifact"
+}

src/common/models/artifact_blob.go

@@ -0,0 +1,18 @@
+package models
+
+import (
+	"time"
+)
+
+// ArtifactAndBlob holds the relationship between manifest and blob.
+type ArtifactAndBlob struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	DigestAF     string    `orm:"column(digest_af)" json:"digest_af"`
+	DigestBlob   string    `orm:"column(digest_blob)" json:"digest_blob"`
+	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+}
+
+// TableName ...
+func (afb *ArtifactAndBlob) TableName() string {
+	return "artifact_blob"
+}

src/common/models/base.go

@@ -37,6 +37,9 @@ func init() {
 		new(JobLog),
 		new(Robot),
 		new(OIDCUser),
+		new(Blob),
+		new(Artifact),
+		new(ArtifactAndBlob),
 		new(CVEWhitelist),
 		new(Quota),
 		new(QuotaUsage),

src/common/models/blob.go

@@ -0,0 +1,19 @@
+package models
+
+import (
+	"time"
+)
+
+// Blob holds the details of a blob.
+type Blob struct {
+	ID           int64     `orm:"pk;auto;column(id)" json:"id"`
+	Digest       string    `orm:"column(digest)" json:"digest"`
+	ContentType  string    `orm:"column(content_type)" json:"content_type"`
+	Size         int64     `orm:"column(size)" json:"size"`
+	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
+}
+
+// TableName ...
+func (b *Blob) TableName() string {
+	return "blob"
+}