add cosign support in replication (#16282)

For the case Harbor-to-Harbor, the accessory can be replicated from source or to target.

Signed-off-by: Wang Yan <wangyan@vmware.com>

src/controller/artifact/model.go

@@ -15,6 +15,7 @@
 package artifact
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/controller/tag"
@@ -30,7 +31,38 @@ type Artifact struct {
 	Tags          []*tag.Tag                 `json:"tags"`           // the list of tags that attached to the artifact
 	AdditionLinks map[string]*AdditionLink   `json:"addition_links"` // the resource link for build history(image), values.yaml(chart), dependency(chart), etc
 	Labels        []*model.Label             `json:"labels"`
-	Accessories   []accessoryModel.Accessory `json:"accessories"`
+	Accessories   []accessoryModel.Accessory `json:"-"`
+}
+
+// UnmarshalJSON to customize the accessories unmarshal
+func (artifact *Artifact) UnmarshalJSON(data []byte) error {
+	type Alias Artifact
+	ali := &struct {
+		*Alias
+		AccessoryItems []interface{} `json:"accessories,omitempty"`
+	}{
+		Alias: (*Alias)(artifact),
+	}
+
+	if err := json.Unmarshal(data, &ali); err != nil {
+		return err
+	}
+
+	if len(ali.AccessoryItems) > 0 {
+		for _, item := range ali.AccessoryItems {
+			data, err := json.Marshal(item)
+			if err != nil {
+				return err
+			}
+			acc, err := accessoryModel.ToAccessory(data)
+			if err != nil {
+				return err
+			}
+			artifact.Accessories = append(artifact.Accessories, acc)
+		}
+	}
+
+	return nil
 }
 
 // SetAdditionLink set a addition link

src/controller/artifact/model_test.go

@@ -0,0 +1,104 @@
+package artifact
+
+import (
+	"encoding/json"
+	"github.com/goharbor/harbor/src/pkg/accessory/model/cosign"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestUnmarshalJSONWithACC(t *testing.T) {
+	data := []byte(`[{"accessories":[{"artifact_id":9,"creation_time":"2022-01-20T09:18:50.993Z","digest":"sha256:a7caa2636af890178a0b8c4cdbc47ced4dbdf29a1680e9e50823e85ce35b28d3","icon":"","id":4,"size":501,"subject_artifact_id":8,"type":"signature.cosign"}],
+	"addition_links":{"build_history":{"absolute":false,"href":"/api/v2.0/projects/source_project011642670285/repositories/redis/artifacts/sha256:e4b315ad03a1d1d9ff0c111e648a1a91066c09ead8352d3d6a48fa971a82922c/additions/build_history"},
+	"vulnerabilities":{"absolute":false,"href":"/api/v2.0/projects/source_project011642670285/repositories/redis/artifacts/sha256:e4b315ad03a1d1d9ff0c111e648a1a91066c09ead8352d3d6a48fa971a82922c/additions/vulnerabilities"}},
+	"digest":"sha256:e4b315ad03a1d1d9ff0c111e648a1a91066c09ead8352d3d6a48fa971a82922c",
+	"extra_attrs":{"architecture":"amd64","author":"","config":{"Cmd":["redis-server"],"Entrypoint":["docker-entrypoint.sh"],"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOSU_VERSION=1.11","REDIS_VERSION=5.0.7","REDIS_DOWNLOAD_URL=redis-5.0.7.tar.gz","REDIS_DOWNLOAD_SHA=61db74eabf6801f057fd24b590"],"ExposedPorts":{"6379/tcp":{}},"Volumes":{"/data":{}},"WorkingDir":"/data"},"created":"2020-01-03T01:29:15.570681619Z","os":"linux"},"icon":"sha256:0048162a053eef4d4ce3fe7518615bef084403614f8bca43b40ae2e762e11e06","id":8,"labels":null,"manifest_media_type":"application/vnd.docker.distribution.manifest.v2+json","media_type":"application/vnd.docker.container.image.v1+json","project_id":7,"pull_time":"2022-01-20T09:18:50.783Z","push_time":"2022-01-20T09:18:50.290Z","references":null,"repository_id":5,"size":35804754,
+	"tags":[{"artifact_id":8,"id":6,"immutable":false,"name":"latest","pull_time":"2022-01-20T09:18:50.783Z","push_time":"2022-01-20T09:18:50.303Z","repository_id":5,"signed":false}],"type":"IMAGE"}]`)
+
+	var artifact []Artifact
+	if err := json.Unmarshal(data, &artifact); err != nil {
+		t.Fail()
+	}
+
+	assert.Equal(t, int64(9), artifact[0].Accessories[0].GetData().ArtifactID)
+	assert.Equal(t, "latest", artifact[0].Tags[0].Name)
+	assert.Equal(t, "amd64", artifact[0].ExtraAttrs["architecture"])
+
+	_, ok := artifact[0].Accessories[0].(*cosign.Signature)
+	assert.True(t, ok)
+}
+
+func TestUnmarshalJSONWithACCPartial(t *testing.T) {
+	data := []byte(`[{"accessories":[{"artifact_id":9,"creation_time":"2022-01-20T09:18:50.993Z","digest":"sha256:a7caa2636af890178a0b8c4cdbc47ced4dbdf29a1680e9e50823e85ce35b28d3","icon":"","id":4,"size":501,"subject_artifact_id":8,"type":"signature.cosign"}, {"artifact_id":2, "type":"signature.cosign"}],
+	"digest":"sha256:e4b315ad03a1d1d9ff0c111e648a1a91066c09ead8352d3d6a48fa971a82922c","tags":[{"artifact_id":8,"id":6,"immutable":false,"name":"latest","pull_time":"2022-01-20T09:18:50.783Z","push_time":"2022-01-20T09:18:50.303Z","repository_id":5,"signed":false}],"type":"IMAGE"}]`)
+
+	var artifact []Artifact
+	if err := json.Unmarshal(data, &artifact); err != nil {
+		t.Fail()
+	}
+
+	assert.Equal(t, int64(9), artifact[0].Accessories[0].GetData().ArtifactID)
+	assert.Equal(t, int64(2), artifact[0].Accessories[1].GetData().ArtifactID)
+	assert.Equal(t, "latest", artifact[0].Tags[0].Name)
+	_, ok := artifact[0].Accessories[1].(*cosign.Signature)
+	assert.True(t, ok)
+}
+
+func TestUnmarshalJSONWithACCUnknownType(t *testing.T) {
+	data := []byte(`[{"accessories":[{"artifact_id":9,"creation_time":"2022-01-20T09:18:50.993Z","digest":"sha256:a7caa2636af890178a0b8c4cdbc47ced4dbdf29a1680e9e50823e85ce35b28d3","icon":"","id":4,"size":501,"subject_artifact_id":8}],
+	"digest":"sha256:e4b315ad03a1d1d9ff0c111e648a1a91066c09ead8352d3d6a48fa971a82922c","tags":[{"artifact_id":8,"id":6,"immutable":false,"name":"latest","pull_time":"2022-01-20T09:18:50.783Z","push_time":"2022-01-20T09:18:50.303Z","repository_id":5,"signed":false}],"type":"IMAGE"}]`)
+
+	var artifact []Artifact
+	err := json.Unmarshal(data, &artifact)
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), "accessory type  not support")
+}
+
+func TestUnmarshalJSONWithoutACC(t *testing.T) {
+	data := []byte(`[{"addition_links":{"build_history":{"absolute":false,"href":"/api/v2.0/projects/source_project011642670285/repositories/redis/artifacts/sha256:e4b315ad03a1d1d9ff0c111e648a1a91066c09ead8352d3d6a48fa971a82922c/additions/build_history"},
+"vulnerabilities":{"absolute":false,"href":"/api/v2.0/projects/source_project011642670285/repositories/redis/artifacts/sha256:e4b315ad03a1d1d9ff0c111e648a1a91066c09ead8352d3d6a48fa971a82922c/additions/vulnerabilities"}},
+"digest":"sha256:e4b315ad03a1d1d9ff0c111e648a1a91066c09ead8352d3d6a48fa971a82922c",
+"extra_attrs":{"architecture":"amd64","author":"","config":{"Cmd":["redis-server"],"Entrypoint":["docker-entrypoint.sh"],"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOSU_VERSION=1.11","REDIS_VERSION=5.0.7","REDIS_DOWNLOAD_URL=redis-5.0.7.tar.gz","REDIS_DOWNLOAD_SHA=61db74eabf6801f057fd24b590"],"ExposedPorts":{"6379/tcp":{}},"Volumes":{"/data":{}},"WorkingDir":"/data"},"created":"2020-01-03T01:29:15.570681619Z","os":"linux"},"icon":"sha256:0048162a053eef4d4ce3fe7518615bef084403614f8bca43b40ae2e762e11e06","id":8,"labels":null,"manifest_media_type":"application/vnd.docker.distribution.manifest.v2+json","media_type":"application/vnd.docker.container.image.v1+json","project_id":7,"pull_time":"2022-01-20T09:18:50.783Z","push_time":"2022-01-20T09:18:50.290Z","references":null,"repository_id":5,"size":35804754,
+"tags":[{"artifact_id":8,"id":6,"immutable":false,"name":"latest","pull_time":"2022-01-20T09:18:50.783Z","push_time":"2022-01-20T09:18:50.303Z","repository_id":5,"signed":false}],"type":"IMAGE"}]`)
+
+	var artifact []Artifact
+	if err := json.Unmarshal(data, &artifact); err != nil {
+		t.Fail()
+	}
+
+	assert.Equal(t, "latest", artifact[0].Tags[0].Name)
+	assert.Equal(t, "amd64", artifact[0].ExtraAttrs["architecture"])
+}
+
+func TestUnmarshalJSONWithAccNull(t *testing.T) {
+	data := []byte(`{"accessories":null,"addition_links":{"build_history":{"absolute":false,"href":"/api/v2.0/projects/project-1643104251947/repositories/test3/artifacts/sha256:fc00fd623137fa47bd5b3f/additions/build_history"}},
+"digest":"sha256:fc00fd623137fa47bd5b3f2","extra_attrs":{"architecture":"amd64","author":"","config":{"Cmd":["dd","if=/dev/urandom","of=test","bs=1M","count=1"],"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"created":"2022-01-25T09:51:13.904772229Z","os":"linux"},"icon":"sha256:0048162a053eef4d4ce3fe7518615bef084403614f8bca43b40ae2e762e11e06","id":12,"labels":null,"manifest_media_type":"application/vnd.docker.distribution.manifest.v2+json","media_type":"application/vnd.docker.container.image.v1+json","project_id":8,"pull_time":"0001-01-01T00:00:00.000Z","push_time":"2022-01-25T09:51:14.394Z","references":null,"repository_id":6,"size":1816010,"tags":[{"artifact_id":12,"id":13,"immutable":false,"name":"1.0","pull_time":"0001-01-01T00:00:00.000Z","push_time":"2022-01-25T09:51:14.406Z","repository_id":6,"signed":false}],"type":"IMAGE"}`)
+
+	var artifact Artifact
+	if err := json.Unmarshal(data, &artifact); err != nil {
+		t.Fail()
+	}
+
+	assert.Equal(t, "1.0", artifact.Tags[0].Name)
+	assert.Equal(t, "amd64", artifact.ExtraAttrs["architecture"])
+}
+
+func TestUnmarshalJSONWithNull(t *testing.T) {
+	data := []byte(`{}`)
+	var artifact Artifact
+	if err := json.Unmarshal(data, &artifact); err != nil {
+		t.Fail()
+	}
+	assert.Equal(t, "", artifact.Digest)
+}
+
+func TestUnmarshalJSONWithPartial(t *testing.T) {
+	data := []byte(`{"digest":"sha256:1234","media_type":"application/vnd.docker.container.image.v1+json","project_id":8,"pull_time":"0001-01-01T00:00:00.000Z","push_time":"2022-01-25T09:51:14.394Z","references":null}`)
+	var artifact Artifact
+	if err := json.Unmarshal(data, &artifact); err != nil {
+		t.Fail()
+	}
+	assert.Equal(t, "sha256:1234", artifact.Digest)
+	assert.Equal(t, "", artifact.Type)
+	assert.Equal(t, "application/vnd.docker.container.image.v1+json", artifact.MediaType)
+}

src/pkg/accessory/model/accessory.go

@@ -15,7 +15,9 @@
 package model
 
 import (
+	"encoding/json"
 	"fmt"
+	"github.com/goharbor/harbor/src/lib/errors"
 	"sync"
 	"time"
 )
@@ -35,6 +37,8 @@ type RefProvider interface {
 }
 
 /*
+RefIdentifier
+
 Soft reference: The accessory is not tied to the subject manifest.
 Hard reference: The accessory is tied to the subject manifest.
 
@@ -55,7 +59,7 @@ type RefIdentifier interface {
 }
 
 const (
-	// TypeNone
+	// TypeNone ...
 	TypeNone = "base"
 	// TypeCosignSignature ...
 	TypeCosignSignature = "signature.cosign"
@@ -63,20 +67,20 @@ const (
 
 // AccessoryData ...
 type AccessoryData struct {
-	ID            int64
+	ID            int64     `json:"id"`
-	ArtifactID    int64
+	ArtifactID    int64     `json:"artifact_id"`
-	SubArtifactID int64
+	SubArtifactID int64     `json:"subject_artifact_id"`
-	Type          string
+	Type          string    `json:"type"`
-	Size          int64
+	Size          int64     `json:"size"`
-	Digest        string
+	Digest        string    `json:"digest"`
-	CreatTime     time.Time
+	CreatTime     time.Time `json:"creation_time"`
 }
 
-// Accessory: Independent, but linked to an existing subject artifact, which enabling the extendibility of an OCI artifact.
+// Accessory Independent, but linked to an existing subject artifact, which enabling the extensibility of an OCI artifact
 type Accessory interface {
 	RefProvider
 	RefIdentifier
-	// Define whether shows in the artifact list response.
+	// Display Define whether shows in the artifact list response.
 	Display() bool
 	GetData() AccessoryData
 }
@@ -89,7 +93,7 @@ var (
 	lock      sync.RWMutex
 )
 
-// Register register accessory factory for type
+// Register accessory factory for type
 func Register(typ string, factory NewAccessoryFunc) {
 	lock.Lock()
 	defer lock.Unlock()
@@ -110,3 +114,16 @@ func New(typ string, data AccessoryData) (Accessory, error) {
 	data.Type = typ
 	return factory(data), nil
 }
+
+// ToAccessory converts json object to Accessory
+func ToAccessory(acc []byte) (Accessory, error) {
+	var data AccessoryData
+	if err := json.Unmarshal(acc, &data); err != nil {
+		return nil, err
+	}
+	factory, ok := factories[data.Type]
+	if !ok {
+		return nil, errors.Errorf("accessory type %s not support", data.Type)
+	}
+	return factory(data), nil
+}

src/pkg/accessory/model/accessory_test.go

@@ -50,6 +50,12 @@ func (suite *AccessoryTestSuite) TestNew() {
 	}
 }
 
+func (suite *AccessoryTestSuite) TestToAccessory() {
+	data := []byte(`{"artifact_id":9,"creation_time":"2022-01-20T09:18:50.993Z","digest":"sha256:1234","icon":"","id":4,"size":501,"subject_artifact_id":8,"type":"signature.cosign"}`)
+	_, err := ToAccessory(data)
+	suite.NotNil(err)
+}
+
 func TestAccessoryTestSuite(t *testing.T) {
 	suite.Run(t, new(AccessoryTestSuite))
 }

src/pkg/reg/adapter/harbor/v2/client.go

@@ -16,13 +16,13 @@ package v2
 
 import (
 	"fmt"
-
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/controller/artifact"
 	"github.com/goharbor/harbor/src/lib/encode/repository"
 	"github.com/goharbor/harbor/src/pkg/reg/adapter/harbor/base"
 	"github.com/goharbor/harbor/src/pkg/reg/model"
 	repomodel "github.com/goharbor/harbor/src/pkg/repository/model"
+	tagmodel "github.com/goharbor/harbor/src/pkg/tag/model/tag"
 )
 
 type client struct {
@@ -48,7 +48,7 @@ func (c *client) listRepositories(project *base.Project) ([]*model.Repository, e
 func (c *client) listArtifacts(repo string) ([]*model.Artifact, error) {
 	project, repo := utils.ParseRepository(repo)
 	repo = repository.Encode(repo)
-	url := fmt.Sprintf("%s/projects/%s/repositories/%s/artifacts?with_label=true",
+	url := fmt.Sprintf("%s/projects/%s/repositories/%s/artifacts?with_label=true&with_accessory=true",
 		c.BasePath(), project, repo)
 	artifacts := []*artifact.Artifact{}
 	if err := c.C.GetAndIteratePagination(url, &artifacts); err != nil {
@@ -67,10 +67,40 @@ func (c *client) listArtifacts(repo string) ([]*model.Artifact, error) {
 			art.Tags = append(art.Tags, tag.Name)
 		}
 		arts = append(arts, art)
+
+		// For Harbor v2 clients, it has to append the accessory objects behind the subject artifact it has.
+		for _, acc := range artifact.Accessories {
+			art := &model.Artifact{
+				Type:   artifact.Type,
+				Digest: acc.GetData().Digest,
+			}
+			tags, err := c.listTags(project, repo, acc.GetData().Digest)
+			if err != nil {
+				return nil, err
+			}
+			for _, tag := range tags {
+				art.Tags = append(art.Tags, tag)
+			}
+			arts = append(arts, art)
+		}
 	}
 	return arts, nil
 }
 
+func (c *client) listTags(project, repo, digest string) ([]string, error) {
+	tags := []*tagmodel.Tag{}
+	url := fmt.Sprintf("%s/projects/%s/repositories/%s/artifacts/%s/tags",
+		c.BasePath(), project, repo, digest)
+	if err := c.C.GetAndIteratePagination(url, &tags); err != nil {
+		return nil, err
+	}
+	var tagNames []string
+	for _, tag := range tags {
+		tagNames = append(tagNames, tag.Name)
+	}
+	return tagNames, nil
+}
+
 func (c *client) deleteTag(repo, tag string) error {
 	project, repo := utils.ParseRepository(repo)
 	repo = repository.Encode(repo)