harbor/src/controller/artifact/controller.go

745 lines
23 KiB
Go

// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package artifact
import (
"container/list"
"context"
stderrors "errors"
"fmt"
"strings"
"time"
"github.com/goharbor/harbor/src/pkg"
accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model"
"github.com/goharbor/harbor/src/controller/artifact/processor/chart"
"github.com/goharbor/harbor/src/controller/artifact/processor/cnab"
"github.com/goharbor/harbor/src/controller/artifact/processor/image"
"github.com/goharbor/harbor/src/controller/artifact/processor/wasm"
"github.com/goharbor/harbor/src/lib/icon"
"github.com/goharbor/harbor/src/controller/artifact/processor"
"github.com/goharbor/harbor/src/controller/event/metadata"
"github.com/goharbor/harbor/src/controller/tag"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg/accessory"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/artifactrash"
"github.com/goharbor/harbor/src/pkg/artifactrash/model"
"github.com/goharbor/harbor/src/pkg/blob"
"github.com/goharbor/harbor/src/pkg/immutable/match"
"github.com/goharbor/harbor/src/pkg/immutable/match/rule"
"github.com/goharbor/harbor/src/pkg/label"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/notifier/event"
"github.com/goharbor/harbor/src/pkg/registry"
"github.com/goharbor/harbor/src/pkg/repository"
"github.com/goharbor/harbor/src/pkg/signature"
model_tag "github.com/goharbor/harbor/src/pkg/tag/model/tag"
"github.com/opencontainers/go-digest"
)
var (
// Ctl is a global artifact controller instance
Ctl = NewController()
)
var (
// ErrBreak error to break walk
ErrBreak = stderrors.New("break")
// ErrSkip error to skip walk the children of the artifact
ErrSkip = stderrors.New("skip")
// icon digests for each known type
defaultIcons = map[string]string{
image.ArtifactTypeImage: icon.DigestOfIconImage,
chart.ArtifactTypeChart: icon.DigestOfIconChart,
cnab.ArtifactTypeCNAB: icon.DigestOfIconCNAB,
wasm.ArtifactTypeWASM: icon.DigestOfIconWASM,
}
)
// Controller defines the operations related with artifacts and tags
type Controller interface {
// Ensure the artifact specified by the digest exists under the repository,
// creates it if it doesn't exist. If tags are provided, ensure they exist
// and are attached to the artifact. If the tags don't exist, create them first.
// The "created" will be set as true when the artifact is created
Ensure(ctx context.Context, repository, digest string, option *ArtOption) (created bool, id int64, err error)
// Count returns the total count of artifacts according to the query.
// The artifacts that referenced by others and without tags are not counted
Count(ctx context.Context, query *q.Query) (total int64, err error)
// List artifacts according to the query, specify the properties returned with option
// The artifacts that referenced by others and without tags are not returned
List(ctx context.Context, query *q.Query, option *Option) (artifacts []*Artifact, err error)
// Get the artifact specified by ID, specify the properties returned with option
Get(ctx context.Context, id int64, option *Option) (artifact *Artifact, err error)
// Get the artifact specified by repository name and reference, the reference can be tag or digest,
// specify the properties returned with option
GetByReference(ctx context.Context, repository, reference string, option *Option) (artifact *Artifact, err error)
// Delete the artifact specified by artifact ID
Delete(ctx context.Context, id int64) (err error)
// Copy the artifact specified by "srcRepo" and "reference" into the repository specified by "dstRepo"
Copy(ctx context.Context, srcRepo, reference, dstRepo string) (id int64, err error)
// UpdatePullTime updates the pull time for the artifact. If the tagID is provides, update the pull
// time of the tag as well
UpdatePullTime(ctx context.Context, artifactID int64, tagID int64, time time.Time) (err error)
// GetAddition returns the addition of the artifact.
// The addition is different according to the artifact type:
// build history for image; values.yaml, readme and dependencies for chart, etc
GetAddition(ctx context.Context, artifactID int64, additionType string) (addition *processor.Addition, err error)
// AddLabel to the specified artifact
AddLabel(ctx context.Context, artifactID int64, labelID int64) (err error)
// RemoveLabel from the specified artifact
RemoveLabel(ctx context.Context, artifactID int64, labelID int64) (err error)
// Walk walks the artifact tree rooted at root, calling walkFn for each artifact in the tree, including root.
Walk(ctx context.Context, root *Artifact, walkFn func(*Artifact) error, option *Option) error
}
// NewController creates an instance of the default artifact controller
func NewController() Controller {
return &controller{
tagCtl: tag.Ctl,
repoMgr: pkg.RepositoryMgr,
artMgr: pkg.ArtifactMgr,
artrashMgr: artifactrash.Mgr,
blobMgr: blob.Mgr,
sigMgr: signature.GetManager(),
labelMgr: label.Mgr,
immutableMtr: rule.NewRuleMatcher(),
regCli: registry.Cli,
abstractor: NewAbstractor(),
accessoryMgr: accessory.Mgr,
}
}
type controller struct {
tagCtl tag.Controller
repoMgr repository.Manager
artMgr artifact.Manager
artrashMgr artifactrash.Manager
blobMgr blob.Manager
sigMgr signature.Manager
labelMgr label.Manager
immutableMtr match.ImmutableTagMatcher
regCli registry.Client
abstractor Abstractor
accessoryMgr accessory.Manager
}
type ArtOption struct {
Tags []string
Accs []*accessorymodel.AccessoryData
}
func (c *controller) Ensure(ctx context.Context, repository, digest string, option *ArtOption) (bool, int64, error) {
created, artifact, err := c.ensureArtifact(ctx, repository, digest)
if err != nil {
return false, 0, err
}
if option != nil {
for _, tag := range option.Tags {
if err = c.tagCtl.Ensure(ctx, artifact.RepositoryID, artifact.ID, tag); err != nil {
return false, 0, err
}
}
for _, acc := range option.Accs {
if err = c.accessoryMgr.Ensure(ctx, artifact.ID, acc.ArtifactID, acc.Size, acc.Digest, acc.Type); err != nil {
return false, 0, err
}
}
}
// fire event
e := &metadata.PushArtifactEventMetadata{
Ctx: ctx,
Artifact: artifact,
}
if option != nil && len(option.Tags) > 0 {
e.Tag = option.Tags[0]
}
notification.AddEvent(ctx, e)
return created, artifact.ID, nil
}
// ensure the artifact exists under the repository, create it if doesn't exist.
func (c *controller) ensureArtifact(ctx context.Context, repository, digest string) (bool, *artifact.Artifact, error) {
art, err := c.artMgr.GetByDigest(ctx, repository, digest)
// the artifact already exists under the repository, return directly
if err == nil {
return false, art, nil
}
// got other error when get the artifact, return the error
if !errors.IsErr(err, errors.NotFoundCode) {
return false, nil, err
}
// the artifact doesn't exist under the repository, create it first
repo, err := c.repoMgr.GetByName(ctx, repository)
if err != nil {
return false, nil, err
}
artifact := &artifact.Artifact{
ProjectID: repo.ProjectID,
RepositoryID: repo.RepositoryID,
RepositoryName: repository,
Digest: digest,
PushTime: time.Now(),
}
// abstract the metadata for the artifact
if err = c.abstractor.AbstractMetadata(ctx, artifact); err != nil {
return false, nil, err
}
// populate the artifact type
artifact.Type = processor.Get(artifact.MediaType).GetArtifactType(ctx, artifact)
// create it
// use orm.WithTransaction here to avoid the issue:
// https://www.postgresql.org/message-id/002e01c04da9%24a8f95c20%2425efe6c1%40lasting.ro
created := false
if err = orm.WithTransaction(func(ctx context.Context) error {
id, err := c.artMgr.Create(ctx, artifact)
if err != nil {
return err
}
created = true
artifact.ID = id
return nil
})(orm.SetTransactionOpNameToContext(ctx, "tx-ensure-artifact")); err != nil {
// got error that isn't conflict error, return directly
if !errors.IsConflictErr(err) {
return false, nil, err
}
// if got conflict error, try to get the artifact again
artifact, err = c.artMgr.GetByDigest(ctx, repository, digest)
if err != nil {
return false, nil, err
}
}
return created, artifact, nil
}
func (c *controller) Count(ctx context.Context, query *q.Query) (int64, error) {
if query != nil {
// ignore the page number and size
query = &q.Query{
Keywords: query.Keywords,
}
}
arts, err := c.List(ctx, query, nil)
if err != nil {
return int64(0), err
}
return int64(len(arts)), nil
}
func (c *controller) List(ctx context.Context, query *q.Query, option *Option) ([]*Artifact, error) {
arts, err := c.artMgr.List(ctx, query)
if err != nil {
return nil, err
}
var res []*Artifact
// Only the displayed accessory will in the artifact list
for _, art := range arts {
accs, err := c.accessoryMgr.List(ctx, q.New(q.KeyWords{"ArtifactID": art.ID, "digest": art.Digest}))
if err != nil {
return nil, err
}
if len(accs) == 0 || (len(accs) > 0 && accs[0].Display()) {
res = append(res, c.assembleArtifact(ctx, art, option))
}
}
return res, nil
}
func (c *controller) Get(ctx context.Context, id int64, option *Option) (*Artifact, error) {
art, err := c.artMgr.Get(ctx, id)
if err != nil {
return nil, err
}
return c.assembleArtifact(ctx, art, option), nil
}
func (c *controller) GetByReference(ctx context.Context, repository, reference string, option *Option) (*Artifact, error) {
// the reference is tag
if _, err := digest.Parse(reference); err != nil {
return c.getByTag(ctx, repository, reference, option)
}
// the reference is digest
return c.getByDigest(ctx, repository, reference, option)
}
func (c *controller) getByDigest(ctx context.Context, repository, digest string, option *Option) (*Artifact, error) {
art, err := c.artMgr.GetByDigest(ctx, repository, digest)
if err != nil {
return nil, err
}
return c.assembleArtifact(ctx, art, option), nil
}
func (c *controller) getByTag(ctx context.Context, repository, tag string, option *Option) (*Artifact, error) {
repo, err := c.repoMgr.GetByName(ctx, repository)
if err != nil {
return nil, err
}
tags, err := c.tagCtl.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"RepositoryID": repo.RepositoryID,
"Name": tag,
},
}, nil)
if err != nil {
return nil, err
}
if len(tags) == 0 {
return nil, errors.New(nil).WithCode(errors.NotFoundCode).
WithMessage("artifact %s:%s not found", repository, tag)
}
return c.Get(ctx, tags[0].ArtifactID, option)
}
func (c *controller) Delete(ctx context.Context, id int64) error {
accs, err := c.accessoryMgr.List(ctx, q.New(q.KeyWords{"ArtifactID": id}))
if err != nil {
return err
}
return orm.WithTransaction(func(ctx context.Context) error {
return c.deleteDeeply(ctx, id, true, len(accs) > 0)
})(orm.SetTransactionOpNameToContext(ctx, "tx-delete-artifact-delete"))
}
// "isRoot" is used to specify whether the artifact is the root parent artifact
// the error handling logic for the root parent artifact and others is different
// "isAccessory" is used to specify whether the artifact is an accessory.
func (c *controller) deleteDeeply(ctx context.Context, id int64, isRoot, isAccessory bool) error {
art, err := c.Get(ctx, id, &Option{WithTag: true, WithAccessory: true})
if err != nil {
// return nil if the nonexistent artifact isn't the root parent
if !isRoot && errors.IsErr(err, errors.NotFoundCode) {
return nil
}
return err
}
if isAccessory {
if err := c.accessoryMgr.DeleteAccessories(ctx, q.New(q.KeyWords{"ArtifactID": art.ID, "Digest": art.Digest})); err != nil && !errors.IsErr(err, errors.NotFoundCode) {
return err
}
}
// the child artifact is referenced by some tags, skip
if !isRoot && len(art.Tags) > 0 {
return nil
}
parents, err := c.artMgr.ListReferences(ctx, &q.Query{
Keywords: map[string]interface{}{
"ChildID": id,
},
})
if err != nil {
return err
}
if len(parents) > 0 {
// the root artifact is referenced by other artifacts
if isRoot {
return errors.New(nil).WithCode(errors.ViolateForeignKeyConstraintCode).
WithMessage("the deleting artifact is referenced by others")
}
// the child artifact is referenced by other artifacts, skip
return nil
}
// delete accessories if contains any
for _, acc := range art.Accessories {
// only hard ref accessory should be removed
if acc.IsHard() {
if err = c.deleteDeeply(ctx, acc.GetData().ArtifactID, true, true); err != nil {
return err
}
}
}
// delete child artifacts if contains any
for _, reference := range art.References {
// delete reference
if err = c.artMgr.DeleteReference(ctx, reference.ID); err != nil &&
!errors.IsErr(err, errors.NotFoundCode) {
return err
}
if err = c.deleteDeeply(ctx, reference.ChildID, false, false); err != nil {
return err
}
}
// delete all tags that attached to the root artifact
if isRoot {
var ids []int64
for _, tag := range art.Tags {
ids = append(ids, tag.ID)
}
if err = c.tagCtl.DeleteTags(ctx, ids); err != nil {
return err
}
}
// remove labels added to the artifact
if err := c.labelMgr.RemoveAllFrom(ctx, id); err != nil {
return err
}
// delete the artifact itself
if err = c.artMgr.Delete(ctx, art.ID); err != nil {
// the child artifact doesn't exist, skip
if !isRoot && errors.IsErr(err, errors.NotFoundCode) {
return nil
}
return err
}
blobs, err := c.blobMgr.List(ctx, q.New(q.KeyWords{"artifactDigest": art.Digest}))
if err != nil {
return err
}
// clean associations between blob and project when the blob is not needed by project
if err := c.blobMgr.CleanupAssociationsForProject(ctx, art.ProjectID, blobs); err != nil {
return err
}
// use orm.WithTransaction here to avoid the issue:
// https://www.postgresql.org/message-id/002e01c04da9%24a8f95c20%2425efe6c1%40lasting.ro
if err = orm.WithTransaction(func(ctx context.Context) error {
_, err = c.artrashMgr.Create(ctx, &model.ArtifactTrash{
MediaType: art.MediaType,
ManifestMediaType: art.ManifestMediaType,
RepositoryName: art.RepositoryName,
Digest: art.Digest,
})
return err
})(orm.SetTransactionOpNameToContext(ctx, "tx-delete-artifact-deeply")); err != nil && !errors.IsErr(err, errors.ConflictCode) {
return err
}
// only fire event for the root parent artifact
if isRoot {
var tags []string
for _, tag := range art.Tags {
tags = append(tags, tag.Name)
}
notification.AddEvent(ctx, &metadata.DeleteArtifactEventMetadata{
Ctx: ctx,
Artifact: &art.Artifact,
Tags: tags,
})
}
return nil
}
func (c *controller) Copy(ctx context.Context, srcRepo, reference, dstRepo string) (int64, error) {
dstAccs := make([]*accessorymodel.AccessoryData, 0)
return c.copyDeeply(ctx, srcRepo, reference, dstRepo, true, false, &dstAccs)
}
// as we call the docker registry APIs in the registry client directly,
// this bypass our own logic(ensure, fire event, etc.) inside the registry handlers,
// these logic must be covered explicitly here.
// "copyDeeply" iterates the child artifacts and copy them first
func (c *controller) copyDeeply(ctx context.Context, srcRepo, reference, dstRepo string, isRoot, isAcc bool, dstAccs *[]*accessorymodel.AccessoryData) (int64, error) {
var option *Option
option = &Option{WithTag: true, WithAccessory: true}
if isAcc {
option = &Option{WithTag: true}
}
srcArt, err := c.GetByReference(ctx, srcRepo, reference, option)
if err != nil {
return 0, err
}
digest := srcArt.Digest
// check the existence of artifact in the destination repository
dstArt, err := c.GetByReference(ctx, dstRepo, digest, option)
if err == nil {
// the child artifact already exists under the destination repository, skip
if !isRoot {
return dstArt.ID, nil
}
// the root parent already exists, goto next step to ensure artifact: create artifact & references, copy tags & accessories.
goto ensureArt
}
if !errors.IsErr(err, errors.NotFoundCode) {
return 0, err
}
// the artifact doesn't exist under the destination repository, continue to copy
// copy child artifacts if contains any
for _, reference := range srcArt.References {
if _, err = c.copyDeeply(ctx, srcRepo, reference.ChildDigest, dstRepo, false, false, dstAccs); err != nil {
return 0, err
}
}
// copy accessory if contains any
for _, acc := range srcArt.Accessories {
dstAcc := &accessorymodel.AccessoryData{
Digest: acc.GetData().Digest,
Type: acc.GetData().Type,
Size: acc.GetData().Size,
}
*dstAccs = append(*dstAccs, dstAcc)
id, err := c.copyDeeply(ctx, srcRepo, acc.GetData().Digest, dstRepo, false, true, dstAccs)
if err != nil {
return 0, err
}
dstAcc.ArtifactID = id
}
// copy the parent artifact into the backend docker registry
if err := c.regCli.Copy(srcRepo, digest, dstRepo, digest, false); err != nil {
return 0, err
}
ensureArt:
// only copy the tags of outermost artifact
var tags []string
for _, tag := range srcArt.Tags {
tags = append(tags, tag.Name)
}
// ensure the parent artifact exist in the database
artopt := &ArtOption{
Tags: tags,
}
if !isAcc {
artopt.Accs = *dstAccs
}
_, id, err := c.Ensure(ctx, dstRepo, digest, artopt)
if err != nil {
return 0, err
}
return id, nil
}
func (c *controller) UpdatePullTime(ctx context.Context, artifactID int64, tagID int64, time time.Time) error {
if err := c.artMgr.UpdatePullTime(ctx, artifactID, time); err != nil {
return err
}
// update tag pull time if artifact has tag
if tagID != 0 {
tg, err := c.tagCtl.Get(ctx, tagID, nil)
if err != nil {
return err
}
if tg.ArtifactID != artifactID {
return fmt.Errorf("tag %d isn't attached to artifact %d", tagID, artifactID)
}
return c.tagCtl.Update(ctx, &tag.Tag{
Tag: model_tag.Tag{
ID: tg.ID,
PullTime: time,
},
}, "PullTime")
}
return nil
}
func (c *controller) GetAddition(ctx context.Context, artifactID int64, addition string) (*processor.Addition, error) {
artifact, err := c.artMgr.Get(ctx, artifactID)
if err != nil {
return nil, err
}
return processor.Get(artifact.MediaType).AbstractAddition(ctx, artifact, addition)
}
func (c *controller) AddLabel(ctx context.Context, artifactID int64, labelID int64) (err error) {
defer func() {
if err == nil {
// trigger label artifact event
e := &event.Event{}
metaData := &metadata.ArtifactLabeledMetadata{
ArtifactID: artifactID,
LabelID: labelID,
Ctx: ctx,
}
if err := e.Build(metaData); err == nil {
if err := e.Publish(); err != nil {
log.Error(errors.Wrap(err, "mark label to resource handler: event publish"))
}
} else {
log.Error(errors.Wrap(err, "mark label to resource handler: event build"))
}
}
}()
err = c.labelMgr.AddTo(ctx, labelID, artifactID)
return
}
func (c *controller) RemoveLabel(ctx context.Context, artifactID int64, labelID int64) error {
return c.labelMgr.RemoveFrom(ctx, labelID, artifactID)
}
func (c *controller) Walk(ctx context.Context, root *Artifact, walkFn func(*Artifact) error, option *Option) error {
queue := list.New()
queue.PushBack(root)
walked := map[string]bool{}
for queue.Len() > 0 {
elem := queue.Front()
queue.Remove(elem)
artifact := elem.Value.(*Artifact)
walked[artifact.Digest] = true
if err := walkFn(artifact); err != nil {
if err == ErrBreak {
return nil
} else if err == ErrSkip {
continue
}
return err
}
if len(artifact.References) > 0 {
var ids []int64
for _, ref := range artifact.References {
ids = append(ids, ref.ChildID)
}
// HACK: base=* in KeyWords to filter all artifacts
children, err := c.List(ctx, q.New(q.KeyWords{"id__in": ids, "base": "*"}), option)
if err != nil {
return err
}
for _, child := range children {
if !walked[child.Digest] {
queue.PushBack(child)
}
}
}
if len(artifact.Accessories) > 0 {
var ids []int64
for _, acc := range artifact.Accessories {
ids = append(ids, acc.GetData().ArtifactID)
}
children, err := c.List(ctx, q.New(q.KeyWords{"id__in": ids, "base": "*"}), option)
if err != nil {
return err
}
for _, child := range children {
if !walked[child.Digest] {
queue.PushBack(child)
}
}
}
}
return nil
}
// assemble several part into a single artifact
func (c *controller) assembleArtifact(ctx context.Context, art *artifact.Artifact, option *Option) *Artifact {
artifact := &Artifact{
Artifact: *art,
}
// populate addition links
c.populateAdditionLinks(ctx, artifact)
// populate icon for the known artifact types
c.populateIcon(artifact)
if option == nil {
return artifact
}
if option.WithTag {
c.populateTags(ctx, artifact, option.TagOption)
}
if option.WithLabel {
c.populateLabels(ctx, artifact)
}
if option.WithAccessory {
c.populateAccessories(ctx, artifact)
}
return artifact
}
func (c *controller) populateIcon(art *Artifact) {
if len(art.Icon) == 0 {
if i, ok := defaultIcons[art.Type]; ok {
art.Icon = i
} else {
art.Icon = icon.DigestOfIconDefault
}
}
}
func (c *controller) populateTags(ctx context.Context, art *Artifact, option *tag.Option) {
tags, err := c.tagCtl.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"artifact_id": art.ID,
},
}, option)
if err != nil {
log.Errorf("failed to list tag of artifact %d: %v", art.ID, err)
return
}
art.Tags = tags
}
func (c *controller) populateLabels(ctx context.Context, art *Artifact) {
labels, err := c.labelMgr.ListByArtifact(ctx, art.ID)
if err != nil {
log.Errorf("failed to list labels of artifact %d: %v", art.ID, err)
return
}
art.Labels = labels
}
func (c *controller) populateAdditionLinks(ctx context.Context, artifact *Artifact) {
types := processor.Get(artifact.MediaType).ListAdditionTypes(ctx, &artifact.Artifact)
if len(types) > 0 {
version := lib.GetAPIVersion(ctx)
for _, t := range types {
artifact.SetAdditionLink(strings.ToLower(t), version)
}
}
}
func (c *controller) populateAccessories(ctx context.Context, art *Artifact) {
accs, err := c.accessoryMgr.List(ctx, q.New(q.KeyWords{"SubjectArtifactID": art.ID}))
if err != nil {
log.Errorf("failed to list accessories of artifact %d: %v", art.ID, err)
return
}
art.Accessories = accs
}