Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-22 16:48:30 +01:00
Code Issues Projects Releases Wiki Activity

Make sure replication work with OCI artifacts(phase 1)

Browse Source 
This commit updates the definition of replicated resource(artifacts replace the vtags) and refactor the filter part

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2020-03-03 08:36:47 +08:00
parent 5888ecf158
commit d4ba023457
29 changed files with 666 additions and 892 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
src/replication/adapter/adapter.go
View File

@ -20,7 +20,6 @@ import (
"io"
"github.com/docker/distribution"
"github.com/goharbor/harbor/src/replication/filter"
"github.com/goharbor/harbor/src/replication/model"
)
@ -72,59 +71,6 @@ type ChartRegistry interface {
DeleteChart(name, version string) error
}
// Repository defines an repository object, it can be image repository, chart repository and etc.
type Repository struct {
ResourceType string `json:"resource_type"`
Name string `json:"name"`
}
// GetName returns the name
func (r *Repository) GetName() string {
return r.Name
}
// GetFilterableType returns the filterable type
func (r *Repository) GetFilterableType() filter.FilterableType {
return filter.FilterableTypeRepository
}
// GetResourceType returns the resource type
func (r *Repository) GetResourceType() string {
return r.ResourceType
}
// GetLabels returns the labels
func (r *Repository) GetLabels() []string {
return nil
}
// VTag defines an vTag object, it can be image tag, chart version and etc.
type VTag struct {
ResourceType string `json:"resource_type"`
Name string `json:"name"`
Labels []string `json:"labels"`
}
// GetFilterableType returns the filterable type
func (v *VTag) GetFilterableType() filter.FilterableType {
return filter.FilterableTypeVTag
}
// GetResourceType returns the resource type
func (v *VTag) GetResourceType() string {
return v.ResourceType
}
// GetName returns the name
func (v *VTag) GetName() string {
return v.Name
}
// GetLabels returns the labels
func (v *VTag) GetLabels() []string {
return v.Labels
}
// RegisterFactory registers one adapter factory to the registry
func RegisterFactory(t model.RegistryType, factory Factory) error {
if len(t) == 0 {

3
src/replication/adapter/aliacr/adapter.go
View File

@ -253,8 +253,7 @@ func (a *adapter) FetchImages(filters []*model.Filter) (resources []*model.Resou
Repository: &model.Repository{
Name: filepath.Join(repo.RepoNamespace, repo.RepoName),
},
Vtags: filterTags,
Labels: []string{},
Vtags: filterTags,
},
}
}

12
src/replication/adapter/harbor/adapter.go
View File

@ -284,18 +284,6 @@ func (a *adapter) getProject(name string) (*project, error) {
return nil, nil
}
func (a *adapter) getRepositories(projectID int64) ([]*adp.Repository, error) {
repositories := []*adp.Repository{}
url := fmt.Sprintf("%s/api/%s/repositories?project_id=%d&page=1&page_size=500", a.getURL(), api.APIVersion, projectID)
if err := a.client.GetAndIteratePagination(url, &repositories); err != nil {
return nil, err
}
for _, repository := range repositories {
repository.ResourceType = string(model.ResourceTypeImage)
}
return repositories, nil
}
// when the adapter is created for local Harbor, returns the "http://127.0.0.1:8080"
// as URL to avoid issue https://github.com/goharbor/harbor-helm/issues/222
// when harbor is deployed on Kubernetes

39
src/replication/adapter/harbor/chart_registry.go
View File

@ -17,6 +17,7 @@ package harbor
import (
"bytes"
"fmt"
"github.com/goharbor/harbor/src/replication/filter"
"io"
"io/ioutil"
"mime/multipart"
@ -25,7 +26,6 @@ import (
"github.com/goharbor/harbor/src/common/api"
common_http "github.com/goharbor/harbor/src/common/http"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
)
@ -47,14 +47,15 @@ type chartVersionMetadata struct {
}
func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
projects, err := a.listCandidateProjects(filters)
projects, err := a.listProjects(filters)
if err != nil {
return nil, err
}
resources := []*model.Resource{}
for _, project := range projects {
url := fmt.Sprintf("%s/api/%s/chartrepo/%s/charts", a.getURL(), api.APIVersion, project.Name)
repositories := []*adp.Repository{}
repositories := []*model.Repository{}
if err := a.client.Get(url, &repositories); err != nil {
return nil, err
}
@ -63,13 +64,12 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error
}
for _, repository := range repositories {
repository.Name = fmt.Sprintf("%s/%s", project.Name, repository.Name)
repository.ResourceType = string(model.ResourceTypeChart)
}
for _, filter := range filters {
if err = filter.DoFilter(&repositories); err != nil {
return nil, err
}
repositories, err = filter.DoFilterRepositories(repositories, filters)
if err != nil {
return nil, err
}
for _, repository := range repositories {
name := strings.SplitN(repository.Name, "/", 2)[1]
url := fmt.Sprintf("%s/api/%s/chartrepo/%s/charts/%s", a.getURL(), api.APIVersion, project.Name, name)
@ -80,25 +80,26 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error
if len(versions) == 0 {
continue
}
vTags := []*adp.VTag{}
var artifacts []*model.Artifact
for _, version := range versions {
var labels []string
for _, label := range version.Labels {
labels = append(labels, label.Name)
}
vTags = append(vTags, &adp.VTag{
Name: version.Version,
Labels: labels,
ResourceType: string(model.ResourceTypeChart),
artifacts = append(artifacts, &model.Artifact{
Tags: []string{version.Version},
Labels: labels,
})
}
for _, filter := range filters {
if err = filter.DoFilter(&vTags); err != nil {
return nil, err
}
artifacts, err = filter.DoFilterArtifacts(artifacts, filters)
if err != nil {
return nil, err
}
if len(artifacts) == 0 {
continue
}
for _, vTag := range vTags {
for _, artifact := range artifacts {
resources = append(resources, &model.Resource{
Type: model.ResourceTypeChart,
Registry: a.registry,
@ -107,7 +108,7 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error
Name: repository.Name,
Metadata: project.Metadata,
},
Vtags: []string{vTag.Name},
Artifacts: []*model.Artifact{artifact},
},
})
}

8
src/replication/adapter/harbor/chart_registry_test.go
View File

@ -77,8 +77,8 @@ func TestFetchCharts(t *testing.T) {
assert.Equal(t, 2, len(resources))
assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
assert.Equal(t, "library/harbor", resources[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
assert.Equal(t, 1, len(resources[0].Metadata.Artifacts))
assert.Equal(t, "1.0", resources[0].Metadata.Artifacts[0].Tags[0])
// not nil filter
filters := []*model.Filter{
{
@ -95,8 +95,8 @@ func TestFetchCharts(t *testing.T) {
require.Equal(t, 1, len(resources))
assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
assert.Equal(t, "library/harbor", resources[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
assert.Equal(t, 1, len(resources[0].Metadata.Artifacts))
assert.Equal(t, "1.0", resources[0].Metadata.Artifacts[0].Tags[0])
}
func TestChartExist(t *testing.T) {

109
src/replication/adapter/harbor/image_registry.go
View File

@ -16,35 +16,33 @@ package harbor
import (
"fmt"
"strings"
"github.com/goharbor/harbor/src/api/artifact"
"github.com/goharbor/harbor/src/common/api"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/filter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"strings"
)
func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
projects, err := a.listCandidateProjects(filters)
projects, err := a.listProjects(filters)
if err != nil {
return nil, err
}
resources := []*model.Resource{}
var resources []*model.Resource
for _, project := range projects {
repositories, err := a.getRepositories(project.ID)
repositories, err := a.listRepositories(project, filters)
if err != nil {
return nil, err
}
if len(repositories) == 0 {
continue
}
for _, filter := range filters {
if err = filter.DoFilter(&repositories); err != nil {
return nil, err
}
}
var rawResources = make([]*model.Resource, len(repositories))
runner := utils.NewLimitedConcurrentRunner(adp.MaxConcurrency)
@ -54,27 +52,15 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
index := i
repo := r
runner.AddTask(func() error {
vTags, err := a.getTags(repo.Name)
artifacts, err := a.listArtifacts(repo.Name, filters)
if err != nil {
return fmt.Errorf("List tags for repo '%s' error: %v", repo.Name, err)
return fmt.Errorf("failed to list artifacts of repository '%s': %v", repo.Name, err)
}
if len(vTags) == 0 {
if len(artifacts) == 0 {
rawResources[index] = nil
return nil
}
for _, filter := range filters {
if err = filter.DoFilter(&vTags); err != nil {
return fmt.Errorf("Filter tags %v error: %v", vTags, err)
}
}
if len(vTags) == 0 {
rawResources[index] = nil
return nil
}
tags := []string{}
for _, vTag := range vTags {
tags = append(tags, vTag.Name)
}
rawResources[index] = &model.Resource{
Type: model.ResourceTypeImage,
Registry: a.registry,
@ -83,10 +69,9 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
Name: repo.Name,
Metadata: project.Metadata,
},
Vtags: tags,
Artifacts: artifacts,
},
}
return nil
})
}
@ -106,7 +91,7 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
return resources, nil
}
func (a *adapter) listCandidateProjects(filters []*model.Filter) ([]*project, error) {
func (a *adapter) listProjects(filters []*model.Filter) ([]*project, error) {
pattern := ""
for _, filter := range filters {
if filter.Type == model.FilterTypeName {
@ -114,7 +99,7 @@ func (a *adapter) listCandidateProjects(filters []*model.Filter) ([]*project, er
break
}
}
projects := []*project{}
var projects []*project
if len(pattern) > 0 {
substrings := strings.Split(pattern, "/")
projectPattern := substrings[0]
@ -133,7 +118,7 @@ func (a *adapter) listCandidateProjects(filters []*model.Filter) ([]*project, er
}
}
if len(projects) > 0 {
names := []string{}
var names []string
for _, project := range projects {
names = append(names, project.Name)
}
@ -143,35 +128,43 @@ func (a *adapter) listCandidateProjects(filters []*model.Filter) ([]*project, er
return a.getProjects("")
}
// override the default implementation from the default image registry
// by calling Harbor API directly
func (a *adapter) DeleteManifest(repository, reference string) error {
url := fmt.Sprintf("%s/api/v2.0/repositories/%s/tags/%s", a.url, repository, reference)
return a.client.Delete(url)
}
func (a *adapter) getTags(repository string) ([]*adp.VTag, error) {
url := fmt.Sprintf("%s/api/v2.0/repositories/%s/tags", a.getURL(), repository)
tags := []*struct {
Name string `json:"name"`
Labels []*struct {
Name string `json:"name"`
}
}{}
if err := a.client.Get(url, &tags); err != nil {
func (a *adapter) listRepositories(project *project, filters []*model.Filter) ([]*model.Repository, error) {
repositories := []*models.RepoRecord{}
url := fmt.Sprintf("%s/api/%s/projects/%s/repositories", a.getURL(), api.APIVersion, project.Name)
if err := a.client.GetAndIteratePagination(url, &repositories); err != nil {
return nil, err
}
vTags := []*adp.VTag{}
for _, tag := range tags {
var labels []string
for _, label := range tag.Labels {
labels = append(labels, label.Name)
}
vTags = append(vTags, &adp.VTag{
Name: tag.Name,
Labels: labels,
ResourceType: string(model.ResourceTypeImage),
var repos []*model.Repository
for _, repository := range repositories {
repos = append(repos, &model.Repository{
Name: repository.Name,
Metadata: project.Metadata,
})
}
return vTags, nil
return filter.DoFilterRepositories(repos, filters)
}
func (a *adapter) listArtifacts(repository string, filters []*model.Filter) ([]*model.Artifact, error) {
project, repository := utils.ParseRepository(repository)
url := fmt.Sprintf("%s/api/%s/projects/%s/repositories/%s/artifacts?with_label=true",
a.getURL(), api.APIVersion, project, repository)
artifacts := []*artifact.Artifact{}
if err := a.client.Get(url, &artifacts); err != nil {
return nil, err
}
var arts []*model.Artifact
for _, artifact := range artifacts {
art := &model.Artifact{
Type: artifact.Type,
Digest: artifact.Digest,
}
for _, label := range artifact.Labels {
art.Labels = append(art.Labels, label.Name)
}
for _, tag := range artifact.Tags {
art.Tags = append(art.Tags, tag.Name)
}
arts = append(arts, art)
}
return filter.DoFilterArtifacts(arts, filters)
}

84
src/replication/adapter/harbor/image_registry_test.go
View File

@ -26,6 +26,41 @@ import (
func TestFetchImages(t *testing.T) {
server := test.NewServer([]*test.RequestHandlerMapping{
{
Method: http.MethodGet,
Pattern: "/api/v2.0/projects/library/repositories/hello-world/artifacts",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[
{
"digest": "digest1",
"tags": [
{
"name": "1.0"
}
]
},
{
"digest": "digest2",
"tags": [
{
"name": "2.0"
}
]
}
]`
w.Write([]byte(data))
},
},
{
Method: http.MethodGet,
Pattern: "/api/v2.0/projects/library/repositories",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "library/hello-world"
}]`
w.Write([]byte(data))
},
},
{
Method: http.MethodGet,
Pattern: "/api/v2.0/projects",
@ -37,28 +72,6 @@ func TestFetchImages(t *testing.T) {
w.Write([]byte(data))
},
},
{
Method: http.MethodGet,
Pattern: "/api/v2.0/repositories/library/hello-world/tags",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "1.0"
},{
"name": "2.0"
}]`
w.Write([]byte(data))
},
},
{
Method: http.MethodGet,
Pattern: "/api/v2.0/repositories",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "library/hello-world"
}]`
w.Write([]byte(data))
},
},
}...)
defer server.Close()
registry := &model.Registry{
@ -72,9 +85,9 @@ func TestFetchImages(t *testing.T) {
assert.Equal(t, 1, len(resources))
assert.Equal(t, model.ResourceTypeImage, resources[0].Type)
assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name)
assert.Equal(t, 2, len(resources[0].Metadata.Vtags))
assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
assert.Equal(t, "2.0", resources[0].Metadata.Vtags[1])
assert.Equal(t, 2, len(resources[0].Metadata.Artifacts))
assert.Equal(t, "1.0", resources[0].Metadata.Artifacts[0].Tags[0])
assert.Equal(t, "2.0", resources[0].Metadata.Artifacts[1].Tags[0])
// not nil filter
filters := []*model.Filter{
{
@ -91,23 +104,6 @@ func TestFetchImages(t *testing.T) {
assert.Equal(t, 1, len(resources))
assert.Equal(t, model.ResourceTypeImage, resources[0].Type)
assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
}
func TestDeleteManifest(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodDelete,
Pattern: "/api/v2.0/repositories/library/hello-world/tags/1.0",
Handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}})
defer server.Close()
registry := &model.Registry{
URL: server.URL,
}
adapter, err := newAdapter(registry)
require.Nil(t, err)
err = adapter.DeleteManifest("library/hello-world", "1.0")
require.Nil(t, err)
assert.Equal(t, 1, len(resources[0].Metadata.Artifacts))
assert.Equal(t, "1.0", resources[0].Metadata.Artifacts[0].Tags[0])
}

40
src/replication/adapter/helmhub/chart_registry.go
View File

@ -16,13 +16,13 @@ package helmhub
import (
"fmt"
"github.com/goharbor/harbor/src/replication/filter"
"io"
"io/ioutil"
"net/http"
"strings"
"github.com/goharbor/harbor/src/common/utils/log"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/pkg/errors"
)
@ -34,19 +34,16 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error
}
resources := []*model.Resource{}
repositories := []*adp.Repository{}
var repositories []*model.Repository
for _, chart := range charts.Data {
repository := &adp.Repository{
ResourceType: string(model.ResourceTypeChart),
Name: chart.ID,
}
repositories = append(repositories, repository)
repositories = append(repositories, &model.Repository{
Name: chart.ID,
})
}
for _, filter := range filters {
if err = filter.DoFilter(&repositories); err != nil {
return nil, err
}
repositories, err = filter.DoFilterRepositories(repositories, filters)
if err != nil {
return nil, err
}
for _, repository := range repositories {
@ -56,21 +53,22 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error
return nil, err
}
vTags := []*adp.VTag{}
var artifacts []*model.Artifact
for _, version := range versionList.Data {
vTags = append(vTags, &adp.VTag{
Name: version.Attributes.Version,
ResourceType: string(model.ResourceTypeChart),
artifacts = append(artifacts, &model.Artifact{
Tags: []string{version.Attributes.Version},
})
}
for _, filter := range filters {
if err = filter.DoFilter(&vTags); err != nil {
return nil, err
}
artifacts, err = filter.DoFilterArtifacts(artifacts, filters)
if err != nil {
return nil, err
}
if len(artifacts) == 0 {
continue
}
for _, vTag := range vTags {
for _, artifact := range artifacts {
resources = append(resources, &model.Resource{
Type: model.ResourceTypeChart,
Registry: a.registry,
@ -78,7 +76,7 @@ func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error
Repository: &model.Repository{
Name: repository.Name,
},
Vtags: []string{vTag.Name},
Artifacts: []*model.Artifact{artifact},
},
})
}

8
src/replication/adapter/helmhub/chart_registry_test.go
View File

@ -36,8 +36,8 @@ func TestFetchCharts(t *testing.T) {
require.Nil(t, err)
assert.NotZero(t, len(resources))
assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.NotNil(t, resources[0].Metadata.Vtags[0])
assert.Equal(t, 1, len(resources[0].Metadata.Artifacts))
assert.NotNil(t, resources[0].Metadata.Artifacts[0].Tags[0])
// filter 2
filters = []*model.Filter{
{
@ -50,8 +50,8 @@ func TestFetchCharts(t *testing.T) {
assert.NotZero(t, len(resources))
assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
assert.Equal(t, "harbor/harbor", resources[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.NotNil(t, resources[0].Metadata.Vtags[0])
assert.Equal(t, 1, len(resources[0].Metadata.Artifacts))
assert.NotNil(t, resources[0].Metadata.Artifacts[0].Tags[0])
}
func TestChartExist(t *testing.T) {

1
src/replication/adapter/huawei/huawei_adapter.go
View File

@ -120,7 +120,6 @@ func (a *adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceMetada
metadata := &model.ResourceMetadata{
Repository: resourceMetadata.Repository,
Vtags: resourceMetadata.Vtags,
Labels: resourceMetadata.Labels,
}
return metadata, nil
}

1
src/replication/adapter/huawei/image_registry.go
View File

@ -151,7 +151,6 @@ func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource {
resource.Metadata = &model.ResourceMetadata{
Repository: repository,
Vtags: repo.Tags,
Labels: []string{},
}
resource.Deleted = false
resource.Override = false

55
src/replication/adapter/native/adapter.go
View File

@ -22,9 +22,9 @@ import (
ierror "github.com/goharbor/harbor/src/internal/error"
"github.com/goharbor/harbor/src/pkg/registry"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/filter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"sync"
)
func init() {
@ -37,8 +37,7 @@ func init() {
var _ adp.Adapter = &Adapter{}
type factory struct {
}
type factory struct{}
// Create ...
func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
@ -53,7 +52,6 @@ func (f *factory) AdapterPattern() *model.AdapterPattern {
// Adapter implements an adapter for Docker registry. It can be used to all registries
// that implement the registry V2 API
type Adapter struct {
sync.RWMutex
registry *model.Registry
registry.Client
}
@ -127,18 +125,13 @@ func (a *Adapter) HealthCheck() (model.HealthStatus, error) {
// FetchImages ...
func (a *Adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
repositories, err := a.getRepositories(filters)
repositories, err := a.listRepositories(filters)
if err != nil {
return nil, err
}
if len(repositories) == 0 {
return nil, nil
}
for _, filter := range filters {
if err = filter.DoFilter(&repositories); err != nil {
return nil, err
}
}
var rawResources = make([]*model.Resource, len(repositories))
runner := utils.NewLimitedConcurrentRunner(adp.MaxConcurrency)
@ -148,25 +141,13 @@ func (a *Adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
index := i
repo := r
runner.AddTask(func() error {
vTags, err := a.getVTags(repo.Name)
artifacts, err := a.listArtifacts(repo.Name, filters)
if err != nil {
return fmt.Errorf("List tags for repo '%s' error: %v", repo.Name, err)
return fmt.Errorf("failed to list artifacts of repository %s: %v", repo.Name, err)
}
if len(vTags) == 0 {
if len(artifacts) == 0 {
return nil
}
for _, filter := range filters {
if err = filter.DoFilter(&vTags); err != nil {
return fmt.Errorf("Filter tags %v error: %v", vTags, err)
}
}
if len(vTags) == 0 {
return nil
}
tags := []string{}
for _, vTag := range vTags {
tags = append(tags, vTag.Name)
}
rawResources[index] = &model.Resource{
Type: model.ResourceTypeImage,
Registry: a.registry,
@ -174,7 +155,7 @@ func (a *Adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
Repository: &model.Repository{
Name: repo.Name,
},
Vtags: tags,
Artifacts: artifacts,
},
}
@ -197,7 +178,7 @@ func (a *Adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
return resources, nil
}
func (a *Adapter) getRepositories(filters []*model.Filter) ([]*adp.Repository, error) {
func (a *Adapter) listRepositories(filters []*model.Filter) ([]*model.Repository, error) {
pattern := ""
for _, filter := range filters {
if filter.Type == model.FilterTypeName {
@ -219,29 +200,27 @@ func (a *Adapter) getRepositories(filters []*model.Filter) ([]*adp.Repository, e
}
}
result := []*adp.Repository{}
var result []*model.Repository
for _, repository := range repositories {
result = append(result, &adp.Repository{
ResourceType: string(model.ResourceTypeImage),
Name: repository,
result = append(result, &model.Repository{
Name: repository,
})
}
return result, nil
return filter.DoFilterRepositories(result, filters)
}
func (a *Adapter) getVTags(repository string) ([]*adp.VTag, error) {
func (a *Adapter) listArtifacts(repository string, filters []*model.Filter) ([]*model.Artifact, error) {
tags, err := a.ListTags(repository)
if err != nil {
return nil, err
}
var result []*adp.VTag
var artifacts []*model.Artifact
for _, tag := range tags {
result = append(result, &adp.VTag{
ResourceType: string(model.ResourceTypeImage),
Name: tag,
artifacts = append(artifacts, &model.Artifact{
Tags: []string{tag},
})
}
return result, nil
return filter.DoFilterArtifacts(artifacts, filters)
}
// PingSimple checks whether the registry is available. It checks the connectivity and certificate (if TLS enabled)

88
src/replication/adapter/native/adapter_test.go
View File

@ -128,24 +128,43 @@ func Test_native_FetchImages(t *testing.T) {
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag11"},
},
},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag2", "tag13"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag11"},
},
{
Tags: []string{"tag2"},
},
{
Tags: []string{"tag13"},
},
},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/c3/3level"},
Vtags: []string{"tag4"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag4"},
},
},
},
},
},
wantErr: false,
},
{
name: "only special repository",
filters: []*model.Filter{
@ -158,7 +177,11 @@ func Test_native_FetchImages(t *testing.T) {
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag11"},
},
},
},
},
},
@ -176,18 +199,27 @@ func Test_native_FetchImages(t *testing.T) {
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag11"},
},
},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag11"},
},
},
},
},
},
wantErr: false,
},
{
name: "special repository and special tag",
filters: []*model.Filter{
@ -204,7 +236,11 @@ func Test_native_FetchImages(t *testing.T) {
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag2"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag2"},
},
},
},
},
},
@ -223,7 +259,17 @@ func Test_native_FetchImages(t *testing.T) {
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag2", "tag13"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag11"},
},
{
Tags: []string{"tag2"},
},
{
Tags: []string{"tag13"},
},
},
},
},
},
@ -241,13 +287,24 @@ func Test_native_FetchImages(t *testing.T) {
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/a1"},
Vtags: []string{"tag11"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag11"},
},
},
},
},
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag13"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag11"},
},
{
Tags: []string{"tag13"},
},
},
},
},
},
@ -269,7 +326,14 @@ func Test_native_FetchImages(t *testing.T) {
{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{Name: "test/b2"},
Vtags: []string{"tag11", "tag13"},
Artifacts: []*model.Artifact{
{
Tags: []string{"tag11"},
},
{
Tags: []string{"tag13"},
},
},
},
},
},
@ -288,7 +352,7 @@ func Test_native_FetchImages(t *testing.T) {
for i, resource := range resources {
require.NotNil(t, resource.Metadata)
assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository)
assert.ElementsMatch(t, tt.want[i].Metadata.Vtags, resource.Metadata.Vtags)
assert.ElementsMatch(t, tt.want[i].Metadata.Artifacts, resource.Metadata.Artifacts)
}
}
})

2
src/replication/event/handler.go
View File

@ -51,7 +51,7 @@ type handler struct {
func (h *handler) Handle(event *Event) error {
if event == nil || event.Resource == nil ||
event.Resource.Metadata == nil ||
len(event.Resource.Metadata.Vtags) == 0 {
len(event.Resource.Metadata.Artifacts) == 0 {
return errors.New("invalid event")
}
var policies []*model.Policy

12
src/replication/event/handler_test.go
View File

@ -278,7 +278,11 @@ func TestHandle(t *testing.T) {
Repository: &model.Repository{
Name: "library/hello-world",
},
Vtags: []string{"latest"},
Artifacts: []*model.Artifact{
{
Tags: []string{"latest"},
},
},
},
},
Type: EventTypeImagePush,
@ -292,7 +296,11 @@ func TestHandle(t *testing.T) {
Repository: &model.Repository{
Name: "library/hello-world",
},
Vtags: []string{"latest"},
Artifacts: []*model.Artifact{
{
Tags: []string{"latest"},
},
},
},
},
Type: EventTypeImageDelete,

173
src/replication/filter/artifact.go Normal file
View File

@ -0,0 +1,173 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package filter
import (
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
// DoFilterArtifacts filter the artifacts according to the filters
func DoFilterArtifacts(artifacts []*model.Artifact, filters []*model.Filter) ([]*model.Artifact, error) {
fl, err := BuildArtifactFilters(filters)
if err != nil {
return nil, err
}
return fl.Filter(artifacts)
}
// BuildArtifactFilters from the defined filters
func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) {
var fs ArtifactFilters
for _, filter := range filters {
var f ArtifactFilter
switch filter.Type {
case model.FilterTypeLabel:
f = &artifactLabelFilter{
labels: filter.Value.([]string),
}
case model.FilterTypeTag:
f = &artifactTagFilter{
pattern: filter.Value.(string),
}
}
if f != nil {
fs = append(fs, f)
}
}
return fs, nil
}
// ArtifactFilter filter the artifacts
type ArtifactFilter interface {
Filter([]*model.Artifact) ([]*model.Artifact, error)
}
// ArtifactFilters is an array of artifact filter
type ArtifactFilters []ArtifactFilter
// Filter artifacts
func (a ArtifactFilters) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
var err error
for _, filter := range a {
artifacts, err = filter.Filter(artifacts)
if err != nil {
return nil, err
}
}
return artifacts, nil
}
type artifactTypeFilter struct {
types []string
}
func (a *artifactTypeFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
if len(a.types) == 0 {
return artifacts, nil
}
var result []*model.Artifact
for _, artifact := range artifacts {
for _, t := range a.types {
if artifact.Type == t {
result = append(result, artifact)
continue
}
}
}
return result, nil
}
// filter the artifacts according to the labels. Only the artifact contains all labels defined
// in the filter is the valid one
type artifactLabelFilter struct {
labels []string
}
func (a *artifactLabelFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
if len(a.labels) == 0 {
return artifacts, nil
}
var result []*model.Artifact
for _, artifact := range artifacts {
labels := map[string]struct{}{}
for _, label := range artifact.Labels {
labels[label] = struct{}{}
}
match := true
for _, label := range a.labels {
if _, exist := labels[label]; !exist {
match = false
break
}
}
// add the artifact to the result list if it contains all labels defined for the filter
if match {
result = append(result, artifact)
}
}
return result, nil
}
// filter artifacts according to whether the artifact is tagged or untagged artifact
type artifactTaggedFilter struct {
tagged bool
}
func (a *artifactTaggedFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
var result []*model.Artifact
for _, artifact := range artifacts {
if a.tagged && len(artifact.Tags) > 0 ||
!a.tagged && len(artifact.Tags) == 0 {
result = append(result, artifact)
}
}
return result, nil
}
type artifactTagFilter struct {
pattern string
}
func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
if len(a.pattern) == 0 {
return artifacts, nil
}
var result []*model.Artifact
for _, artifact := range artifacts {
var tags []string
for _, tag := range artifact.Tags {
match, err := util.Match(a.pattern, tag)
if err != nil {
return nil, err
}
if match {
tags = append(tags, tag)
continue
}
}
if len(tags) == 0 {
continue
}
// copy a new artifact here to avoid changing the original one
result = append(result, &model.Artifact{
Type: artifact.Type,
Digest: artifact.Digest,
Labels: artifact.Labels,
Tags: tags,
})
}
return result, nil
}

253
src/replication/filter/filter.go
View File

@ -1,253 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package filter
import (
"errors"
"reflect"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication/util"
)
// const definitions
const (
FilterableTypeRepository = "repository"
FilterableTypeVTag = "vtag"
)
// FilterableType specifies the type of the filterable
type FilterableType string
// Filterable defines the methods that a filterable object must implement
type Filterable interface {
// return what the type of the filterable object is(repository or vtag)
GetFilterableType() FilterableType
// return the resource type of the filterable object(image, chart, ...)
GetResourceType() string
GetName() string
GetLabels() []string
}
// Filter defines the methods that a filter must implement
type Filter interface {
// return whether the filter is applied to the specified Filterable
ApplyTo(Filterable) bool
Filter(...Filterable) ([]Filterable, error)
}
// NewResourceTypeFilter return a Filter to filter candidates according to the resource type
func NewResourceTypeFilter(resourceType string) Filter {
return &resourceTypeFilter{
resourceType: resourceType,
}
}
// NewRepositoryNameFilter return a Filter to filter the repositories according to the name
func NewRepositoryNameFilter(pattern string) Filter {
return &nameFilter{
filterableType: FilterableTypeRepository,
pattern: pattern,
}
}
// NewVTagNameFilter return a Filter to filter the vtags according to the name
func NewVTagNameFilter(pattern string) Filter {
return &nameFilter{
filterableType: FilterableTypeVTag,
pattern: pattern,
}
}
// NewVTagLabelFilter return a Filter to filter vtags according to the label
func NewVTagLabelFilter(labels []string) Filter {
return &labelFilter{
labels: labels,
}
}
type resourceTypeFilter struct {
resourceType string
}
func (r *resourceTypeFilter) ApplyTo(filterable Filterable) bool {
if filterable == nil {
return false
}
switch filterable.GetFilterableType() {
case FilterableTypeRepository, FilterableTypeVTag:
return true
default:
return false
}
}
func (r *resourceTypeFilter) Filter(filterables ...Filterable) ([]Filterable, error) {
result := []Filterable{}
for _, filterable := range filterables {
if filterable.GetResourceType() == r.resourceType {
result = append(result, filterable)
}
}
return result, nil
}
type nameFilter struct {
filterableType FilterableType
pattern string
}
func (n *nameFilter) ApplyTo(filterable Filterable) bool {
if filterable == nil {
return false
}
if filterable.GetFilterableType() == n.filterableType {
return true
}
return false
}
func (n *nameFilter) Filter(filterables ...Filterable) ([]Filterable, error) {
result := []Filterable{}
for _, filterable := range filterables {
name := filterable.GetName()
match, err := util.Match(n.pattern, name)
if err != nil {
return nil, err
}
if match {
log.Debugf("%q matches the pattern %q of name filter", name, n.pattern)
result = append(result, filterable)
continue
}
log.Debugf("%q doesn't match the pattern %q of name filter, skip", name, n.pattern)
}
return result, nil
}
type labelFilter struct {
labels []string
}
func (l *labelFilter) ApplyTo(filterable Filterable) bool {
if filterable == nil {
return false
}
if filterable.GetFilterableType() == FilterableTypeVTag {
return true
}
return false
}
func (l *labelFilter) Filter(filterables ...Filterable) ([]Filterable, error) {
// if no specified label in the filter, just returns the input filterable
// candidate as the result
if len(l.labels) == 0 {
return filterables, nil
}
result := []Filterable{}
for _, filterable := range filterables {
labels := map[string]struct{}{}
for _, label := range filterable.GetLabels() {
labels[label] = struct{}{}
}
match := true
for _, label := range l.labels {
if _, exist := labels[label]; !exist {
match = false
break
}
}
// add the filterable to the result list if it contains
// all labels defined for the filter
if match {
result = append(result, filterable)
}
}
return result, nil
}
// DoFilter is a util function to help filter filterables easily.
// The parameter "filterables" must be a pointer points to a slice
// whose elements must be Filterable. After applying all the "filters"
// to the "filterables", the result is put back into the variable
// "filterables"
func DoFilter(filterables interface{}, filters ...Filter) error {
if filterables == nil || len(filters) == 0 {
return nil
}
value := reflect.ValueOf(filterables)
// make sure the input is a pointer
if value.Kind() != reflect.Ptr {
return errors.New("the type of input should be pointer to a Filterable slice")
}
sliceValue := value.Elem()
// make sure the input is a pointer points to a slice
if sliceValue.Type().Kind() != reflect.Slice {
return errors.New("the type of input should be pointer to a Filterable slice")
}
filterableType := reflect.TypeOf((*Filterable)(nil)).Elem()
elemType := sliceValue.Type().Elem()
// make sure the input is a pointer points to a Filterable slice
if !elemType.Implements(filterableType) {
return errors.New("the type of input should be pointer to a Filterable slice")
}
// convert the input to Filterable slice
items := []Filterable{}
for i := 0; i < sliceValue.Len(); i++ {
items = append(items, sliceValue.Index(i).Interface().(Filterable))
}
// do filter
var err error
items, err = doFilter(items, filters...)
if err != nil {
return err
}
// convert back to the origin type
result := reflect.MakeSlice(reflect.SliceOf(elemType), 0, len(items))
for _, item := range items {
result = reflect.Append(result, reflect.ValueOf(item))
}
value.Elem().Set(result)
return nil
}
func doFilter(filterables []Filterable, filters ...Filter) ([]Filterable, error) {
var appliedTo, notAppliedTo []Filterable
var err error
for _, filter := range filters {
appliedTo, notAppliedTo = nil, nil
for _, filterable := range filterables {
if filter.ApplyTo(filterable) {
appliedTo = append(appliedTo, filterable)
} else {
notAppliedTo = append(notAppliedTo, filterable)
}
}
filterables, err = filter.Filter(appliedTo...)
if err != nil {
return nil, err
}
filterables = append(filterables, notAppliedTo...)
}
return filterables, nil
}

170
src/replication/filter/filter_test.go
View File

@ -1,170 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package filter
import (
"reflect"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type fakeFilterable struct {
filterableType FilterableType
resourceType string
name string
labels []string
}
func (f *fakeFilterable) GetFilterableType() FilterableType {
return f.filterableType
}
func (f *fakeFilterable) GetResourceType() string {
return f.resourceType
}
func (f *fakeFilterable) GetName() string {
return f.name
}
func (f *fakeFilterable) GetLabels() []string {
return f.labels
}
func TestFilterOfResourceTypeFilter(t *testing.T) {
filterable := &fakeFilterable{
filterableType: FilterableTypeRepository,
resourceType: "image",
name: "library/hello-world",
}
filter := NewResourceTypeFilter("image")
result, err := filter.Filter(filterable)
require.Nil(t, nil, err)
if assert.Equal(t, 1, len(result)) {
assert.True(t, reflect.DeepEqual(filterable, result[0]))
}
filter = NewResourceTypeFilter("chart")
result, err = filter.Filter(filterable)
require.Nil(t, nil, err)
assert.Equal(t, 0, len(result))
}
func TestApplyToOfResourceTypeFilter(t *testing.T) {
filterable := &fakeFilterable{
filterableType: FilterableTypeRepository,
}
filter := NewResourceTypeFilter("image")
assert.True(t, filter.ApplyTo(filterable))
filterable.filterableType = FilterableTypeVTag
assert.True(t, filter.ApplyTo(filterable))
filterable.filterableType = FilterableType("unknown")
assert.False(t, filter.ApplyTo(filterable))
}
func TestFilterOfNameFilter(t *testing.T) {
filterable := &fakeFilterable{
name: "foo",
}
// pass the filter
filter := &nameFilter{
pattern: "*",
}
result, err := filter.Filter(filterable)
require.Nil(t, err)
if assert.Equal(t, 1, len(result)) {
assert.True(t, reflect.DeepEqual(filterable, result[0].(*fakeFilterable)))
}
// cannot pass the filter
filter.pattern = "cannotpass"
result, err = filter.Filter(filterable)
require.Nil(t, err)
assert.Equal(t, 0, len(result))
}
func TestApplyToOfNameFilter(t *testing.T) {
filterable := &fakeFilterable{
filterableType: FilterableTypeRepository,
}
filter := &nameFilter{
filterableType: FilterableTypeRepository,
}
assert.True(t, filter.ApplyTo(filterable))
filterable.filterableType = FilterableTypeVTag
assert.False(t, filter.ApplyTo(filterable))
}
func TestFilterOfLabelFilter(t *testing.T) {
filterable := &fakeFilterable{
labels: []string{"production"},
}
// pass the filter
filter := &labelFilter{
labels: []string{"production"},
}
result, err := filter.Filter(filterable)
require.Nil(t, err)
if assert.Equal(t, 1, len(result)) {
assert.True(t, reflect.DeepEqual(filterable, result[0].(*fakeFilterable)))
}
// cannot pass the filter
filter.labels = []string{"production", "ci-pass"}
result, err = filter.Filter(filterable)
require.Nil(t, err)
assert.Equal(t, 0, len(result))
}
func TestApplyToOfLabelFilter(t *testing.T) {
filterable := &fakeFilterable{
filterableType: FilterableTypeRepository,
}
filter := labelFilter{}
assert.False(t, filter.ApplyTo(filterable))
filterable.filterableType = FilterableTypeVTag
assert.True(t, filter.ApplyTo(filterable))
}
func TestDoFilter(t *testing.T) {
tag1 := &fakeFilterable{
filterableType: FilterableTypeVTag,
name: "1.0",
labels: []string{"production"},
}
tag2 := &fakeFilterable{
filterableType: FilterableTypeVTag,
name: "latest",
labels: []string{"dev"},
}
filterables := []Filterable{tag1, tag2}
filters := []Filter{
NewVTagNameFilter("*"),
NewVTagLabelFilter([]string{"production"}),
}
err := DoFilter(&filterables, filters...)
require.Nil(t, err)
if assert.Equal(t, 1, len(filterables)) {
assert.True(t, reflect.DeepEqual(tag1, filterables[0]))
}
}

89
src/replication/filter/repository.go Normal file
View File

@ -0,0 +1,89 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package filter
import (
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
)
// DoFilterRepositories filter repositories according to the filters
func DoFilterRepositories(repositories []*model.Repository, filters []*model.Filter) ([]*model.Repository, error) {
fl, err := BuildRepositoryFilters(filters)
if err != nil {
return nil, err
}
return fl.Filter(repositories)
}
// BuildRepositoryFilters from the defined filters
func BuildRepositoryFilters(filters []*model.Filter) (RepositoryFilters, error) {
var fs RepositoryFilters
for _, filter := range filters {
var f RepositoryFilter
switch filter.Type {
case model.FilterTypeName:
f = &repositoryNameFilter{
pattern: filter.Value.(string),
}
}
if f != nil {
fs = append(fs, f)
}
}
return fs, nil
}
// RepositoryFilter filter repositoreis
type RepositoryFilter interface {
Filter([]*model.Repository) ([]*model.Repository, error)
}
// RepositoryFilters is an array of repository filters
type RepositoryFilters []RepositoryFilter
// Filter repositories
func (r RepositoryFilters) Filter(repositories []*model.Repository) ([]*model.Repository, error) {
var err error
for _, filter := range r {
repositories, err = filter.Filter(repositories)
if err != nil {
return nil, err
}
}
return repositories, nil
}
type repositoryNameFilter struct {
pattern string
}
func (r *repositoryNameFilter) Filter(repositories []*model.Repository) ([]*model.Repository, error) {
if len(r.pattern) == 0 {
return repositories, nil
}
var result []*model.Repository
for _, repository := range repositories {
match, err := util.Match(r.pattern, repository.Name)
if err != nil {
return nil, err
}
if match {
result = append(result, repository)
continue
}
}
return result, nil
}

59
src/replication/filter/resource.go Normal file
View File

@ -0,0 +1,59 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package filter
import "github.com/goharbor/harbor/src/replication/model"
// DoFilterResources filter resources according to the filters
func DoFilterResources(resources []*model.Resource, filters []*model.Filter) ([]*model.Resource, error) {
repoFilters, err := BuildRepositoryFilters(filters)
if err != nil {
return nil, err
}
artFilters, err := BuildArtifactFilters(filters)
if err != nil {
return nil, err
}
var result []*model.Resource
for _, resource := range resources {
repositories, err := repoFilters.Filter([]*model.Repository{resource.Metadata.Repository})
if err != nil {
return nil, err
}
if len(repositories) == 0 {
continue
}
artifacts, err := artFilters.Filter(resource.Metadata.Artifacts)
if err != nil {
return nil, err
}
if len(artifacts) == 0 {
continue
}
result = append(result, &model.Resource{
Type: resource.Type,
Metadata: &model.ResourceMetadata{
Repository: repositories[0],
Artifacts: artifacts,
},
Registry: resource.Registry,
ExtendedInfo: resource.ExtendedInfo,
Deleted: resource.Deleted,
Override: resource.Override,
})
}
return result, nil
}

31
src/replication/model/policy.go
View File

@ -16,13 +16,10 @@ package model
import (
"fmt"
"time"
"github.com/goharbor/harbor/src/replication/filter"
"github.com/astaxie/beego/validation"
"github.com/goharbor/harbor/src/common/models"
"github.com/robfig/cron"
"time"
)
// const definition
@ -149,32 +146,6 @@ type Filter struct {
Value interface{} `json:"value"`
}
// DoFilter filter the filterables
// The parameter "filterables" must be a pointer points to a slice
// whose elements must be Filterable. After applying the filter
// to the "filterables", the result is put back into the variable
// "filterables"
func (f *Filter) DoFilter(filterables interface{}) error {
var ft filter.Filter
switch f.Type {
case FilterTypeName:
ft = filter.NewRepositoryNameFilter(f.Value.(string))
case FilterTypeTag:
ft = filter.NewVTagNameFilter(f.Value.(string))
case FilterTypeLabel:
labels, ok := f.Value.([]string)
if ok {
ft = filter.NewVTagLabelFilter(labels)
}
case FilterTypeResource:
ft = filter.NewResourceTypeFilter(f.Value.(string))
default:
return fmt.Errorf("unsupported filter type: %s", f.Type)
}
return filter.DoFilter(filterables, ft)
}
// TriggerType represents the type of trigger.
type TriggerType string

44
src/replication/model/resource.go
View File

@ -28,29 +28,6 @@ func (r ResourceType) Valid() bool {
return len(r) > 0
}
// ResourceMetadata of resource
type ResourceMetadata struct {
Repository *Repository `json:"repository"`
Vtags []string `json:"v_tags"`
// TODO the labels should be put into tag and repository level?
Labels []string `json:"labels"`
}
// GetResourceName returns the name of the resource
// TODO remove
func (r *ResourceMetadata) GetResourceName() string {
if r.Repository == nil {
return ""
}
return r.Repository.Name
}
// Repository info of the resource
type Repository struct {
Name string `json:"name"`
Metadata map[string]interface{} `json:"metadata"`
}
// Resource represents the general replicating content
type Resource struct {
Type ResourceType `json:"type"`
@ -62,3 +39,24 @@ type Resource struct {
// indicate whether the resource can be overridden
Override bool `json:"override"`
}
// ResourceMetadata of resource
type ResourceMetadata struct {
Repository *Repository `json:"repository"`
Artifacts []*Artifact `json:"artifacts"`
Vtags []string `json:"v_tags"` // deprecated, use Artifacts instead
}
// Repository info of the resource
type Repository struct {
Name string `json:"name"`
Metadata map[string]interface{} `json:"metadata"`
}
// Artifact is the individual unit that can be replicated
type Artifact struct {
Type string `json:"type"`
Digest string `json:"digest"`
Labels []string `json:"labels"`
Tags []string `json:"tags"`
}

32
src/replication/model/resource_test.go
View File

@ -13,35 +13,3 @@
// limitations under the License.
package model
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetResourceName(t *testing.T) {
r := &ResourceMetadata{}
assert.Equal(t, "", r.GetResourceName())
r = &ResourceMetadata{
Repository: &Repository{
Name: "library",
},
}
assert.Equal(t, "library", r.GetResourceName())
r = &ResourceMetadata{
Repository: &Repository{
Name: "hello-world",
},
}
assert.Equal(t, "hello-world", r.GetResourceName())
r = &ResourceMetadata{
Repository: &Repository{
Name: "library/hello-world",
},
}
assert.Equal(t, "library/hello-world", r.GetResourceName())
}

6
src/replication/operation/flow/deletion_test.go
View File

@ -39,7 +39,11 @@ func TestRunOfDeletionFlow(t *testing.T) {
Repository: &model.Repository{
Name: "library/hello-world",
},
Vtags: []string{"latest"},
Artifacts: []*model.Artifact{
{
Tags: []string{"latest"},
},
},
},
},
}

99
src/replication/operation/flow/stage.go
View File

@ -17,6 +17,7 @@ package flow
import (
"errors"
"fmt"
"github.com/goharbor/harbor/src/replication/filter"
"time"
"github.com/goharbor/harbor/src/common/utils/log"
@ -110,75 +111,12 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc
// apply the filters to the resources and returns the filtered resources
func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*model.Resource, error) {
var res []*model.Resource
for _, resource := range resources {
match := true
FILTER_LOOP:
for _, filter := range filters {
switch filter.Type {
case model.FilterTypeResource:
resourceType, ok := filter.Value.(model.ResourceType)
if !ok {
return nil, fmt.Errorf("%v is not a valid string", filter.Value)
}
if model.ResourceType(resourceType) != resource.Type {
match = false
break FILTER_LOOP
}
case model.FilterTypeName:
pattern, ok := filter.Value.(string)
if !ok {
return nil, fmt.Errorf("%v is not a valid string", filter.Value)
}
if resource.Metadata == nil {
match = false
break FILTER_LOOP
}
m, err := util.Match(pattern, resource.Metadata.Repository.Name)
if err != nil {
return nil, err
}
if !m {
match = false
break FILTER_LOOP
}
case model.FilterTypeTag:
pattern, ok := filter.Value.(string)
if !ok {
return nil, fmt.Errorf("%v is not a valid string", filter.Value)
}
if resource.Metadata == nil {
match = false
break FILTER_LOOP
}
var versions []string
for _, version := range resource.Metadata.Vtags {
m, err := util.Match(pattern, version)
if err != nil {
return nil, err
}
if m {
versions = append(versions, version)
}
}
if len(versions) == 0 {
match = false
break FILTER_LOOP
}
// NOTE: the property "Vtags" of the origin resource struct is overrided here
resource.Metadata.Vtags = versions
case model.FilterTypeLabel:
// TODO add support to label
default:
return nil, fmt.Errorf("unsupportted filter type: %v", filter.Type)
}
}
if match {
res = append(res, resource)
}
resources, err := filter.DoFilterResources(resources, filters)
if err != nil {
return nil, err
}
log.Debug("filter resources completed")
return res, nil
return resources, nil
}
// assemble the source resources by filling the registry information
@ -208,7 +146,8 @@ func assembleDestinationResources(resources []*model.Resource,
Name: replaceNamespace(resource.Metadata.Repository.Name, policy.DestNamespace),
Metadata: resource.Metadata.Repository.Metadata,
},
Vtags: resource.Metadata.Vtags,
Vtags: resource.Metadata.Vtags,
Artifacts: resource.Metadata.Artifacts,
}
result = append(result, res)
}
@ -334,16 +273,24 @@ func getResourceName(res *model.Resource) string {
if meta == nil {
return ""
}
repositoryName := meta.Repository.Name
if len(meta.Vtags) == 0 {
return repositoryName
n := 0
if len(meta.Artifacts) > 0 {
for _, artifact := range meta.Artifacts {
// contains tags
if len(artifact.Tags) > 0 {
n += len(artifact.Tags)
continue
}
// contains no tag, count digest
if len(artifact.Digest) > 0 {
n++
}
}
} else {
n = len(meta.Vtags)
}
if len(meta.Vtags) == 1 {
return repositoryName + ":[" + meta.Vtags[0] + "]"
}
return fmt.Sprintf("%s:[%s ... %d in total]", repositoryName, meta.Vtags[0], len(meta.Vtags))
return fmt.Sprintf("%s [%d in total]", meta.Repository.Name, n)
}
// repository:c namespace:n -> n/c

31
src/replication/operation/flow/stage_test.go
View File

@ -223,9 +223,11 @@ func TestFilterResources(t *testing.T) {
Repository: &model.Repository{
Name: "library/hello-world",
},
Vtags: []string{"latest"},
// TODO test labels
Labels: nil,
Artifacts: []*model.Artifact{
{
Tags: []string{"latest"},
},
},
},
Deleted: true,
Override: true,
@ -236,9 +238,14 @@ func TestFilterResources(t *testing.T) {
Repository: &model.Repository{
Name: "library/harbor",
},
Vtags: []string{"0.2.0", "0.3.0"},
// TODO test labels
Labels: nil,
Artifacts: []*model.Artifact{
{
Tags: []string{"0.2.0"},
},
{
Tags: []string{"0.3.0"},
},
},
},
Deleted: true,
Override: true,
@ -249,9 +256,11 @@ func TestFilterResources(t *testing.T) {
Repository: &model.Repository{
Name: "library/mysql",
},
Vtags: []string{"1.0"},
// TODO test labels
Labels: nil,
Artifacts: []*model.Artifact{
{
Tags: []string{"1.0"},
},
},
},
Deleted: true,
Override: true,
@ -279,8 +288,8 @@ func TestFilterResources(t *testing.T) {
require.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "library/harbor", res[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(res[0].Metadata.Vtags))
assert.Equal(t, "0.2.0", res[0].Metadata.Vtags[0])
assert.Equal(t, 1, len(res[0].Metadata.Artifacts))
assert.Equal(t, "0.2.0", res[0].Metadata.Artifacts[0].Tags[0])
}
func TestAssembleSourceResources(t *testing.T) {

6
src/replication/operation/scheduler/scheduler_test.go
View File

@ -57,8 +57,7 @@ func generateData() ([]*ScheduleItem, error) {
Repository: &model.Repository{
Name: "namespace1",
},
Vtags: []string{"latest"},
Labels: []string{"latest"},
Vtags: []string{"latest"},
},
Registry: &model.Registry{
Credential: &model.Credential{},
@ -69,8 +68,7 @@ func generateData() ([]*ScheduleItem, error) {
Repository: &model.Repository{
Name: "namespace2",
},
Vtags: []string{"v1", "v2"},
Labels: []string{"latest"},
Vtags: []string{"v1", "v2"},
},
Registry: &model.Registry{
Credential: &model.Credential{},

12
src/replication/transfer/chart/transfer.go
View File

@ -57,18 +57,18 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error {
// delete the chart on destination registry
if dst.Deleted {
return t.delete(&chart{
name: dst.Metadata.GetResourceName(),
version: dst.Metadata.Vtags[0],
name: dst.Metadata.Repository.Name,
version: dst.Metadata.Artifacts[0].Tags[0],
})
}
srcChart := &chart{
name: src.Metadata.GetResourceName(),
version: src.Metadata.Vtags[0],
name: src.Metadata.Repository.Name,
version: src.Metadata.Artifacts[0].Tags[0],
}
dstChart := &chart{
name: dst.Metadata.GetResourceName(),
version: dst.Metadata.Vtags[0],
name: dst.Metadata.Repository.Name,
version: dst.Metadata.Artifacts[0].Tags[0],
}
// copy the chart from source registry to the destination
return t.copy(srcChart, dstChart, dst.Override)

37
src/replication/transfer/image/transfer.go
View File

@ -60,22 +60,33 @@ func (t *transfer) Transfer(src *model.Resource, dst *model.Resource) error {
// delete the repository on destination registry
if dst.Deleted {
return t.delete(&repository{
repository: dst.Metadata.GetResourceName(),
tags: dst.Metadata.Vtags,
})
return t.delete(t.convert(dst))
}
srcRepo := &repository{
repository: src.Metadata.GetResourceName(),
tags: src.Metadata.Vtags,
}
dstRepo := &repository{
repository: dst.Metadata.GetResourceName(),
tags: dst.Metadata.Vtags,
}
// copy the repository from source registry to the destination
return t.copy(srcRepo, dstRepo, dst.Override)
return t.copy(t.convert(src), t.convert(dst), dst.Override)
}
func (t *transfer) convert(resource *model.Resource) *repository {
repository := &repository{
repository: resource.Metadata.Repository.Name,
}
for _, artifact := range resource.Metadata.Artifacts {
if len(artifact.Tags) > 0 {
repository.tags = append(repository.tags, artifact.Tags...)
continue
}
// no tags
if len(artifact.Digest) > 0 {
repository.tags = append(repository.tags, artifact.Digest)
}
}
if len(repository.tags) > 0 {
return repository
}
// fallback to vtags
repository.tags = resource.Metadata.Vtags
return repository
}
func (t *transfer) initialize(src *model.Resource, dst *model.Resource) error {