Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-20 23:57:42 +01:00
Code Issues Projects Releases Wiki Activity

Remove the namespace concept in replication

Browse Source 
Update the replication logic to remove the "namespace"

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2019-04-15 16:46:50 +08:00
parent d6f08dead6
commit 2f1d2257d5
50 changed files with 452 additions and 707 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/chartserver/handler_manipulation.go
View File

@ -12,8 +12,9 @@ import (
"github.com/goharbor/harbor/src/replication/model"
helm_repo "k8s.io/helm/pkg/repo"
"github.com/goharbor/harbor/src/common/utils/log"
"os"
"github.com/goharbor/harbor/src/common/utils/log"
)
// ListCharts gets the chart list under the namespace
@ -84,11 +85,8 @@ func (c *Controller) DeleteChartVersion(namespace, chartName, version string) er
Type: model.ResourceTypeChart,
Deleted: true,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: namespace,
},
Repository: &model.Repository{
Name: chartName,
Name: fmt.Sprintf("%s/%s", namespace, chartName),
},
Vtags: []string{version},
},

16
src/chartserver/reverse_proxy.go
View File

@ -4,11 +4,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/common"
hlog "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication"
rep_event "github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
"io/ioutil"
"log"
"net/http"
@ -17,6 +12,12 @@ import (
"os"
"strconv"
"strings"
"github.com/goharbor/harbor/src/common"
hlog "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication"
rep_event "github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
)
const (
@ -100,11 +101,8 @@ func modifyResponse(res *http.Response) error {
Resource: &model.Resource{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: chartUploadSplitted[0],
},
Repository: &model.Repository{
Name: chartUploadSplitted[1],
Name: fmt.Sprintf("%s/%s", chartUploadSplitted[0], chartUploadSplitted[1]),
},
Vtags: []string{chartUploadSplitted[2]},
},

88
src/core/api/registry.go
View File

@ -389,57 +389,60 @@ func (t *RegistryAPI) GetInfo() {
}
// GetNamespace get the namespace of a registry
// TODO remove
func (t *RegistryAPI) GetNamespace() {
var registry *model.Registry
var err error
/*
var registry *model.Registry
var err error
id, err := t.GetInt64FromPath(":id")
if err != nil || id < 0 {
t.HandleBadRequest(fmt.Sprintf("invalid registry ID %s", t.GetString(":id")))
return
}
if id > 0 {
registry, err = t.manager.Get(id)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("failed to get registry %d: %v", id, err))
id, err := t.GetInt64FromPath(":id")
if err != nil || id < 0 {
t.HandleBadRequest(fmt.Sprintf("invalid registry ID %s", t.GetString(":id")))
return
}
} else if id == 0 {
registry = event.GetLocalRegistry()
}
if id > 0 {
registry, err = t.manager.Get(id)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("failed to get registry %d: %v", id, err))
return
}
} else if id == 0 {
registry = event.GetLocalRegistry()
}
if registry == nil {
t.HandleNotFound(fmt.Sprintf("registry %d not found", id))
return
}
if registry == nil {
t.HandleNotFound(fmt.Sprintf("registry %d not found", id))
return
}
if !adapter.HasFactory(registry.Type) {
t.HandleInternalServerError(fmt.Sprintf("no adapter factory found for %s", registry.Type))
return
}
if !adapter.HasFactory(registry.Type) {
t.HandleInternalServerError(fmt.Sprintf("no adapter factory found for %s", registry.Type))
return
}
regFactory, err := adapter.GetFactory(registry.Type)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("fail to get adapter factory %s", registry.Type))
return
}
regAdapter, err := regFactory(registry)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("fail to get adapter %s", registry.Type))
return
}
regFactory, err := adapter.GetFactory(registry.Type)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("fail to get adapter factory %s", registry.Type))
return
}
regAdapter, err := regFactory(registry)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("fail to get adapter %s", registry.Type))
return
}
query := &model.NamespaceQuery{
Name: t.GetString("name"),
}
npResults, err := regAdapter.ListNamespaces(query)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("fail to list namespaces %s %v", registry.Type, err))
return
}
query := &model.NamespaceQuery{
Name: t.GetString("name"),
}
npResults, err := regAdapter.ListNamespaces(query)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("fail to list namespaces %s %v", registry.Type, err))
return
}
t.Data["json"] = npResults
t.ServeJSON()
t.Data["json"] = npResults
t.ServeJSON()
*/
}
// merge "SupportedResourceTypes" into "SupportedResourceFilters" for UI to render easier
@ -450,7 +453,6 @@ func process(info *model.RegistryInfo) *model.RegistryInfo {
in := &model.RegistryInfo{
Type: info.Type,
Description: info.Description,
SupportNamespace: info.SupportNamespace,
SupportedTriggers: info.SupportedTriggers,
}
filters := []*model.FilterStyle{}

2
src/core/api/replication_execution_test.go
View File

@ -89,7 +89,6 @@ func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) {
SrcRegistry: &model.Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
}, nil
}
if id == 2 {
@ -99,7 +98,6 @@ func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) {
SrcRegistry: &model.Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
}, nil
}
return nil, nil

11
src/core/api/replication_policy_test.go
View File

@ -128,7 +128,6 @@ func TestReplicationPolicyAPICreate(t *testing.T) {
SrcRegistry: &model.Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
},
},
code: http.StatusBadRequest,
@ -140,8 +139,7 @@ func TestReplicationPolicyAPICreate(t *testing.T) {
url: "/api/replication/policies",
credential: sysAdmin,
bodyJSON: &model.Policy{
Name: "policy01",
SrcNamespaces: []string{"library"},
Name: "policy01",
},
},
code: http.StatusBadRequest,
@ -157,7 +155,6 @@ func TestReplicationPolicyAPICreate(t *testing.T) {
SrcRegistry: &model.Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
},
},
code: http.StatusConflict,
@ -173,7 +170,6 @@ func TestReplicationPolicyAPICreate(t *testing.T) {
SrcRegistry: &model.Registry{
ID: 2,
},
SrcNamespaces: []string{"library"},
},
},
code: http.StatusNotFound,
@ -189,7 +185,6 @@ func TestReplicationPolicyAPICreate(t *testing.T) {
SrcRegistry: &model.Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
},
},
code: http.StatusCreated,
@ -296,7 +291,6 @@ func TestReplicationPolicyAPIUpdate(t *testing.T) {
SrcRegistry: &model.Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
},
},
code: http.StatusBadRequest,
@ -312,7 +306,6 @@ func TestReplicationPolicyAPIUpdate(t *testing.T) {
SrcRegistry: &model.Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
},
},
code: http.StatusConflict,
@ -328,7 +321,6 @@ func TestReplicationPolicyAPIUpdate(t *testing.T) {
SrcRegistry: &model.Registry{
ID: 2,
},
SrcNamespaces: []string{"library"},
},
},
code: http.StatusNotFound,
@ -344,7 +336,6 @@ func TestReplicationPolicyAPIUpdate(t *testing.T) {
SrcRegistry: &model.Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
},
},
code: http.StatusOK,

6
src/core/api/repository.go
View File

@ -333,12 +333,8 @@ func (ra *RepositoryAPI) Delete() {
Resource: &model.Resource{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: projectName,
// TODO filling the metadata
},
Repository: &model.Repository{
Name: strings.TrimPrefix(repoName, projectName+"/"),
Name: repoName,
},
Vtags: []string{tag},
},

7
src/core/service/notifications/registry/handler.go
View File

@ -119,12 +119,9 @@ func (n *NotificationHandler) Post() {
Resource: &model.Resource{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: project,
// TODO filling the metadata
},
Repository: &model.Repository{
Name: strings.TrimPrefix(repository, project+"/"),
Name: repository,
// TODO filling the metadata
},
Vtags: []string{tag},
},

6
src/replication/adapter/adapter.go
View File

@ -30,12 +30,6 @@ type Factory func(*model.Registry) (Adapter, error)
type Adapter interface {
// Info return the information of this adapter
Info() (*model.RegistryInfo, error)
// Lists the available namespaces under the specified registry with the
// provided credential/token
ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error)
// ConvertResourceMetadata converts the namespace and repository part of the resource metadata
// to the one that the adapter can handle
ConvertResourceMetadata(*model.ResourceMetadata, *model.Namespace) (*model.ResourceMetadata, error)
// PrepareForPush does the prepare work that needed for pushing/uploading the resource
// eg: create the namespace or repository
PrepareForPush(*model.Resource) error

2
src/replication/adapter/chart_registry.go
View File

@ -22,7 +22,7 @@ import (
// ChartRegistry defines the capabilities that a chart registry should have
type ChartRegistry interface {
FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error)
FetchCharts(filters []*model.Filter) ([]*model.Resource, error)
ChartExist(name, version string) (bool, error)
DownloadChart(name, version string) (io.ReadCloser, error)
UploadChart(name, version string, chart io.Reader) error

37
src/replication/adapter/dockerhub/adapter.go
View File

@ -56,8 +56,7 @@ var _ adp.Adapter = (*adapter)(nil)
// Info returns information of the registry
func (a *adapter) Info() (*model.RegistryInfo, error) {
return &model.RegistryInfo{
Type: model.RegistryTypeDockerHub,
SupportNamespace: true,
Type: model.RegistryTypeDockerHub,
SupportedResourceTypes: []model.ResourceType{
model.ResourceTypeRepository,
},
@ -78,12 +77,6 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
}, nil
}
// ConvertResourceMetadata converts the namespace and repository part of the resource metadata
// to the one that the adapter can handle
func (a *adapter) ConvertResourceMetadata(meta *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
return meta, nil
}
// PrepareForPush does the prepare work that needed for pushing/uploading the resource
// eg: create the namespace or repository
func (a *adapter) PrepareForPush(resource *model.Resource) error {
@ -93,18 +86,25 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error {
if resource.Metadata == nil {
return errors.New("the metadata of resource cannot be null")
}
if resource.Metadata.Namespace == nil {
if resource.Metadata.Repository == nil {
return errors.New("the namespace of resource cannot be null")
}
if len(resource.Metadata.Namespace.Name) == 0 {
if len(resource.Metadata.Repository.Name) == 0 {
return errors.New("the name of the namespace cannot be null")
}
namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
// Docker Hub doesn't support the repository contains no "/"
// just skip here and the following task will fail
if len(namespace) == 0 {
log.Debug("the namespace is empty, skip")
return nil
}
err := a.CreateNamespace(&model.Namespace{
Name: resource.Metadata.Namespace.Name,
Name: namespace,
})
if err != nil {
return fmt.Errorf("create namespace '%s' in DockerHub error: %v", resource.Metadata.Namespace.Name, err)
return fmt.Errorf("create namespace '%s' in DockerHub error: %v", namespace, err)
}
return nil
@ -218,7 +218,7 @@ func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) {
}
// FetchImages fetches images
func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
var repos []Repo
nameFilter, err := a.getStringFilterValue(model.FilterTypeName, filters)
if err != nil {
@ -229,11 +229,15 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*
return nil, err
}
namespaces, err := a.ListNamespaces(nil)
if err != nil {
return nil, err
}
for _, ns := range namespaces {
page := 1
pageSize := 100
for {
pageRepos, err := a.getRepos(ns, "", page, pageSize)
pageRepos, err := a.getRepos(ns.Name, "", page, pageSize)
if err != nil {
return nil, fmt.Errorf("get repos for namespace '%s' from DockerHub error: %v", ns, err)
}
@ -300,11 +304,8 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*
Type: model.ResourceTypeRepository,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: repo.Namespace,
},
Repository: &model.Repository{
Name: repo.Name,
Name: fmt.Sprintf("%s/%s", repo.Namespace, repo.Name),
},
Vtags: tags,
},

3
src/replication/adapter/dockerhub/adapter_test.go
View File

@ -39,7 +39,8 @@ func TestListNamespaces(t *testing.T) {
}
assert := assert.New(t)
adapter := getAdapter(t)
ad := getAdapter(t)
adapter := ad.(*adapter)
namespaces, err := adapter.ListNamespaces(nil)
assert.Nil(err)

84
src/replication/adapter/harbor/adapter.go
View File

@ -18,7 +18,6 @@ import (
"errors"
"fmt"
"net/http"
"strings"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
@ -91,8 +90,7 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
func (a *adapter) Info() (*model.RegistryInfo, error) {
info := &model.RegistryInfo{
Type: model.RegistryTypeHarbor,
SupportNamespace: true,
Type: model.RegistryTypeHarbor,
SupportedResourceTypes: []model.ResourceType{
model.ResourceTypeRepository,
},
@ -130,65 +128,6 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
return info, nil
}
func (a *adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Namespace, error) {
var namespaces []*model.Namespace
name := ""
if query != nil {
name = query.Name
}
projects, err := a.getProjects(name)
if err != nil {
return nil, err
}
for _, project := range projects {
namespaces = append(namespaces, &model.Namespace{
Name: project.Name,
Metadata: project.Metadata,
})
}
return namespaces, nil
}
func (a *adapter) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
if metadata == nil {
return nil, errors.New("the metadata cannot be null")
}
name := metadata.GetResourceName()
strs := strings.SplitN(name, "/", 2)
if len(strs) < 2 {
return nil, fmt.Errorf("unsupported resource name %s, at least contains one '/'", name)
}
meta := &model.ResourceMetadata{
Vtags: metadata.Vtags,
Labels: metadata.Labels,
}
meta.Namespace = &model.Namespace{
Name: strs[0],
}
if metadata.Namespace != nil {
meta.Namespace.Metadata = metadata.Namespace.Metadata
}
meta.Repository = &model.Repository{
Name: strs[1],
}
if metadata.Repository != nil {
meta.Repository.Metadata = metadata.Repository.Metadata
}
// replace the namespace if it is specified
if namespace == nil || len(namespace.Name) == 0 {
return meta, nil
}
if strings.Contains(namespace.Name, "/") {
return nil, fmt.Errorf("the namespace %s cannot contain '/'", namespace.Name)
}
meta.Namespace.Name = namespace.Name
if namespace.Metadata != nil {
meta.Namespace.Metadata = namespace.Metadata
}
return meta, nil
}
func (a *adapter) PrepareForPush(resource *model.Resource) error {
if resource == nil {
return errors.New("the resource cannot be null")
@ -196,18 +135,25 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error {
if resource.Metadata == nil {
return errors.New("the metadata of resource cannot be null")
}
if resource.Metadata.Namespace == nil {
return errors.New("the namespace of resource cannot be null")
if resource.Metadata.Repository == nil {
return errors.New("the repository of resource cannot be null")
}
if len(resource.Metadata.Namespace.Name) == 0 {
return errors.New("the name of the namespace cannot be null")
if len(resource.Metadata.Repository.Name) == 0 {
return errors.New("the name of the repository cannot be null")
}
projectName, _ := util.ParseRepository(resource.Metadata.Repository.Name)
// harbor doesn't support the repository contains no "/"
// just skip here and the following task will fail
if len(projectName) == 0 {
log.Debug("the project name is empty, skip")
return nil
}
project := &struct {
Name string `json:"project_name"`
Metadata map[string]interface{} `json:"metadata"`
}{
Name: resource.Metadata.Namespace.Name,
Metadata: resource.Metadata.Namespace.Metadata,
Name: projectName,
// TODO handle the public
}
// TODO
@ -234,7 +180,7 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error {
err := a.client.Post(a.coreServiceURL+"/api/projects", project)
if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict {
log.Debugf("got 409 when trying to create project %s", resource.Metadata.Namespace.Name)
log.Debugf("got 409 when trying to create project %s", projectName)
return nil
}
return err

53
src/replication/adapter/harbor/adapter_test.go
View File

@ -73,38 +73,6 @@ func TestInfo(t *testing.T) {
server.Close()
}
func TestListNamespaces(t *testing.T) {
// project exists
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodGet,
Pattern: "/api/projects",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "library",
"metadata": {"public":true}
},{
"name": "library1",
"metadata": {"public":true}
}]`
w.Write([]byte(data))
},
})
defer server.Close()
registry := &model.Registry{
URL: server.URL,
}
adapter, err := newAdapter(registry)
require.Nil(t, err)
npQuery := &model.NamespaceQuery{
Name: "lib",
}
namespace, err := adapter.ListNamespaces(npQuery)
require.Nil(t, err)
assert.Equal(t, 2, len(namespace))
assert.Equal(t, "library", namespace[0].Name)
assert.True(t, namespace[0].Metadata["public"].(bool))
}
func TestPrepareForPush(t *testing.T) {
server := test.NewServer(&test.RequestHandlerMapping{
Method: http.MethodPost,
@ -124,30 +92,23 @@ func TestPrepareForPush(t *testing.T) {
// nil metadata
err = adapter.PrepareForPush(&model.Resource{})
require.NotNil(t, err)
// nil namespace
// nil repository
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{},
})
require.NotNil(t, err)
// nil namespace name
// nil repository name
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{},
},
})
require.NotNil(t, err)
// nil namespace name
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{},
Repository: &model.Repository{},
},
})
require.NotNil(t, err)
// project doesn't exist
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
Repository: &model.Repository{
Name: "library/hello-world",
},
},
})
@ -170,8 +131,8 @@ func TestPrepareForPush(t *testing.T) {
require.Nil(t, err)
err = adapter.PrepareForPush(&model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
Repository: &model.Repository{
Name: "library/hello-world",
},
},
})

32
src/replication/adapter/harbor/chart_registry.go
View File

@ -24,14 +24,14 @@ import (
"strings"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/replication/model"
)
// TODO review the logic in this file
type chart struct {
Name string `json:"name"`
Name string `json:"name"`
Project string
}
func (c *chart) Match(filters []*model.Filter) (bool, error) {
@ -41,10 +41,8 @@ func (c *chart) Match(filters []*model.Filter) (bool, error) {
supportedFilters = append(supportedFilters, filter)
}
}
// trim the project part
_, name := utils.ParseRepository(c.Name)
item := &FilterItem{
Value: name,
Value: fmt.Sprintf("%s/%s", c.Project, c.Name),
}
return item.Match(supportedFilters)
}
@ -77,20 +75,28 @@ type chartVersionMetadata struct {
URLs []string `json:"urls"`
}
func (a *adapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
// TODO optimize the performance
projects, err := a.getProjects("")
if err != nil {
return nil, err
}
resources := []*model.Resource{}
for _, namespace := range namespaces {
url := fmt.Sprintf("%s/api/chartrepo/%s/charts", a.coreServiceURL, namespace)
for _, project := range projects {
url := fmt.Sprintf("%s/api/chartrepo/%s/charts", a.coreServiceURL, project.Name)
charts := []*chart{}
if err := a.client.Get(url, &charts); err != nil {
return nil, err
}
for _, chart := range charts {
chart.Project = project.Name
}
charts, err := filterCharts(charts, filters)
if err != nil {
return nil, err
}
for _, chart := range charts {
url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s", a.coreServiceURL, namespace, chart.Name)
url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s", a.coreServiceURL, project.Name, chart.Name)
chartVersions := []*chartVersion{}
if err := a.client.Get(url, &chartVersions); err != nil {
return nil, err
@ -104,18 +110,14 @@ func (a *adapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*
Type: model.ResourceTypeChart,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: namespace,
// TODO filling the metadata
},
Repository: &model.Repository{
Name: chart.Name,
Name: fmt.Sprintf("%s/%s", project.Name, chart.Name),
// TODO handle the metadata
},
Vtags: []string{version.Version},
},
})
}
}
}
return resources, nil

35
src/replication/adapter/harbor/chart_registry_test.go
View File

@ -27,6 +27,17 @@ import (
func TestFetchCharts(t *testing.T) {
server := test.NewServer([]*test.RequestHandlerMapping{
{
Method: http.MethodGet,
Pattern: "/api/projects",
Handler: func(w http.ResponseWriter, r *http.Request) {
data := `[{
"name": "library",
"metadata": {"public":true}
}]`
w.Write([]byte(data))
},
},
{
Method: http.MethodGet,
Pattern: "/api/chartrepo/library/charts/harbor",
@ -58,12 +69,30 @@ func TestFetchCharts(t *testing.T) {
}
adapter, err := newAdapter(registry)
require.Nil(t, err)
resources, err := adapter.FetchCharts([]string{"library"}, nil)
// nil filter
resources, err := adapter.FetchCharts(nil)
require.Nil(t, err)
assert.Equal(t, 2, len(resources))
assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
assert.Equal(t, "harbor", resources[0].Metadata.Repository.Name)
assert.Equal(t, "library", resources[0].Metadata.Namespace.Name)
assert.Equal(t, "library/harbor", resources[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
// not nil filter
filters := []*model.Filter{
{
Type: model.FilterTypeName,
Value: "library/*",
},
{
Type: model.FilterTypeTag,
Value: "1.0",
},
}
resources, err = adapter.FetchCharts(filters)
require.Nil(t, err)
require.Equal(t, 1, len(resources))
assert.Equal(t, model.ResourceTypeChart, resources[0].Type)
assert.Equal(t, "library/harbor", resources[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
}

33
src/replication/adapter/harbor/image_registry.go
View File

@ -16,9 +16,7 @@ package harbor
import (
"fmt"
"strings"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/replication/model"
)
@ -33,10 +31,8 @@ func (r *repository) Match(filters []*model.Filter) (bool, error) {
supportedFilters = append(supportedFilters, filter)
}
}
// trim the project part
_, name := utils.ParseRepository(r.Name)
item := &FilterItem{
Value: name,
Value: r.Name,
}
return item.Match(supportedFilters)
}
@ -58,22 +54,14 @@ func (t *tag) Match(filters []*model.Filter) (bool, error) {
return item.Match(supportedFilters)
}
func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
if len(namespaces) == 0 {
nms, err := a.ListNamespaces(nil)
if err != nil {
return nil, err
}
for _, nm := range nms {
namespaces = append(namespaces, nm.Name)
}
func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
// TODO optimize the performance
projects, err := a.getProjects("")
if err != nil {
return nil, err
}
resources := []*model.Resource{}
for _, namespace := range namespaces {
project, err := a.getProject(namespace)
if err != nil {
return nil, err
}
for _, project := range projects {
repositories := []*repository{}
url := fmt.Sprintf("%s/api/repositories?project_id=%d", a.coreServiceURL, project.ID)
if err = a.client.Get(url, &repositories); err != nil {
@ -104,12 +92,9 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*
Type: model.ResourceTypeRepository,
Registry: a.registry,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: namespace,
// TODO filling the metadata
},
Repository: &model.Repository{
Name: strings.TrimPrefix(repository.Name, namespace+"/"),
Name: repository.Name,
// TODO handle the metadata
},
Vtags: vtags,
},

27
src/replication/adapter/harbor/image_registry_test.go
View File

@ -66,26 +66,33 @@ func TestFetchImages(t *testing.T) {
}
adapter, err := newAdapter(registry)
require.Nil(t, err)
// not nil namespaces
resources, err := adapter.FetchImages([]string{"library"}, nil)
// nil filter
resources, err := adapter.FetchImages(nil)
require.Nil(t, err)
assert.Equal(t, 1, len(resources))
assert.Equal(t, model.ResourceTypeRepository, resources[0].Type)
assert.Equal(t, "hello-world", resources[0].Metadata.Repository.Name)
assert.Equal(t, "library", resources[0].Metadata.Namespace.Name)
assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name)
assert.Equal(t, 2, len(resources[0].Metadata.Vtags))
assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
assert.Equal(t, "2.0", resources[0].Metadata.Vtags[1])
// nil namespaces
resources, err = adapter.FetchImages(nil, nil)
// not nil filter
filters := []*model.Filter{
{
Type: model.FilterTypeName,
Value: "library/*",
},
{
Type: model.FilterTypeTag,
Value: "1.0",
},
}
resources, err = adapter.FetchImages(filters)
require.Nil(t, err)
assert.Equal(t, 1, len(resources))
assert.Equal(t, model.ResourceTypeRepository, resources[0].Type)
assert.Equal(t, "hello-world", resources[0].Metadata.Repository.Name)
assert.Equal(t, "library", resources[0].Metadata.Namespace.Name)
assert.Equal(t, 2, len(resources[0].Metadata.Vtags))
assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(resources[0].Metadata.Vtags))
assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0])
assert.Equal(t, "2.0", resources[0].Metadata.Vtags[1])
}
func TestDeleteManifest(t *testing.T) {

11
src/replication/adapter/huawei/huawei_adapter.go
View File

@ -10,6 +10,8 @@ import (
"regexp"
"strings"
"github.com/goharbor/harbor/src/replication/util"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
@ -110,7 +112,6 @@ func (adapter Adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Nam
// ConvertResourceMetadata convert resource metadata for Huawei SWR
func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
metadata := &model.ResourceMetadata{
Namespace: namespace,
Repository: resourceMetadata.Repository,
Vtags: resourceMetadata.Vtags,
Labels: resourceMetadata.Labels,
@ -121,12 +122,12 @@ func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceM
// PrepareForPush prepare for push to Huawei SWR
func (adapter Adapter) PrepareForPush(resource *model.Resource) error {
namespace := resource.Metadata.Namespace
ns, err := adapter.GetNamespace(namespace.Name)
namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name)
ns, err := adapter.GetNamespace(namespace)
if err != nil {
//
} else {
if ns.Name == namespace.Name {
if ns.Name == namespace {
return nil
}
}
@ -134,7 +135,7 @@ func (adapter Adapter) PrepareForPush(resource *model.Resource) error {
url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL)
namespacebyte, err := json.Marshal(struct {
Namespace string `json:"namespace"`
}{Namespace: namespace.Name})
}{Namespace: namespace})
if err != nil {
return err
}

42
src/replication/adapter/huawei/huawei_adapter_test.go
View File

@ -39,49 +39,15 @@ func TestAdapter_Info(t *testing.T) {
t.Log(info)
}
func TestAdapter_ListNamespaces(t *testing.T) {
namespaces, err := hwAdapter.ListNamespaces(&model.NamespaceQuery{Name: "o"})
if err != nil {
if strings.HasPrefix(err.Error(), "[401]") {
t.Log("huawei ak/sk is not available", err.Error())
} else {
t.Error(err)
}
} else {
for _, namespace := range namespaces {
t.Log(namespace.Name, namespace.Metadata)
}
}
}
func TestAdapter_ConvertResourceMetadata(t *testing.T) {
metadata := &model.ResourceMetadata{}
namespace := &model.Namespace{
Name: "domain_repo_new",
Metadata: make(map[string]interface{}),
}
metadata, err := hwAdapter.ConvertResourceMetadata(metadata, namespace)
if err != nil {
if strings.HasPrefix(err.Error(), "[401]") {
t.Log("huawei ak/sk is not available", err.Error())
} else {
t.Error(err)
}
} else {
t.Log("success convert resource metadata")
t.Log(metadata)
}
}
func TestAdapter_PrepareForPush(t *testing.T) {
namespace := &model.Namespace{
repository := &model.Repository{
Name: "domain_repo_new",
Metadata: make(map[string]interface{}),
}
resource := &model.Resource{}
metadata := &model.ResourceMetadata{Namespace: namespace}
metadata := &model.ResourceMetadata{
Repository: repository,
}
resource.Metadata = metadata
err := hwAdapter.PrepareForPush(resource)
if err != nil {

5
src/replication/adapter/huawei/image_registry.go
View File

@ -84,16 +84,12 @@ func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource {
info["status"] = repo.Status
info["total_range"] = repo.TotalRange
namespace := &model.Namespace{
Name: repo.NamespaceName,
}
repository := &model.Repository{
Name: repo.Name,
Metadata: info,
}
resource.ExtendedInfo = info
resource.Metadata = &model.ResourceMetadata{
Namespace: namespace,
Repository: repository,
Vtags: repo.Tags,
Labels: []string{},
@ -101,7 +97,6 @@ func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource {
resource.Deleted = false
resource.Override = false
resource.Type = model.ResourceTypeRepository
resource.URI = repo.Path
return &resource
}

2
src/replication/adapter/image_registry.go
View File

@ -40,7 +40,7 @@ const (
// ImageRegistry defines the capabilities that an image registry should have
type ImageRegistry interface {
FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error)
FetchImages(filters []*model.Filter) ([]*model.Resource, error)
ManifestExist(repository, reference string) (exist bool, digest string, err error)
PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error)
PushManifest(repository, reference, mediaType string, payload []byte) error

30
src/replication/adapter/native/adapter.go
View File

@ -15,8 +15,6 @@
package native
import (
"errors"
"github.com/goharbor/harbor/src/common/utils/log"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
@ -75,33 +73,5 @@ func (native) Info() (info *model.RegistryInfo, err error) {
}, nil
}
// ConvertResourceMetadata convert src to dst resource
func (native) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
if metadata == nil {
return nil, errors.New("the metadata cannot be null")
}
var result = &model.ResourceMetadata{
Namespace: metadata.Namespace,
Repository: metadata.Repository,
Vtags: metadata.Vtags,
}
// if dest namespace is set, rename metadata namespace
if namespace != nil {
result.Namespace = namespace
}
result.Repository = &model.Repository{Name: result.GetResourceName()}
result.Namespace = nil
return result, nil
}
// PrepareForPush nothing need to do.
func (native) PrepareForPush(*model.Resource) error { return nil }
// ListNamespaces native registry no namespaces, so list empty array.
func (native) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) {
return []*model.Namespace{}, nil
}

89
src/replication/adapter/native/adapter_test.go
View File

@ -50,81 +50,6 @@ func Test_native_Info(t *testing.T) {
assert.Equal(t, model.ResourceTypeRepository, info.SupportedResourceTypes[0])
}
func Test_native_ConvertResourceMetadata(t *testing.T) {
var registry = &model.Registry{URL: "abc"}
var reg, _ = adp.NewDefaultImageRegistry(registry)
var adapter = native{
DefaultImageRegistry: reg,
registry: registry,
}
assert.NotNil(t, adapter)
tests := []struct {
name string
metadata *model.ResourceMetadata
namespace *model.Namespace
want string
wantErr bool
}{
{name: "nil metadata", metadata: nil, wantErr: true},
{
name: "2 level",
metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{Name: "a"},
Repository: &model.Repository{Name: "b"},
},
namespace: nil,
want: "a/b",
wantErr: false,
},
{
name: "2 level rename reomte repository",
metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{Name: "a"},
Repository: &model.Repository{Name: "b"},
},
namespace: &model.Namespace{Name: "c"},
want: "c/b",
wantErr: false,
},
{
name: "3 level",
metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{Name: "a"},
Repository: &model.Repository{Name: "b/c"},
},
namespace: nil,
want: "a/b/c",
wantErr: false,
},
{
name: "3 level rename reomte repository",
metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{Name: "a"},
Repository: &model.Repository{Name: "b/c"},
},
namespace: &model.Namespace{Name: "d"},
want: "d/b/c",
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var convert, err = adapter.ConvertResourceMetadata(tt.metadata, tt.namespace)
if tt.wantErr {
assert.NotNil(t, err)
} else {
assert.Nil(t, err)
assert.NotNil(t, convert)
assert.Nil(t, convert.Namespace)
assert.Equal(t, tt.want, convert.Repository.Name)
assert.Equal(t, tt.want, convert.GetResourceName())
}
})
}
}
func Test_native_PrepareForPush(t *testing.T) {
var registry = &model.Registry{URL: "abc"}
var reg, _ = adp.NewDefaultImageRegistry(registry)
@ -137,17 +62,3 @@ func Test_native_PrepareForPush(t *testing.T) {
var err = adapter.PrepareForPush(nil)
assert.Nil(t, err)
}
func Test_native_ListNamespaces(t *testing.T) {
var registry = &model.Registry{URL: "abc"}
var reg, _ = adp.NewDefaultImageRegistry(registry)
var adapter = native{
DefaultImageRegistry: reg,
registry: registry,
}
assert.NotNil(t, adapter)
var ns, err = adapter.ListNamespaces(nil)
assert.Nil(t, err)
assert.NotNil(t, ns)
}

7
src/replication/adapter/native/image_registry.go
View File

@ -15,7 +15,6 @@
package native
import (
"errors"
"net/http"
"strings"
@ -27,11 +26,7 @@ import (
var _ adp.ImageRegistry = native{}
func (n native) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
if len(namespaces) > 0 {
return nil, errors.New("native registry adapter not support namespace")
}
func (n native) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
nameFilterPattern := ""
tagFilterPattern := ""
for _, filter := range filters {

31
src/replication/adapter/native/image_registry_test.go
View File

@ -65,13 +65,11 @@ func Test_native_FetchImages(t *testing.T) {
assert.NotNil(t, adapter)
tests := []struct {
name string
namespaces []string
filters []*model.Filter
want []*model.Resource
wantErr bool
name string
filters []*model.Filter
want []*model.Resource
wantErr bool
}{
{name: "namespace not empty", namespaces: []string{"ns"}, wantErr: true},
// TODO: discuss: should we report error if not found in the source native registry.
// {
// name: "repository not exist",
@ -119,8 +117,7 @@ func Test_native_FetchImages(t *testing.T) {
wantErr: false,
},
{
name: "only special repository",
namespaces: []string{},
name: "only special repository",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
@ -138,8 +135,7 @@ func Test_native_FetchImages(t *testing.T) {
wantErr: false,
},
{
name: "only special tag",
namespaces: []string{},
name: "only special tag",
filters: []*model.Filter{
{
Type: model.FilterTypeTag,
@ -163,8 +159,7 @@ func Test_native_FetchImages(t *testing.T) {
wantErr: false,
},
{
name: "special repository and special tag",
namespaces: []string{},
name: "special repository and special tag",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
@ -187,8 +182,7 @@ func Test_native_FetchImages(t *testing.T) {
wantErr: false,
},
{
name: "only wildcard repository",
namespaces: []string{},
name: "only wildcard repository",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
@ -206,8 +200,7 @@ func Test_native_FetchImages(t *testing.T) {
wantErr: false,
},
{
name: "only wildcard tag",
namespaces: []string{},
name: "only wildcard tag",
filters: []*model.Filter{
{
Type: model.FilterTypeTag,
@ -231,8 +224,7 @@ func Test_native_FetchImages(t *testing.T) {
wantErr: false,
},
{
name: "wildcard repository and wildcard tag",
namespaces: []string{},
name: "wildcard repository and wildcard tag",
filters: []*model.Filter{
{
Type: model.FilterTypeName,
@ -257,7 +249,7 @@ func Test_native_FetchImages(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var resources, err = adapter.FetchImages(tt.namespaces, tt.filters)
var resources, err = adapter.FetchImages(tt.filters)
if tt.wantErr {
require.Len(t, resources, 0)
require.NotNil(t, err)
@ -265,7 +257,6 @@ func Test_native_FetchImages(t *testing.T) {
require.Equal(t, len(tt.want), len(resources))
for i, resource := range resources {
require.NotNil(t, resource.Metadata)
assert.Equal(t, tt.want[i].Metadata.Namespace, resource.Metadata.Namespace)
assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository)
assert.Equal(t, tt.want[i].Metadata.Vtags, resource.Metadata.Vtags)
}

1
src/replication/dao/models/policy.go
View File

@ -9,7 +9,6 @@ type RepPolicy struct {
Description string `orm:"column(description)" json:"description"`
Creator string `orm:"column(creator)" json:"creator"`
SrcRegistryID int64 `orm:"column(src_registry_id)" json:"src_registry_id"`
SrcNamespaces string `orm:"column(src_namespaces)" json:"src_namespaces"`
DestRegistryID int64 `orm:"column(dest_registry_id)" json:"dest_registry_id"`
DestNamespace string `orm:"column(dest_namespace)" json:"dest_namespace"`
Override bool `orm:"column(override)" json:"override"`

6
src/replication/dao/policy_test.go
View File

@ -17,7 +17,6 @@ var (
Description: "Policy Description",
Creator: "someone",
SrcRegistryID: 123,
SrcNamespaces: "ns1,ns2,ns3",
DestRegistryID: 456,
DestNamespace: "target_ns",
ReplicateDeletion: true,
@ -33,7 +32,6 @@ var (
Description: "Policy Description",
Creator: "someone",
SrcRegistryID: 123,
SrcNamespaces: "ns1,ns2,ns3",
DestRegistryID: 456,
DestNamespace: "target_ns",
ReplicateDeletion: true,
@ -49,7 +47,6 @@ var (
Description: "Policy Description",
Creator: "someone",
SrcRegistryID: 123,
SrcNamespaces: "ns1,ns2,ns3",
DestRegistryID: 456,
DestNamespace: "target_ns",
ReplicateDeletion: true,
@ -126,7 +123,6 @@ func TestGetPolicies(t *testing.T) {
assert.Equal(t, tt.wantPolicies[i].Description, gotPolicy.Description)
assert.Equal(t, tt.wantPolicies[i].Creator, gotPolicy.Creator)
assert.Equal(t, tt.wantPolicies[i].SrcRegistryID, gotPolicy.SrcRegistryID)
assert.Equal(t, tt.wantPolicies[i].SrcNamespaces, gotPolicy.SrcNamespaces)
assert.Equal(t, tt.wantPolicies[i].DestRegistryID, gotPolicy.DestRegistryID)
assert.Equal(t, tt.wantPolicies[i].DestNamespace, gotPolicy.DestNamespace)
assert.Equal(t, tt.wantPolicies[i].ReplicateDeletion, gotPolicy.ReplicateDeletion)
@ -163,7 +159,6 @@ func TestGetRepPolicy(t *testing.T) {
assert.Equal(t, tt.wantPolicy.Description, gotPolicy.Description)
assert.Equal(t, tt.wantPolicy.Creator, gotPolicy.Creator)
assert.Equal(t, tt.wantPolicy.SrcRegistryID, gotPolicy.SrcRegistryID)
assert.Equal(t, tt.wantPolicy.SrcNamespaces, gotPolicy.SrcNamespaces)
assert.Equal(t, tt.wantPolicy.DestRegistryID, gotPolicy.DestRegistryID)
assert.Equal(t, tt.wantPolicy.DestNamespace, gotPolicy.DestNamespace)
assert.Equal(t, tt.wantPolicy.ReplicateDeletion, gotPolicy.ReplicateDeletion)
@ -202,7 +197,6 @@ func TestGetRepPolicyByName(t *testing.T) {
assert.Equal(t, tt.wantPolicy.Description, gotPolicy.Description)
assert.Equal(t, tt.wantPolicy.Creator, gotPolicy.Creator)
assert.Equal(t, tt.wantPolicy.SrcRegistryID, gotPolicy.SrcRegistryID)
assert.Equal(t, tt.wantPolicy.SrcNamespaces, gotPolicy.SrcNamespaces)
assert.Equal(t, tt.wantPolicy.DestRegistryID, gotPolicy.DestRegistryID)
assert.Equal(t, tt.wantPolicy.DestNamespace, gotPolicy.DestNamespace)
assert.Equal(t, tt.wantPolicy.ReplicateDeletion, gotPolicy.ReplicateDeletion)

54
src/replication/event/handler.go
View File

@ -18,6 +18,8 @@ import (
"errors"
"fmt"
"github.com/goharbor/harbor/src/replication/util"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication/config"
"github.com/goharbor/harbor/src/replication/model"
@ -55,10 +57,9 @@ func (h *handler) Handle(event *Event) error {
var policies []*model.Policy
var err error
switch event.Type {
case EventTypeImagePush, EventTypeChartUpload:
policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace.Name)
case EventTypeImageDelete, EventTypeChartDelete:
policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace.Name, true)
case EventTypeImagePush, EventTypeChartUpload,
EventTypeImageDelete, EventTypeChartDelete:
policies, err = h.getRelatedPolicies(event.Resource)
default:
return fmt.Errorf("unsupported event type %s", event.Type)
}
@ -84,22 +85,15 @@ func (h *handler) Handle(event *Event) error {
return nil
}
func (h *handler) getRelatedPolicies(namespace string, replicateDeletion ...bool) ([]*model.Policy, error) {
func (h *handler) getRelatedPolicies(resource *model.Resource) ([]*model.Policy, error) {
_, policies, err := h.policyCtl.List()
if err != nil {
return nil, err
}
result := []*model.Policy{}
for _, policy := range policies {
exist := false
for _, ns := range policy.SrcNamespaces {
if ns == namespace {
exist = true
break
}
}
// contains no namespace that is specified
if !exist {
// disabled
if !policy.Enabled {
continue
}
// has no trigger
@ -110,8 +104,16 @@ func (h *handler) getRelatedPolicies(namespace string, replicateDeletion ...bool
if policy.Trigger.Type != model.TriggerTypeEventBased {
continue
}
// whether replicate deletion doesn't match the value specified in policy
if len(replicateDeletion) > 0 && replicateDeletion[0] != policy.Deletion {
// doesn't replicate deletion
if resource.Deleted && !policy.Deletion {
continue
}
// doesn't match the name filter
m, err := match(policy.Filters, resource)
if err != nil {
return nil, err
}
if !m {
continue
}
result = append(result, policy)
@ -119,6 +121,26 @@ func (h *handler) getRelatedPolicies(namespace string, replicateDeletion ...bool
return result, nil
}
// TODO unify the match logic with other?
func match(filters []*model.Filter, resource *model.Resource) (bool, error) {
match := true
repository := resource.Metadata.Repository.Name
for _, filter := range filters {
if filter.Type != model.FilterTypeName {
continue
}
m, err := util.Match(filter.Value.(string), repository)
if err != nil {
return false, err
}
if !m {
match = false
break
}
}
return match, nil
}
// PopulateRegistries populates the source registry and destination registry properties for policy
func PopulateRegistries(registryMgr registry.Manager, policy *model.Policy) error {
if policy == nil {

105
src/replication/event/handler_test.go
View File

@ -59,34 +59,76 @@ func (f *fakedPolicyController) Create(*model.Policy) (int64, error) {
func (f *fakedPolicyController) List(...*model.PolicyQuery) (int64, []*model.Policy, error) {
polices := []*model.Policy{
{
ID: 1,
SrcNamespaces: []string{"test"},
Deletion: false,
ID: 1,
Enabled: true,
Deletion: true,
Trigger: &model.Trigger{
Type: model.TriggerTypeEventBased,
},
Filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "test/*",
},
},
},
// nil trigger
{
ID: 2,
SrcNamespaces: []string{"library"},
Deletion: true,
Trigger: nil,
ID: 2,
Enabled: true,
Deletion: true,
Trigger: nil,
Filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "library/*",
},
},
},
// doesn't replicate deletion
{
ID: 3,
SrcNamespaces: []string{"library"},
Deletion: false,
ID: 3,
Enabled: true,
Deletion: false,
Trigger: &model.Trigger{
Type: model.TriggerTypeEventBased,
},
Filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "library/*",
},
},
},
// replicate deletion
{
ID: 4,
SrcNamespaces: []string{"library"},
Deletion: true,
ID: 4,
Enabled: true,
Deletion: true,
Trigger: &model.Trigger{
Type: model.TriggerTypeEventBased,
},
Filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "library/*",
},
},
},
// disabled
{
ID: 5,
Enabled: false,
Deletion: true,
Trigger: &model.Trigger{
Type: model.TriggerTypeEventBased,
},
Filters: []*model.Filter{
{
Type: model.FilterTypeName,
Value: "library/*",
},
},
},
}
return int64(len(polices)), polices, nil
@ -134,13 +176,26 @@ func TestGetRelatedPolicies(t *testing.T) {
handler := &handler{
policyCtl: &fakedPolicyController{},
}
policies, err := handler.getRelatedPolicies("library")
policies, err := handler.getRelatedPolicies(&model.Resource{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/hello-world",
},
},
})
require.Nil(t, err)
assert.Equal(t, 2, len(policies))
assert.Equal(t, int64(3), policies[0].ID)
assert.Equal(t, int64(4), policies[1].ID)
policies, err = handler.getRelatedPolicies("library", true)
policies, err = handler.getRelatedPolicies(&model.Resource{
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/hello-world",
},
},
Deleted: true,
})
require.Nil(t, err)
assert.Equal(t, 1, len(policies))
assert.Equal(t, int64(4), policies[0].ID)
@ -159,11 +214,8 @@ func TestHandle(t *testing.T) {
err = handler.Handle(&Event{
Resource: &model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{},
},
@ -176,11 +228,8 @@ func TestHandle(t *testing.T) {
err = handler.Handle(&Event{
Resource: &model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{"latest"},
},
@ -193,11 +242,8 @@ func TestHandle(t *testing.T) {
err = handler.Handle(&Event{
Resource: &model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{"latest"},
},
@ -210,11 +256,8 @@ func TestHandle(t *testing.T) {
err = handler.Handle(&Event{
Resource: &model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{"latest"},
},

2
src/replication/model/namespace.go
View File

@ -14,6 +14,8 @@
package model
// TODO remove
// Namespace represents the full path of resource isolation unit;
// if the namespace has hierarchical structure, e.g organization->team,
// it should be converted to organization.team

3
src/replication/model/policy.go
View File

@ -42,8 +42,7 @@ type Policy struct {
// TODO consider to remove this property?
Creator string `json:"creator"`
// source
SrcRegistry *Registry `json:"src_registry"`
SrcNamespaces []string `json:"src_namespaces"`
SrcRegistry *Registry `json:"src_registry"`
// destination
// TODO rename to DstRegistry
DestRegistry *Registry `json:"dest_registry"`

4
src/replication/model/policy_test.go
View File

@ -61,7 +61,6 @@ func TestValidOfPolicy(t *testing.T) {
DestRegistry: &Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
Filters: []*Filter{
{
Type: "invalid_type",
@ -80,7 +79,6 @@ func TestValidOfPolicy(t *testing.T) {
DestRegistry: &Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
Filters: []*Filter{
{
Type: FilterTypeName,
@ -103,7 +101,6 @@ func TestValidOfPolicy(t *testing.T) {
DestRegistry: &Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
Filters: []*Filter{
{
Type: FilterTypeName,
@ -126,7 +123,6 @@ func TestValidOfPolicy(t *testing.T) {
DestRegistry: &Registry{
ID: 1,
},
SrcNamespaces: []string{"library"},
Filters: []*Filter{
{
Type: FilterTypeName,

1
src/replication/model/registry.go
View File

@ -109,7 +109,6 @@ type FilterStyle struct {
type RegistryInfo struct {
Type RegistryType `json:"type"`
Description string `json:"description"`
SupportNamespace bool `json:"support_namespace"`
SupportedResourceTypes []ResourceType `json:"-"`
SupportedResourceFilters []*FilterStyle `json:"supported_resource_filters"`
SupportedTriggers []TriggerType `json:"supported_triggers"`

16
src/replication/model/resource.go
View File

@ -30,7 +30,6 @@ func (r ResourceType) Valid() bool {
// ResourceMetadata of resource
type ResourceMetadata struct {
Namespace *Namespace `json:"namespace"`
Repository *Repository `json:"repository"`
Vtags []string `json:"v_tags"`
// TODO the labels should be put into tag and repository level?
@ -38,18 +37,12 @@ type ResourceMetadata struct {
}
// GetResourceName returns the name of the resource
// TODO remove
func (r *ResourceMetadata) GetResourceName() string {
name := ""
if r.Namespace != nil && len(r.Namespace.Name) > 0 {
name += r.Namespace.Name
if r.Repository == nil {
return ""
}
if r.Repository != nil && len(r.Repository.Name) > 0 {
if len(name) > 0 {
name += "/"
}
name += r.Repository.Name
}
return name
return r.Repository.Name
}
// Repository info of the resource
@ -61,7 +54,6 @@ type Repository struct {
// Resource represents the general replicating content
type Resource struct {
Type ResourceType `json:"type"`
URI string `json:"uri"`
Metadata *ResourceMetadata `json:"metadata"`
Registry *Registry `json:"registry"`
ExtendedInfo map[string]interface{} `json:"extended_info"`

7
src/replication/model/resource_test.go
View File

@ -25,7 +25,7 @@ func TestGetResourceName(t *testing.T) {
assert.Equal(t, "", r.GetResourceName())
r = &ResourceMetadata{
Namespace: &Namespace{
Repository: &Repository{
Name: "library",
},
}
@ -39,11 +39,8 @@ func TestGetResourceName(t *testing.T) {
assert.Equal(t, "hello-world", r.GetResourceName())
r = &ResourceMetadata{
Namespace: &Namespace{
Name: "library",
},
Repository: &Repository{
Name: "hello-world",
Name: "library/hello-world",
},
}
assert.Equal(t, "library/hello-world", r.GetResourceName())

3
src/replication/operation/controller.go
View File

@ -58,6 +58,9 @@ type controller struct {
}
func (c *controller) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
if !policy.Enabled {
return 0, fmt.Errorf("the policy %d is diabled", policy.ID)
}
// only support one tag if the resource is specified as we append the tag name as a filter
// when creating the flow in function "createFlow"
if resource != nil && len(resource.Metadata.Vtags) != 1 {

49
src/replication/operation/controller_test.go
View File

@ -129,48 +129,20 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
SupportedTriggers: []model.TriggerType{model.TriggerTypeManual},
}, nil
}
func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) {
return nil, nil
}
func (f *fakedAdapter) ConvertResourceMetadata(*model.ResourceMetadata, *model.Namespace) (*model.ResourceMetadata, error) {
return &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
},
Vtags: []string{"latest"},
}, nil
}
func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
return nil
}
func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
return model.Healthy, nil
}
func (f *fakedAdapter) GetNamespace(ns string) (*model.Namespace, error) {
var namespace *model.Namespace
if ns == "library" {
namespace = &model.Namespace{
Name: "library",
Metadata: map[string]interface{}{
"public": true,
},
}
}
return namespace, nil
}
func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) ([]*model.Resource, error) {
return []*model.Resource{
{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{"latest"},
},
@ -205,11 +177,8 @@ func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter)
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "harbor",
Name: "library/harbor",
},
Vtags: []string{"0.2.0"},
},
@ -239,7 +208,6 @@ func TestStartReplication(t *testing.T) {
err := adapter.RegisterFactory(model.RegistryTypeHarbor, fakedAdapterFactory)
require.Nil(t, err)
config.Config = &config.Configuration{}
// the resource contains Vtags whose length isn't 1
policy := &model.Policy{
SrcRegistry: &model.Registry{
Type: model.RegistryTypeHarbor,
@ -251,15 +219,18 @@ func TestStartReplication(t *testing.T) {
resource := &model.Resource{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{"1.0", "2.0"},
},
}
// policy is disabled
_, err = ctl.StartReplication(policy, resource, model.TriggerTypeEventBased)
require.NotNil(t, err)
policy.Enabled = true
// the resource contains Vtags whose length isn't 1
_, err = ctl.StartReplication(policy, resource, model.TriggerTypeEventBased)
require.NotNil(t, err)

6
src/replication/operation/flow/copy.go
View File

@ -57,10 +57,8 @@ func (c *copyFlow) Run(interface{}) (int, error) {
log.Infof("no resources need to be replicated for the execution %d, skip", c.executionID)
return 0, nil
}
dstResources, err := assembleDestinationResources(dstAdapter, srcResources, c.policy)
if err != nil {
return 0, err
}
dstResources := assembleDestinationResources(srcResources, c.policy)
if err = prepareForPush(dstAdapter, dstResources); err != nil {
return 0, err

10
src/replication/operation/flow/deletion.go
View File

@ -43,10 +43,6 @@ func NewDeletionFlow(executionMgr execution.Manager, scheduler scheduler.Schedul
}
func (d *deletionFlow) Run(interface{}) (int, error) {
_, dstAdapter, err := initialize(d.policy)
if err != nil {
return 0, err
}
// filling the registry information
for _, resource := range d.resources {
resource.Registry = d.policy.SrcRegistry
@ -60,10 +56,8 @@ func (d *deletionFlow) Run(interface{}) (int, error) {
log.Infof("no resources need to be replicated for the execution %d, skip", d.executionID)
return 0, nil
}
dstResources, err := assembleDestinationResources(dstAdapter, srcResources, d.policy)
if err != nil {
return 0, err
}
dstResources := assembleDestinationResources(srcResources, d.policy)
items, err := preprocess(d.scheduler, srcResources, dstResources)
if err != nil {

5
src/replication/operation/flow/deletion_test.go
View File

@ -36,11 +36,8 @@ func TestRunOfDeletionFlow(t *testing.T) {
resources := []*model.Resource{
{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{"latest"},
},

48
src/replication/operation/flow/stage.go
View File

@ -87,14 +87,14 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ImageRegistry interface")
}
res, err = reg.FetchImages(policy.SrcNamespaces, filters)
res, err = reg.FetchImages(filters)
} else if typ == model.ResourceTypeChart {
// charts
reg, ok := adapter.(adp.ChartRegistry)
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ChartRegistry interface")
}
res, err = reg.FetchCharts(policy.SrcNamespaces, filters)
res, err = reg.FetchCharts(filters)
} else {
return nil, fmt.Errorf("unsupported resource type %s", typ)
}
@ -135,8 +135,7 @@ func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*m
match = false
break FILTER_LOOP
}
// TODO filter only the repository part?
m, err := util.Match(pattern, resource.Metadata.GetResourceName())
m, err := util.Match(pattern, resource.Metadata.Repository.Name)
if err != nil {
return nil, err
}
@ -184,39 +183,35 @@ func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*m
}
// assemble the destination resources by filling the metadata, registry and override properties
func assembleDestinationResources(adapter adp.Adapter, resources []*model.Resource,
policy *model.Policy) ([]*model.Resource, error) {
func assembleDestinationResources(resources []*model.Resource,
policy *model.Policy) []*model.Resource {
var result []*model.Resource
var namespace *model.Namespace
if len(policy.DestNamespace) > 0 {
namespace = &model.Namespace{
Name: policy.DestNamespace,
}
}
for _, resource := range resources {
metadata, err := adapter.ConvertResourceMetadata(resource.Metadata, namespace)
if err != nil {
return nil, fmt.Errorf("failed to convert the resource metadata of %s: %v", resource.Metadata.GetResourceName(), err)
}
res := &model.Resource{
Type: resource.Type,
Metadata: metadata,
Registry: policy.DestRegistry,
ExtendedInfo: resource.ExtendedInfo,
Deleted: resource.Deleted,
Override: policy.Override,
}
res.Metadata = &model.ResourceMetadata{
Repository: &model.Repository{
Name: replaceNamespace(resource.Metadata.Repository.Name, policy.DestNamespace),
Metadata: resource.Metadata.Repository.Metadata,
},
Vtags: resource.Metadata.Vtags,
}
result = append(result, res)
}
log.Debug("assemble the destination resources completed")
return result, nil
return result
}
// do the prepare work for pushing/uploading the resources: create the namespace or repository
func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error {
// TODO need to consider how to handle that both contains public/private namespace
for _, resource := range resources {
name := resource.Metadata.GetResourceName()
name := resource.Metadata.Repository.Name
if err := adapter.PrepareForPush(resource); err != nil {
return fmt.Errorf("failed to do the prepare work for pushing/uploading %s: %v", name, err)
}
@ -318,12 +313,23 @@ func getResourceName(res *model.Resource) string {
return ""
}
if len(meta.Vtags) == 0 {
return meta.GetResourceName()
return meta.Repository.Name
}
if len(meta.Vtags) <= 5 {
return meta.GetResourceName() + ":[" + strings.Join(meta.Vtags, ",") + "]"
return meta.Repository.Name + ":[" + strings.Join(meta.Vtags, ",") + "]"
}
return fmt.Sprintf("%s:[%s ... %d in total]", meta.GetResourceName(), strings.Join(meta.Vtags[:5], ","), len(meta.Vtags))
}
// repository:c namespace:n -> n/c
// repository:b/c namespace:n -> n/c
// repository:a/b/c namespace:n -> n/c
func replaceNamespace(repository string, namespace string) string {
if len(namespace) == 0 {
return repository
}
_, rest := util.ParseRepository(repository)
return fmt.Sprintf("%s/%s", namespace, rest)
}

98
src/replication/operation/flow/stage_test.go
View File

@ -45,15 +45,6 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) {
SupportedTriggers: []model.TriggerType{model.TriggerTypeManual},
}, nil
}
func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) {
return nil, nil
}
func (f *fakedAdapter) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) {
if namespace != nil {
metadata.Namespace = namespace
}
return metadata, nil
}
func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
return nil
@ -61,28 +52,13 @@ func (f *fakedAdapter) PrepareForPush(*model.Resource) error {
func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) {
return model.Healthy, nil
}
func (f *fakedAdapter) GetNamespace(ns string) (*model.Namespace, error) {
var namespace *model.Namespace
if ns == "library" {
namespace = &model.Namespace{
Name: "library",
Metadata: map[string]interface{}{
"public": true,
},
}
}
return namespace, nil
}
func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) ([]*model.Resource, error) {
func (f *fakedAdapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
return []*model.Resource{
{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{"latest"},
},
@ -112,16 +88,13 @@ func (f *fakedAdapter) PullBlob(repository, digest string) (size int64, blob io.
func (f *fakedAdapter) PushBlob(repository, digest string, size int64, blob io.Reader) error {
return nil
}
func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
func (f *fakedAdapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
return []*model.Resource{
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "harbor",
Name: "library/harbor",
},
Vtags: []string{"0.2.0"},
},
@ -240,11 +213,8 @@ func TestFilterResources(t *testing.T) {
{
Type: model.ResourceTypeRepository,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{"latest"},
// TODO test labels
@ -256,11 +226,8 @@ func TestFilterResources(t *testing.T) {
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "harbor",
Name: "library/harbor",
},
Vtags: []string{"0.2.0", "0.3.0"},
// TODO test labels
@ -272,11 +239,8 @@ func TestFilterResources(t *testing.T) {
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "mysql",
Name: "library/mysql",
},
Vtags: []string{"1.0"},
// TODO test labels
@ -307,23 +271,18 @@ func TestFilterResources(t *testing.T) {
res, err := filterResources(resources, filters)
require.Nil(t, err)
assert.Equal(t, 1, len(res))
assert.Equal(t, "library", res[0].Metadata.Namespace.Name)
assert.Equal(t, "harbor", res[0].Metadata.Repository.Name)
assert.Equal(t, "library/harbor", res[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(res[0].Metadata.Vtags))
assert.Equal(t, "0.2.0", res[0].Metadata.Vtags[0])
}
func TestAssembleDestinationResources(t *testing.T) {
adapter := &fakedAdapter{}
resources := []*model.Resource{
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{"latest"},
},
@ -335,12 +294,10 @@ func TestAssembleDestinationResources(t *testing.T) {
DestNamespace: "test",
Override: true,
}
res, err := assembleDestinationResources(adapter, resources, policy)
require.Nil(t, err)
res := assembleDestinationResources(resources, policy)
assert.Equal(t, 1, len(res))
assert.Equal(t, model.ResourceTypeChart, res[0].Type)
assert.Equal(t, "hello-world", res[0].Metadata.Repository.Name)
assert.Equal(t, "test", res[0].Metadata.Namespace.Name)
assert.Equal(t, "test/hello-world", res[0].Metadata.Repository.Name)
assert.Equal(t, 1, len(res[0].Metadata.Vtags))
assert.Equal(t, "latest", res[0].Metadata.Vtags[0])
}
@ -351,11 +308,8 @@ func TestPreprocess(t *testing.T) {
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "library/hello-world",
},
Vtags: []string{"latest"},
},
@ -366,11 +320,8 @@ func TestPreprocess(t *testing.T) {
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "test",
},
Repository: &model.Repository{
Name: "hello-world",
Name: "test/hello-world",
},
Vtags: []string{"latest"},
},
@ -409,3 +360,26 @@ func TestSchedule(t *testing.T) {
require.Nil(t, err)
assert.Equal(t, 1, n)
}
func TestReplaceNamespace(t *testing.T) {
// empty namespace
repository := "c"
namespace := ""
result := replaceNamespace(repository, namespace)
assert.Equal(t, "c", result)
// repository contains no "/"
repository = "c"
namespace = "n"
result = replaceNamespace(repository, namespace)
assert.Equal(t, "n/c", result)
// repository contains only one "/"
repository = "b/c"
namespace = "n"
result = replaceNamespace(repository, namespace)
assert.Equal(t, "n/c", result)
// repository contains more than one "/"
repository = "a/b/c"
namespace = "n"
result = replaceNamespace(repository, namespace)
assert.Equal(t, "n/c", result)
}

4
src/replication/operation/scheduler/scheduler_test.go
View File

@ -50,7 +50,7 @@ func TestStop(t *testing.T) {
func generateData() ([]*ScheduleItem, error) {
srcResource := &model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Repository: &model.Repository{
Name: "namespace1",
},
Vtags: []string{"latest"},
@ -62,7 +62,7 @@ func generateData() ([]*ScheduleItem, error) {
}
destResource := &model.Resource{
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Repository: &model.Repository{
Name: "namespace2",
},
Vtags: []string{"v1", "v2"},

11
src/replication/policy/manager/manager.go
View File

@ -18,7 +18,6 @@ import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/goharbor/harbor/src/common/utils/log"
@ -58,19 +57,14 @@ func convertFromPersistModel(policy *persist_models.RepPolicy) (*model.Policy, e
}
}
// 1. parse SrcNamespaces to array
if len(policy.SrcNamespaces) > 0 {
ply.SrcNamespaces = strings.Split(policy.SrcNamespaces, ",")
}
// 2. parse Filters
// parse Filters
filters, err := parseFilters(policy.Filters)
if err != nil {
return nil, err
}
ply.Filters = filters
// 3. parse Trigger
// parse Trigger
trigger, err := parseTrigger(policy.Trigger)
if err != nil {
return nil, err
@ -90,7 +84,6 @@ func convertToPersistModel(policy *model.Policy) (*persist_models.RepPolicy, err
Name: policy.Name,
Description: policy.Description,
Creator: policy.Creator,
SrcNamespaces: strings.Join(policy.SrcNamespaces, ","),
DestNamespace: policy.DestNamespace,
Override: policy.Override,
Enabled: policy.Enabled,

6
src/replication/policy/manager/manager_test.go
View File

@ -54,7 +54,6 @@ func Test_convertFromPersistModel(t *testing.T) {
Description: "Policy Description",
Creator: "someone",
SrcRegistryID: 123,
SrcNamespaces: "ns1,ns2,ns3",
DestRegistryID: 456,
DestNamespace: "target_ns",
ReplicateDeletion: true,
@ -70,7 +69,6 @@ func Test_convertFromPersistModel(t *testing.T) {
SrcRegistry: &model.Registry{
ID: 123,
},
SrcNamespaces: []string{"ns1", "ns2", "ns3"},
DestRegistry: &model.Registry{
ID: 456,
},
@ -103,7 +101,6 @@ func Test_convertFromPersistModel(t *testing.T) {
assert.Equal(t, tt.want.Description, got.Description)
assert.Equal(t, tt.want.Creator, got.Creator)
assert.Equal(t, tt.want.SrcRegistry.ID, got.SrcRegistry.ID)
assert.Equal(t, tt.want.SrcNamespaces, got.SrcNamespaces)
assert.Equal(t, tt.want.DestRegistry.ID, got.DestRegistry.ID)
assert.Equal(t, tt.want.DestNamespace, got.DestNamespace)
assert.Equal(t, tt.want.Deletion, got.Deletion)
@ -133,7 +130,6 @@ func Test_convertToPersistModel(t *testing.T) {
SrcRegistry: &model.Registry{
ID: 123,
},
SrcNamespaces: []string{"ns1", "ns2", "ns3"},
DestRegistry: &model.Registry{
ID: 456,
},
@ -149,7 +145,6 @@ func Test_convertToPersistModel(t *testing.T) {
Description: "Policy Description",
Creator: "someone",
SrcRegistryID: 123,
SrcNamespaces: "ns1,ns2,ns3",
DestRegistryID: 456,
DestNamespace: "target_ns",
ReplicateDeletion: true,
@ -175,7 +170,6 @@ func Test_convertToPersistModel(t *testing.T) {
assert.Equal(t, tt.want.Description, got.Description)
assert.Equal(t, tt.want.Creator, got.Creator)
assert.Equal(t, tt.want.SrcRegistryID, got.SrcRegistryID)
assert.Equal(t, tt.want.SrcNamespaces, got.SrcNamespaces)
assert.Equal(t, tt.want.DestRegistryID, got.DestRegistryID)
assert.Equal(t, tt.want.DestNamespace, got.DestNamespace)
assert.Equal(t, tt.want.ReplicateDeletion, got.ReplicateDeletion)

4
src/replication/replication.go
View File

@ -28,9 +28,9 @@ import (
"github.com/goharbor/harbor/src/replication/registry"
// register the Harbor adapter
_ "github.com/goharbor/harbor/src/replication/adapter/dockerhub"
// register the DockerHub adapter
_ "github.com/goharbor/harbor/src/replication/adapter/harbor"
// register the DockerHub adapter
_ "github.com/goharbor/harbor/src/replication/adapter/dockerhub"
// register the Native adapter
_ "github.com/goharbor/harbor/src/replication/adapter/native"
)

7
src/replication/transfer/chart/transfer_test.go
View File

@ -29,16 +29,13 @@ import (
type fakeRegistry struct{}
func (f *fakeRegistry) FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
func (f *fakeRegistry) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) {
return []*model.Resource{
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Namespace: &model.Namespace{
Name: "library",
},
Repository: &model.Repository{
Name: "harbor",
Name: "library/harbor",
},
Vtags: []string{"0.2.0"},
},

2
src/replication/transfer/repository/transfer_test.go
View File

@ -32,7 +32,7 @@ import (
type fakeRegistry struct{}
func (f *fakeRegistry) FetchImages([]string, []*model.Filter) ([]*model.Resource, error) {
func (f *fakeRegistry) FetchImages([]*model.Filter) ([]*model.Resource, error) {
return nil, nil
}

17
src/replication/util/util.go
View File

@ -17,6 +17,7 @@ package util
import (
"net/http"
"path/filepath"
"strings"
"github.com/goharbor/harbor/src/common/utils/registry"
)
@ -30,3 +31,19 @@ func Match(pattern, str string) (bool, error) {
func GetHTTPTransport(insecure bool) *http.Transport {
return registry.GetHTTPTransport(insecure)
}
// ParseRepository parses the "repository" provided into two parts: namespace and the rest
// the string before the last "/" is the namespace part
// c -> [,c]
// b/c -> [b,c]
// a/b/c -> [a/b,c]
func ParseRepository(repository string) (string, string) {
if len(repository) == 0 {
return "", ""
}
index := strings.LastIndex(repository, "/")
if index == -1 {
return "", repository
}
return repository[:index], repository[index+1:]
}

23
src/replication/util/util_test.go
View File

@ -82,3 +82,26 @@ func TestGetHTTPTransport(t *testing.T) {
transport = GetHTTPTransport(false)
assert.False(t, transport.TLSClientConfig.InsecureSkipVerify)
}
func TestParseRepository(t *testing.T) {
// empty repository
repository := ""
namespace, rest := ParseRepository(repository)
assert.Equal(t, "", namespace)
assert.Equal(t, "", rest)
// repository contains no "/"
repository = "c"
namespace, rest = ParseRepository(repository)
assert.Equal(t, "", namespace)
assert.Equal(t, "c", rest)
// repository contains only one "/"
repository = "b/c"
namespace, rest = ParseRepository(repository)
assert.Equal(t, "b", namespace)
assert.Equal(t, "c", rest)
// repository contains more than one "/"
repository = "a/b/c"
namespace, rest = ParseRepository(repository)
assert.Equal(t, "a/b", namespace)
assert.Equal(t, "c", rest)
}