diff --git a/src/chartserver/handler_manipulation.go b/src/chartserver/handler_manipulation.go index ffcdc3435..9250f2476 100644 --- a/src/chartserver/handler_manipulation.go +++ b/src/chartserver/handler_manipulation.go @@ -12,8 +12,9 @@ import ( "github.com/goharbor/harbor/src/replication/model" helm_repo "k8s.io/helm/pkg/repo" - "github.com/goharbor/harbor/src/common/utils/log" "os" + + "github.com/goharbor/harbor/src/common/utils/log" ) // ListCharts gets the chart list under the namespace @@ -84,11 +85,8 @@ func (c *Controller) DeleteChartVersion(namespace, chartName, version string) er Type: model.ResourceTypeChart, Deleted: true, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: namespace, - }, Repository: &model.Repository{ - Name: chartName, + Name: fmt.Sprintf("%s/%s", namespace, chartName), }, Vtags: []string{version}, }, diff --git a/src/chartserver/reverse_proxy.go b/src/chartserver/reverse_proxy.go index f8d5842d2..058eef566 100644 --- a/src/chartserver/reverse_proxy.go +++ b/src/chartserver/reverse_proxy.go @@ -4,11 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/goharbor/harbor/src/common" - hlog "github.com/goharbor/harbor/src/common/utils/log" - "github.com/goharbor/harbor/src/replication" - rep_event "github.com/goharbor/harbor/src/replication/event" - "github.com/goharbor/harbor/src/replication/model" "io/ioutil" "log" "net/http" @@ -17,6 +12,12 @@ import ( "os" "strconv" "strings" + + "github.com/goharbor/harbor/src/common" + hlog "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/replication" + rep_event "github.com/goharbor/harbor/src/replication/event" + "github.com/goharbor/harbor/src/replication/model" ) const ( @@ -100,11 +101,8 @@ func modifyResponse(res *http.Response) error { Resource: &model.Resource{ Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: chartUploadSplitted[0], - }, Repository: &model.Repository{ - Name: chartUploadSplitted[1], + Name: fmt.Sprintf("%s/%s", chartUploadSplitted[0], chartUploadSplitted[1]), }, Vtags: []string{chartUploadSplitted[2]}, }, diff --git a/src/core/api/registry.go b/src/core/api/registry.go index 2e1a55a22..d9cc7cf1b 100644 --- a/src/core/api/registry.go +++ b/src/core/api/registry.go @@ -389,57 +389,60 @@ func (t *RegistryAPI) GetInfo() { } // GetNamespace get the namespace of a registry +// TODO remove func (t *RegistryAPI) GetNamespace() { - var registry *model.Registry - var err error + /* + var registry *model.Registry + var err error - id, err := t.GetInt64FromPath(":id") - if err != nil || id < 0 { - t.HandleBadRequest(fmt.Sprintf("invalid registry ID %s", t.GetString(":id"))) - return - } - if id > 0 { - registry, err = t.manager.Get(id) - if err != nil { - t.HandleInternalServerError(fmt.Sprintf("failed to get registry %d: %v", id, err)) + id, err := t.GetInt64FromPath(":id") + if err != nil || id < 0 { + t.HandleBadRequest(fmt.Sprintf("invalid registry ID %s", t.GetString(":id"))) return } - } else if id == 0 { - registry = event.GetLocalRegistry() - } + if id > 0 { + registry, err = t.manager.Get(id) + if err != nil { + t.HandleInternalServerError(fmt.Sprintf("failed to get registry %d: %v", id, err)) + return + } + } else if id == 0 { + registry = event.GetLocalRegistry() + } - if registry == nil { - t.HandleNotFound(fmt.Sprintf("registry %d not found", id)) - return - } + if registry == nil { + t.HandleNotFound(fmt.Sprintf("registry %d not found", id)) + return + } - if !adapter.HasFactory(registry.Type) { - t.HandleInternalServerError(fmt.Sprintf("no adapter factory found for %s", registry.Type)) - return - } + if !adapter.HasFactory(registry.Type) { + t.HandleInternalServerError(fmt.Sprintf("no adapter factory found for %s", registry.Type)) + return + } - regFactory, err := adapter.GetFactory(registry.Type) - if err != nil { - t.HandleInternalServerError(fmt.Sprintf("fail to get adapter factory %s", registry.Type)) - return - } - regAdapter, err := regFactory(registry) - if err != nil { - t.HandleInternalServerError(fmt.Sprintf("fail to get adapter %s", registry.Type)) - return - } + regFactory, err := adapter.GetFactory(registry.Type) + if err != nil { + t.HandleInternalServerError(fmt.Sprintf("fail to get adapter factory %s", registry.Type)) + return + } + regAdapter, err := regFactory(registry) + if err != nil { + t.HandleInternalServerError(fmt.Sprintf("fail to get adapter %s", registry.Type)) + return + } - query := &model.NamespaceQuery{ - Name: t.GetString("name"), - } - npResults, err := regAdapter.ListNamespaces(query) - if err != nil { - t.HandleInternalServerError(fmt.Sprintf("fail to list namespaces %s %v", registry.Type, err)) - return - } + query := &model.NamespaceQuery{ + Name: t.GetString("name"), + } + npResults, err := regAdapter.ListNamespaces(query) + if err != nil { + t.HandleInternalServerError(fmt.Sprintf("fail to list namespaces %s %v", registry.Type, err)) + return + } - t.Data["json"] = npResults - t.ServeJSON() + t.Data["json"] = npResults + t.ServeJSON() + */ } // merge "SupportedResourceTypes" into "SupportedResourceFilters" for UI to render easier @@ -450,7 +453,6 @@ func process(info *model.RegistryInfo) *model.RegistryInfo { in := &model.RegistryInfo{ Type: info.Type, Description: info.Description, - SupportNamespace: info.SupportNamespace, SupportedTriggers: info.SupportedTriggers, } filters := []*model.FilterStyle{} diff --git a/src/core/api/replication_execution_test.go b/src/core/api/replication_execution_test.go index ec581d447..c6ed3fdfb 100644 --- a/src/core/api/replication_execution_test.go +++ b/src/core/api/replication_execution_test.go @@ -89,7 +89,6 @@ func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) { SrcRegistry: &model.Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, }, nil } if id == 2 { @@ -99,7 +98,6 @@ func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) { SrcRegistry: &model.Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, }, nil } return nil, nil diff --git a/src/core/api/replication_policy_test.go b/src/core/api/replication_policy_test.go index b84445c08..654eb909b 100644 --- a/src/core/api/replication_policy_test.go +++ b/src/core/api/replication_policy_test.go @@ -128,7 +128,6 @@ func TestReplicationPolicyAPICreate(t *testing.T) { SrcRegistry: &model.Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, }, }, code: http.StatusBadRequest, @@ -140,8 +139,7 @@ func TestReplicationPolicyAPICreate(t *testing.T) { url: "/api/replication/policies", credential: sysAdmin, bodyJSON: &model.Policy{ - Name: "policy01", - SrcNamespaces: []string{"library"}, + Name: "policy01", }, }, code: http.StatusBadRequest, @@ -157,7 +155,6 @@ func TestReplicationPolicyAPICreate(t *testing.T) { SrcRegistry: &model.Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, }, }, code: http.StatusConflict, @@ -173,7 +170,6 @@ func TestReplicationPolicyAPICreate(t *testing.T) { SrcRegistry: &model.Registry{ ID: 2, }, - SrcNamespaces: []string{"library"}, }, }, code: http.StatusNotFound, @@ -189,7 +185,6 @@ func TestReplicationPolicyAPICreate(t *testing.T) { SrcRegistry: &model.Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, }, }, code: http.StatusCreated, @@ -296,7 +291,6 @@ func TestReplicationPolicyAPIUpdate(t *testing.T) { SrcRegistry: &model.Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, }, }, code: http.StatusBadRequest, @@ -312,7 +306,6 @@ func TestReplicationPolicyAPIUpdate(t *testing.T) { SrcRegistry: &model.Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, }, }, code: http.StatusConflict, @@ -328,7 +321,6 @@ func TestReplicationPolicyAPIUpdate(t *testing.T) { SrcRegistry: &model.Registry{ ID: 2, }, - SrcNamespaces: []string{"library"}, }, }, code: http.StatusNotFound, @@ -344,7 +336,6 @@ func TestReplicationPolicyAPIUpdate(t *testing.T) { SrcRegistry: &model.Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, }, }, code: http.StatusOK, diff --git a/src/core/api/repository.go b/src/core/api/repository.go index 000207ae5..af281fda8 100644 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -333,12 +333,8 @@ func (ra *RepositoryAPI) Delete() { Resource: &model.Resource{ Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: projectName, - // TODO filling the metadata - }, Repository: &model.Repository{ - Name: strings.TrimPrefix(repoName, projectName+"/"), + Name: repoName, }, Vtags: []string{tag}, }, diff --git a/src/core/service/notifications/registry/handler.go b/src/core/service/notifications/registry/handler.go index eb670a122..a9cc1de6a 100644 --- a/src/core/service/notifications/registry/handler.go +++ b/src/core/service/notifications/registry/handler.go @@ -119,12 +119,9 @@ func (n *NotificationHandler) Post() { Resource: &model.Resource{ Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: project, - // TODO filling the metadata - }, Repository: &model.Repository{ - Name: strings.TrimPrefix(repository, project+"/"), + Name: repository, + // TODO filling the metadata }, Vtags: []string{tag}, }, diff --git a/src/replication/adapter/adapter.go b/src/replication/adapter/adapter.go index 9c080aa46..5aba12704 100644 --- a/src/replication/adapter/adapter.go +++ b/src/replication/adapter/adapter.go @@ -30,12 +30,6 @@ type Factory func(*model.Registry) (Adapter, error) type Adapter interface { // Info return the information of this adapter Info() (*model.RegistryInfo, error) - // Lists the available namespaces under the specified registry with the - // provided credential/token - ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) - // ConvertResourceMetadata converts the namespace and repository part of the resource metadata - // to the one that the adapter can handle - ConvertResourceMetadata(*model.ResourceMetadata, *model.Namespace) (*model.ResourceMetadata, error) // PrepareForPush does the prepare work that needed for pushing/uploading the resource // eg: create the namespace or repository PrepareForPush(*model.Resource) error diff --git a/src/replication/adapter/chart_registry.go b/src/replication/adapter/chart_registry.go index 5008feb08..fef80e18e 100644 --- a/src/replication/adapter/chart_registry.go +++ b/src/replication/adapter/chart_registry.go @@ -22,7 +22,7 @@ import ( // ChartRegistry defines the capabilities that a chart registry should have type ChartRegistry interface { - FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) + FetchCharts(filters []*model.Filter) ([]*model.Resource, error) ChartExist(name, version string) (bool, error) DownloadChart(name, version string) (io.ReadCloser, error) UploadChart(name, version string, chart io.Reader) error diff --git a/src/replication/adapter/dockerhub/adapter.go b/src/replication/adapter/dockerhub/adapter.go index 2b1068b4f..39df186a3 100644 --- a/src/replication/adapter/dockerhub/adapter.go +++ b/src/replication/adapter/dockerhub/adapter.go @@ -56,8 +56,7 @@ var _ adp.Adapter = (*adapter)(nil) // Info returns information of the registry func (a *adapter) Info() (*model.RegistryInfo, error) { return &model.RegistryInfo{ - Type: model.RegistryTypeDockerHub, - SupportNamespace: true, + Type: model.RegistryTypeDockerHub, SupportedResourceTypes: []model.ResourceType{ model.ResourceTypeRepository, }, @@ -78,12 +77,6 @@ func (a *adapter) Info() (*model.RegistryInfo, error) { }, nil } -// ConvertResourceMetadata converts the namespace and repository part of the resource metadata -// to the one that the adapter can handle -func (a *adapter) ConvertResourceMetadata(meta *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) { - return meta, nil -} - // PrepareForPush does the prepare work that needed for pushing/uploading the resource // eg: create the namespace or repository func (a *adapter) PrepareForPush(resource *model.Resource) error { @@ -93,18 +86,25 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error { if resource.Metadata == nil { return errors.New("the metadata of resource cannot be null") } - if resource.Metadata.Namespace == nil { + if resource.Metadata.Repository == nil { return errors.New("the namespace of resource cannot be null") } - if len(resource.Metadata.Namespace.Name) == 0 { + if len(resource.Metadata.Repository.Name) == 0 { return errors.New("the name of the namespace cannot be null") } + namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name) + // Docker Hub doesn't support the repository contains no "/" + // just skip here and the following task will fail + if len(namespace) == 0 { + log.Debug("the namespace is empty, skip") + return nil + } err := a.CreateNamespace(&model.Namespace{ - Name: resource.Metadata.Namespace.Name, + Name: namespace, }) if err != nil { - return fmt.Errorf("create namespace '%s' in DockerHub error: %v", resource.Metadata.Namespace.Name, err) + return fmt.Errorf("create namespace '%s' in DockerHub error: %v", namespace, err) } return nil @@ -218,7 +218,7 @@ func (a *adapter) getNamespace(namespace string) (*model.Namespace, error) { } // FetchImages fetches images -func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) { +func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { var repos []Repo nameFilter, err := a.getStringFilterValue(model.FilterTypeName, filters) if err != nil { @@ -229,11 +229,15 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]* return nil, err } + namespaces, err := a.ListNamespaces(nil) + if err != nil { + return nil, err + } for _, ns := range namespaces { page := 1 pageSize := 100 for { - pageRepos, err := a.getRepos(ns, "", page, pageSize) + pageRepos, err := a.getRepos(ns.Name, "", page, pageSize) if err != nil { return nil, fmt.Errorf("get repos for namespace '%s' from DockerHub error: %v", ns, err) } @@ -300,11 +304,8 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]* Type: model.ResourceTypeRepository, Registry: a.registry, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: repo.Namespace, - }, Repository: &model.Repository{ - Name: repo.Name, + Name: fmt.Sprintf("%s/%s", repo.Namespace, repo.Name), }, Vtags: tags, }, diff --git a/src/replication/adapter/dockerhub/adapter_test.go b/src/replication/adapter/dockerhub/adapter_test.go index 3cee23365..70a09ad51 100644 --- a/src/replication/adapter/dockerhub/adapter_test.go +++ b/src/replication/adapter/dockerhub/adapter_test.go @@ -39,7 +39,8 @@ func TestListNamespaces(t *testing.T) { } assert := assert.New(t) - adapter := getAdapter(t) + ad := getAdapter(t) + adapter := ad.(*adapter) namespaces, err := adapter.ListNamespaces(nil) assert.Nil(err) diff --git a/src/replication/adapter/harbor/adapter.go b/src/replication/adapter/harbor/adapter.go index 58e9658a4..fae8852f4 100644 --- a/src/replication/adapter/harbor/adapter.go +++ b/src/replication/adapter/harbor/adapter.go @@ -18,7 +18,6 @@ import ( "errors" "fmt" "net/http" - "strings" common_http "github.com/goharbor/harbor/src/common/http" "github.com/goharbor/harbor/src/common/http/modifier" @@ -91,8 +90,7 @@ func newAdapter(registry *model.Registry) (*adapter, error) { func (a *adapter) Info() (*model.RegistryInfo, error) { info := &model.RegistryInfo{ - Type: model.RegistryTypeHarbor, - SupportNamespace: true, + Type: model.RegistryTypeHarbor, SupportedResourceTypes: []model.ResourceType{ model.ResourceTypeRepository, }, @@ -130,65 +128,6 @@ func (a *adapter) Info() (*model.RegistryInfo, error) { return info, nil } -func (a *adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Namespace, error) { - var namespaces []*model.Namespace - name := "" - if query != nil { - name = query.Name - } - projects, err := a.getProjects(name) - if err != nil { - return nil, err - } - - for _, project := range projects { - namespaces = append(namespaces, &model.Namespace{ - Name: project.Name, - Metadata: project.Metadata, - }) - } - - return namespaces, nil -} - -func (a *adapter) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) { - if metadata == nil { - return nil, errors.New("the metadata cannot be null") - } - name := metadata.GetResourceName() - strs := strings.SplitN(name, "/", 2) - if len(strs) < 2 { - return nil, fmt.Errorf("unsupported resource name %s, at least contains one '/'", name) - } - meta := &model.ResourceMetadata{ - Vtags: metadata.Vtags, - Labels: metadata.Labels, - } - meta.Namespace = &model.Namespace{ - Name: strs[0], - } - if metadata.Namespace != nil { - meta.Namespace.Metadata = metadata.Namespace.Metadata - } - meta.Repository = &model.Repository{ - Name: strs[1], - } - if metadata.Repository != nil { - meta.Repository.Metadata = metadata.Repository.Metadata - } - // replace the namespace if it is specified - if namespace == nil || len(namespace.Name) == 0 { - return meta, nil - } - if strings.Contains(namespace.Name, "/") { - return nil, fmt.Errorf("the namespace %s cannot contain '/'", namespace.Name) - } - meta.Namespace.Name = namespace.Name - if namespace.Metadata != nil { - meta.Namespace.Metadata = namespace.Metadata - } - return meta, nil -} func (a *adapter) PrepareForPush(resource *model.Resource) error { if resource == nil { return errors.New("the resource cannot be null") @@ -196,18 +135,25 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error { if resource.Metadata == nil { return errors.New("the metadata of resource cannot be null") } - if resource.Metadata.Namespace == nil { - return errors.New("the namespace of resource cannot be null") + if resource.Metadata.Repository == nil { + return errors.New("the repository of resource cannot be null") } - if len(resource.Metadata.Namespace.Name) == 0 { - return errors.New("the name of the namespace cannot be null") + if len(resource.Metadata.Repository.Name) == 0 { + return errors.New("the name of the repository cannot be null") + } + projectName, _ := util.ParseRepository(resource.Metadata.Repository.Name) + // harbor doesn't support the repository contains no "/" + // just skip here and the following task will fail + if len(projectName) == 0 { + log.Debug("the project name is empty, skip") + return nil } project := &struct { Name string `json:"project_name"` Metadata map[string]interface{} `json:"metadata"` }{ - Name: resource.Metadata.Namespace.Name, - Metadata: resource.Metadata.Namespace.Metadata, + Name: projectName, + // TODO handle the public } // TODO @@ -234,7 +180,7 @@ func (a *adapter) PrepareForPush(resource *model.Resource) error { err := a.client.Post(a.coreServiceURL+"/api/projects", project) if httpErr, ok := err.(*common_http.Error); ok && httpErr.Code == http.StatusConflict { - log.Debugf("got 409 when trying to create project %s", resource.Metadata.Namespace.Name) + log.Debugf("got 409 when trying to create project %s", projectName) return nil } return err diff --git a/src/replication/adapter/harbor/adapter_test.go b/src/replication/adapter/harbor/adapter_test.go index 1deaf0712..d9ccab1c9 100644 --- a/src/replication/adapter/harbor/adapter_test.go +++ b/src/replication/adapter/harbor/adapter_test.go @@ -73,38 +73,6 @@ func TestInfo(t *testing.T) { server.Close() } -func TestListNamespaces(t *testing.T) { - // project exists - server := test.NewServer(&test.RequestHandlerMapping{ - Method: http.MethodGet, - Pattern: "/api/projects", - Handler: func(w http.ResponseWriter, r *http.Request) { - data := `[{ - "name": "library", - "metadata": {"public":true} - },{ - "name": "library1", - "metadata": {"public":true} - }]` - w.Write([]byte(data)) - }, - }) - defer server.Close() - registry := &model.Registry{ - URL: server.URL, - } - adapter, err := newAdapter(registry) - require.Nil(t, err) - npQuery := &model.NamespaceQuery{ - Name: "lib", - } - namespace, err := adapter.ListNamespaces(npQuery) - require.Nil(t, err) - assert.Equal(t, 2, len(namespace)) - assert.Equal(t, "library", namespace[0].Name) - assert.True(t, namespace[0].Metadata["public"].(bool)) -} - func TestPrepareForPush(t *testing.T) { server := test.NewServer(&test.RequestHandlerMapping{ Method: http.MethodPost, @@ -124,30 +92,23 @@ func TestPrepareForPush(t *testing.T) { // nil metadata err = adapter.PrepareForPush(&model.Resource{}) require.NotNil(t, err) - // nil namespace + // nil repository err = adapter.PrepareForPush(&model.Resource{ Metadata: &model.ResourceMetadata{}, }) require.NotNil(t, err) - // nil namespace name + // nil repository name err = adapter.PrepareForPush(&model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{}, - }, - }) - require.NotNil(t, err) - // nil namespace name - err = adapter.PrepareForPush(&model.Resource{ - Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{}, + Repository: &model.Repository{}, }, }) require.NotNil(t, err) // project doesn't exist err = adapter.PrepareForPush(&model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", + Repository: &model.Repository{ + Name: "library/hello-world", }, }, }) @@ -170,8 +131,8 @@ func TestPrepareForPush(t *testing.T) { require.Nil(t, err) err = adapter.PrepareForPush(&model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", + Repository: &model.Repository{ + Name: "library/hello-world", }, }, }) diff --git a/src/replication/adapter/harbor/chart_registry.go b/src/replication/adapter/harbor/chart_registry.go index 6b672fc3b..66acfe2cd 100644 --- a/src/replication/adapter/harbor/chart_registry.go +++ b/src/replication/adapter/harbor/chart_registry.go @@ -24,14 +24,14 @@ import ( "strings" common_http "github.com/goharbor/harbor/src/common/http" - "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/replication/model" ) // TODO review the logic in this file type chart struct { - Name string `json:"name"` + Name string `json:"name"` + Project string } func (c *chart) Match(filters []*model.Filter) (bool, error) { @@ -41,10 +41,8 @@ func (c *chart) Match(filters []*model.Filter) (bool, error) { supportedFilters = append(supportedFilters, filter) } } - // trim the project part - _, name := utils.ParseRepository(c.Name) item := &FilterItem{ - Value: name, + Value: fmt.Sprintf("%s/%s", c.Project, c.Name), } return item.Match(supportedFilters) } @@ -77,20 +75,28 @@ type chartVersionMetadata struct { URLs []string `json:"urls"` } -func (a *adapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) { +func (a *adapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) { + // TODO optimize the performance + projects, err := a.getProjects("") + if err != nil { + return nil, err + } resources := []*model.Resource{} - for _, namespace := range namespaces { - url := fmt.Sprintf("%s/api/chartrepo/%s/charts", a.coreServiceURL, namespace) + for _, project := range projects { + url := fmt.Sprintf("%s/api/chartrepo/%s/charts", a.coreServiceURL, project.Name) charts := []*chart{} if err := a.client.Get(url, &charts); err != nil { return nil, err } + for _, chart := range charts { + chart.Project = project.Name + } charts, err := filterCharts(charts, filters) if err != nil { return nil, err } for _, chart := range charts { - url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s", a.coreServiceURL, namespace, chart.Name) + url := fmt.Sprintf("%s/api/chartrepo/%s/charts/%s", a.coreServiceURL, project.Name, chart.Name) chartVersions := []*chartVersion{} if err := a.client.Get(url, &chartVersions); err != nil { return nil, err @@ -104,18 +110,14 @@ func (a *adapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]* Type: model.ResourceTypeChart, Registry: a.registry, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: namespace, - // TODO filling the metadata - }, Repository: &model.Repository{ - Name: chart.Name, + Name: fmt.Sprintf("%s/%s", project.Name, chart.Name), + // TODO handle the metadata }, Vtags: []string{version.Version}, }, }) } - } } return resources, nil diff --git a/src/replication/adapter/harbor/chart_registry_test.go b/src/replication/adapter/harbor/chart_registry_test.go index adb9268bd..a2830666d 100644 --- a/src/replication/adapter/harbor/chart_registry_test.go +++ b/src/replication/adapter/harbor/chart_registry_test.go @@ -27,6 +27,17 @@ import ( func TestFetchCharts(t *testing.T) { server := test.NewServer([]*test.RequestHandlerMapping{ + { + Method: http.MethodGet, + Pattern: "/api/projects", + Handler: func(w http.ResponseWriter, r *http.Request) { + data := `[{ + "name": "library", + "metadata": {"public":true} + }]` + w.Write([]byte(data)) + }, + }, { Method: http.MethodGet, Pattern: "/api/chartrepo/library/charts/harbor", @@ -58,12 +69,30 @@ func TestFetchCharts(t *testing.T) { } adapter, err := newAdapter(registry) require.Nil(t, err) - resources, err := adapter.FetchCharts([]string{"library"}, nil) + // nil filter + resources, err := adapter.FetchCharts(nil) require.Nil(t, err) assert.Equal(t, 2, len(resources)) assert.Equal(t, model.ResourceTypeChart, resources[0].Type) - assert.Equal(t, "harbor", resources[0].Metadata.Repository.Name) - assert.Equal(t, "library", resources[0].Metadata.Namespace.Name) + assert.Equal(t, "library/harbor", resources[0].Metadata.Repository.Name) + assert.Equal(t, 1, len(resources[0].Metadata.Vtags)) + assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0]) + // not nil filter + filters := []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "library/*", + }, + { + Type: model.FilterTypeTag, + Value: "1.0", + }, + } + resources, err = adapter.FetchCharts(filters) + require.Nil(t, err) + require.Equal(t, 1, len(resources)) + assert.Equal(t, model.ResourceTypeChart, resources[0].Type) + assert.Equal(t, "library/harbor", resources[0].Metadata.Repository.Name) assert.Equal(t, 1, len(resources[0].Metadata.Vtags)) assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0]) } diff --git a/src/replication/adapter/harbor/image_registry.go b/src/replication/adapter/harbor/image_registry.go index ef8a8d07c..ee1d1e015 100644 --- a/src/replication/adapter/harbor/image_registry.go +++ b/src/replication/adapter/harbor/image_registry.go @@ -16,9 +16,7 @@ package harbor import ( "fmt" - "strings" - "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/replication/model" ) @@ -33,10 +31,8 @@ func (r *repository) Match(filters []*model.Filter) (bool, error) { supportedFilters = append(supportedFilters, filter) } } - // trim the project part - _, name := utils.ParseRepository(r.Name) item := &FilterItem{ - Value: name, + Value: r.Name, } return item.Match(supportedFilters) } @@ -58,22 +54,14 @@ func (t *tag) Match(filters []*model.Filter) (bool, error) { return item.Match(supportedFilters) } -func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) { - if len(namespaces) == 0 { - nms, err := a.ListNamespaces(nil) - if err != nil { - return nil, err - } - for _, nm := range nms { - namespaces = append(namespaces, nm.Name) - } +func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { + // TODO optimize the performance + projects, err := a.getProjects("") + if err != nil { + return nil, err } resources := []*model.Resource{} - for _, namespace := range namespaces { - project, err := a.getProject(namespace) - if err != nil { - return nil, err - } + for _, project := range projects { repositories := []*repository{} url := fmt.Sprintf("%s/api/repositories?project_id=%d", a.coreServiceURL, project.ID) if err = a.client.Get(url, &repositories); err != nil { @@ -104,12 +92,9 @@ func (a *adapter) FetchImages(namespaces []string, filters []*model.Filter) ([]* Type: model.ResourceTypeRepository, Registry: a.registry, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: namespace, - // TODO filling the metadata - }, Repository: &model.Repository{ - Name: strings.TrimPrefix(repository.Name, namespace+"/"), + Name: repository.Name, + // TODO handle the metadata }, Vtags: vtags, }, diff --git a/src/replication/adapter/harbor/image_registry_test.go b/src/replication/adapter/harbor/image_registry_test.go index 33dd7484e..51449d2a8 100644 --- a/src/replication/adapter/harbor/image_registry_test.go +++ b/src/replication/adapter/harbor/image_registry_test.go @@ -66,26 +66,33 @@ func TestFetchImages(t *testing.T) { } adapter, err := newAdapter(registry) require.Nil(t, err) - // not nil namespaces - resources, err := adapter.FetchImages([]string{"library"}, nil) + // nil filter + resources, err := adapter.FetchImages(nil) require.Nil(t, err) assert.Equal(t, 1, len(resources)) assert.Equal(t, model.ResourceTypeRepository, resources[0].Type) - assert.Equal(t, "hello-world", resources[0].Metadata.Repository.Name) - assert.Equal(t, "library", resources[0].Metadata.Namespace.Name) + assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name) assert.Equal(t, 2, len(resources[0].Metadata.Vtags)) assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0]) assert.Equal(t, "2.0", resources[0].Metadata.Vtags[1]) - // nil namespaces - resources, err = adapter.FetchImages(nil, nil) + // not nil filter + filters := []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "library/*", + }, + { + Type: model.FilterTypeTag, + Value: "1.0", + }, + } + resources, err = adapter.FetchImages(filters) require.Nil(t, err) assert.Equal(t, 1, len(resources)) assert.Equal(t, model.ResourceTypeRepository, resources[0].Type) - assert.Equal(t, "hello-world", resources[0].Metadata.Repository.Name) - assert.Equal(t, "library", resources[0].Metadata.Namespace.Name) - assert.Equal(t, 2, len(resources[0].Metadata.Vtags)) + assert.Equal(t, "library/hello-world", resources[0].Metadata.Repository.Name) + assert.Equal(t, 1, len(resources[0].Metadata.Vtags)) assert.Equal(t, "1.0", resources[0].Metadata.Vtags[0]) - assert.Equal(t, "2.0", resources[0].Metadata.Vtags[1]) } func TestDeleteManifest(t *testing.T) { diff --git a/src/replication/adapter/huawei/huawei_adapter.go b/src/replication/adapter/huawei/huawei_adapter.go index eb27fa067..ef09d554d 100644 --- a/src/replication/adapter/huawei/huawei_adapter.go +++ b/src/replication/adapter/huawei/huawei_adapter.go @@ -10,6 +10,8 @@ import ( "regexp" "strings" + "github.com/goharbor/harbor/src/replication/util" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/replication/adapter" "github.com/goharbor/harbor/src/replication/model" @@ -110,7 +112,6 @@ func (adapter Adapter) ListNamespaces(query *model.NamespaceQuery) ([]*model.Nam // ConvertResourceMetadata convert resource metadata for Huawei SWR func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) { metadata := &model.ResourceMetadata{ - Namespace: namespace, Repository: resourceMetadata.Repository, Vtags: resourceMetadata.Vtags, Labels: resourceMetadata.Labels, @@ -121,12 +122,12 @@ func (adapter Adapter) ConvertResourceMetadata(resourceMetadata *model.ResourceM // PrepareForPush prepare for push to Huawei SWR func (adapter Adapter) PrepareForPush(resource *model.Resource) error { - namespace := resource.Metadata.Namespace - ns, err := adapter.GetNamespace(namespace.Name) + namespace, _ := util.ParseRepository(resource.Metadata.Repository.Name) + ns, err := adapter.GetNamespace(namespace) if err != nil { // } else { - if ns.Name == namespace.Name { + if ns.Name == namespace { return nil } } @@ -134,7 +135,7 @@ func (adapter Adapter) PrepareForPush(resource *model.Resource) error { url := fmt.Sprintf("%s/dockyard/v2/namespaces", adapter.Registry.URL) namespacebyte, err := json.Marshal(struct { Namespace string `json:"namespace"` - }{Namespace: namespace.Name}) + }{Namespace: namespace}) if err != nil { return err } diff --git a/src/replication/adapter/huawei/huawei_adapter_test.go b/src/replication/adapter/huawei/huawei_adapter_test.go index 6cd771722..d03061fe0 100644 --- a/src/replication/adapter/huawei/huawei_adapter_test.go +++ b/src/replication/adapter/huawei/huawei_adapter_test.go @@ -39,49 +39,15 @@ func TestAdapter_Info(t *testing.T) { t.Log(info) } -func TestAdapter_ListNamespaces(t *testing.T) { - namespaces, err := hwAdapter.ListNamespaces(&model.NamespaceQuery{Name: "o"}) - if err != nil { - if strings.HasPrefix(err.Error(), "[401]") { - t.Log("huawei ak/sk is not available", err.Error()) - } else { - t.Error(err) - } - } else { - for _, namespace := range namespaces { - t.Log(namespace.Name, namespace.Metadata) - } - } -} - -func TestAdapter_ConvertResourceMetadata(t *testing.T) { - metadata := &model.ResourceMetadata{} - - namespace := &model.Namespace{ - Name: "domain_repo_new", - Metadata: make(map[string]interface{}), - } - - metadata, err := hwAdapter.ConvertResourceMetadata(metadata, namespace) - if err != nil { - if strings.HasPrefix(err.Error(), "[401]") { - t.Log("huawei ak/sk is not available", err.Error()) - } else { - t.Error(err) - } - } else { - t.Log("success convert resource metadata") - t.Log(metadata) - } -} - func TestAdapter_PrepareForPush(t *testing.T) { - namespace := &model.Namespace{ + repository := &model.Repository{ Name: "domain_repo_new", Metadata: make(map[string]interface{}), } resource := &model.Resource{} - metadata := &model.ResourceMetadata{Namespace: namespace} + metadata := &model.ResourceMetadata{ + Repository: repository, + } resource.Metadata = metadata err := hwAdapter.PrepareForPush(resource) if err != nil { diff --git a/src/replication/adapter/huawei/image_registry.go b/src/replication/adapter/huawei/image_registry.go index cbc9ec4a9..63e0658eb 100644 --- a/src/replication/adapter/huawei/image_registry.go +++ b/src/replication/adapter/huawei/image_registry.go @@ -84,16 +84,12 @@ func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource { info["status"] = repo.Status info["total_range"] = repo.TotalRange - namespace := &model.Namespace{ - Name: repo.NamespaceName, - } repository := &model.Repository{ Name: repo.Name, Metadata: info, } resource.ExtendedInfo = info resource.Metadata = &model.ResourceMetadata{ - Namespace: namespace, Repository: repository, Vtags: repo.Tags, Labels: []string{}, @@ -101,7 +97,6 @@ func parseRepoQueryResultToResource(repo hwRepoQueryResult) *model.Resource { resource.Deleted = false resource.Override = false resource.Type = model.ResourceTypeRepository - resource.URI = repo.Path return &resource } diff --git a/src/replication/adapter/image_registry.go b/src/replication/adapter/image_registry.go index 2bbca0d7b..cf25eada4 100644 --- a/src/replication/adapter/image_registry.go +++ b/src/replication/adapter/image_registry.go @@ -40,7 +40,7 @@ const ( // ImageRegistry defines the capabilities that an image registry should have type ImageRegistry interface { - FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) + FetchImages(filters []*model.Filter) ([]*model.Resource, error) ManifestExist(repository, reference string) (exist bool, digest string, err error) PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error) PushManifest(repository, reference, mediaType string, payload []byte) error diff --git a/src/replication/adapter/native/adapter.go b/src/replication/adapter/native/adapter.go index 10d610417..ad9626b87 100644 --- a/src/replication/adapter/native/adapter.go +++ b/src/replication/adapter/native/adapter.go @@ -15,8 +15,6 @@ package native import ( - "errors" - "github.com/goharbor/harbor/src/common/utils/log" adp "github.com/goharbor/harbor/src/replication/adapter" "github.com/goharbor/harbor/src/replication/model" @@ -75,33 +73,5 @@ func (native) Info() (info *model.RegistryInfo, err error) { }, nil } -// ConvertResourceMetadata convert src to dst resource -func (native) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) { - if metadata == nil { - return nil, errors.New("the metadata cannot be null") - } - - var result = &model.ResourceMetadata{ - Namespace: metadata.Namespace, - Repository: metadata.Repository, - Vtags: metadata.Vtags, - } - - // if dest namespace is set, rename metadata namespace - if namespace != nil { - result.Namespace = namespace - } - - result.Repository = &model.Repository{Name: result.GetResourceName()} - result.Namespace = nil - - return result, nil -} - // PrepareForPush nothing need to do. func (native) PrepareForPush(*model.Resource) error { return nil } - -// ListNamespaces native registry no namespaces, so list empty array. -func (native) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) { - return []*model.Namespace{}, nil -} diff --git a/src/replication/adapter/native/adapter_test.go b/src/replication/adapter/native/adapter_test.go index 98ba2bef9..d0f0cb642 100644 --- a/src/replication/adapter/native/adapter_test.go +++ b/src/replication/adapter/native/adapter_test.go @@ -50,81 +50,6 @@ func Test_native_Info(t *testing.T) { assert.Equal(t, model.ResourceTypeRepository, info.SupportedResourceTypes[0]) } -func Test_native_ConvertResourceMetadata(t *testing.T) { - var registry = &model.Registry{URL: "abc"} - var reg, _ = adp.NewDefaultImageRegistry(registry) - var adapter = native{ - DefaultImageRegistry: reg, - registry: registry, - } - assert.NotNil(t, adapter) - - tests := []struct { - name string - metadata *model.ResourceMetadata - namespace *model.Namespace - want string - wantErr bool - }{ - {name: "nil metadata", metadata: nil, wantErr: true}, - { - name: "2 level", - metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{Name: "a"}, - Repository: &model.Repository{Name: "b"}, - }, - namespace: nil, - want: "a/b", - wantErr: false, - }, - { - name: "2 level rename reomte repository", - metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{Name: "a"}, - Repository: &model.Repository{Name: "b"}, - }, - namespace: &model.Namespace{Name: "c"}, - want: "c/b", - wantErr: false, - }, - { - name: "3 level", - metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{Name: "a"}, - Repository: &model.Repository{Name: "b/c"}, - }, - namespace: nil, - want: "a/b/c", - wantErr: false, - }, - { - name: "3 level rename reomte repository", - metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{Name: "a"}, - Repository: &model.Repository{Name: "b/c"}, - }, - namespace: &model.Namespace{Name: "d"}, - want: "d/b/c", - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var convert, err = adapter.ConvertResourceMetadata(tt.metadata, tt.namespace) - if tt.wantErr { - assert.NotNil(t, err) - } else { - assert.Nil(t, err) - assert.NotNil(t, convert) - assert.Nil(t, convert.Namespace) - assert.Equal(t, tt.want, convert.Repository.Name) - assert.Equal(t, tt.want, convert.GetResourceName()) - } - }) - } -} - func Test_native_PrepareForPush(t *testing.T) { var registry = &model.Registry{URL: "abc"} var reg, _ = adp.NewDefaultImageRegistry(registry) @@ -137,17 +62,3 @@ func Test_native_PrepareForPush(t *testing.T) { var err = adapter.PrepareForPush(nil) assert.Nil(t, err) } - -func Test_native_ListNamespaces(t *testing.T) { - var registry = &model.Registry{URL: "abc"} - var reg, _ = adp.NewDefaultImageRegistry(registry) - var adapter = native{ - DefaultImageRegistry: reg, - registry: registry, - } - assert.NotNil(t, adapter) - - var ns, err = adapter.ListNamespaces(nil) - assert.Nil(t, err) - assert.NotNil(t, ns) -} diff --git a/src/replication/adapter/native/image_registry.go b/src/replication/adapter/native/image_registry.go index 541c8b4cb..1dda3f93c 100644 --- a/src/replication/adapter/native/image_registry.go +++ b/src/replication/adapter/native/image_registry.go @@ -15,7 +15,6 @@ package native import ( - "errors" "net/http" "strings" @@ -27,11 +26,7 @@ import ( var _ adp.ImageRegistry = native{} -func (n native) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) { - if len(namespaces) > 0 { - return nil, errors.New("native registry adapter not support namespace") - } - +func (n native) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { nameFilterPattern := "" tagFilterPattern := "" for _, filter := range filters { diff --git a/src/replication/adapter/native/image_registry_test.go b/src/replication/adapter/native/image_registry_test.go index 720fc2ded..d09d8f3d4 100644 --- a/src/replication/adapter/native/image_registry_test.go +++ b/src/replication/adapter/native/image_registry_test.go @@ -65,13 +65,11 @@ func Test_native_FetchImages(t *testing.T) { assert.NotNil(t, adapter) tests := []struct { - name string - namespaces []string - filters []*model.Filter - want []*model.Resource - wantErr bool + name string + filters []*model.Filter + want []*model.Resource + wantErr bool }{ - {name: "namespace not empty", namespaces: []string{"ns"}, wantErr: true}, // TODO: discuss: should we report error if not found in the source native registry. // { // name: "repository not exist", @@ -119,8 +117,7 @@ func Test_native_FetchImages(t *testing.T) { wantErr: false, }, { - name: "only special repository", - namespaces: []string{}, + name: "only special repository", filters: []*model.Filter{ { Type: model.FilterTypeName, @@ -138,8 +135,7 @@ func Test_native_FetchImages(t *testing.T) { wantErr: false, }, { - name: "only special tag", - namespaces: []string{}, + name: "only special tag", filters: []*model.Filter{ { Type: model.FilterTypeTag, @@ -163,8 +159,7 @@ func Test_native_FetchImages(t *testing.T) { wantErr: false, }, { - name: "special repository and special tag", - namespaces: []string{}, + name: "special repository and special tag", filters: []*model.Filter{ { Type: model.FilterTypeName, @@ -187,8 +182,7 @@ func Test_native_FetchImages(t *testing.T) { wantErr: false, }, { - name: "only wildcard repository", - namespaces: []string{}, + name: "only wildcard repository", filters: []*model.Filter{ { Type: model.FilterTypeName, @@ -206,8 +200,7 @@ func Test_native_FetchImages(t *testing.T) { wantErr: false, }, { - name: "only wildcard tag", - namespaces: []string{}, + name: "only wildcard tag", filters: []*model.Filter{ { Type: model.FilterTypeTag, @@ -231,8 +224,7 @@ func Test_native_FetchImages(t *testing.T) { wantErr: false, }, { - name: "wildcard repository and wildcard tag", - namespaces: []string{}, + name: "wildcard repository and wildcard tag", filters: []*model.Filter{ { Type: model.FilterTypeName, @@ -257,7 +249,7 @@ func Test_native_FetchImages(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var resources, err = adapter.FetchImages(tt.namespaces, tt.filters) + var resources, err = adapter.FetchImages(tt.filters) if tt.wantErr { require.Len(t, resources, 0) require.NotNil(t, err) @@ -265,7 +257,6 @@ func Test_native_FetchImages(t *testing.T) { require.Equal(t, len(tt.want), len(resources)) for i, resource := range resources { require.NotNil(t, resource.Metadata) - assert.Equal(t, tt.want[i].Metadata.Namespace, resource.Metadata.Namespace) assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository) assert.Equal(t, tt.want[i].Metadata.Vtags, resource.Metadata.Vtags) } diff --git a/src/replication/dao/models/policy.go b/src/replication/dao/models/policy.go index 2c00d7c9f..466ea06d6 100644 --- a/src/replication/dao/models/policy.go +++ b/src/replication/dao/models/policy.go @@ -9,7 +9,6 @@ type RepPolicy struct { Description string `orm:"column(description)" json:"description"` Creator string `orm:"column(creator)" json:"creator"` SrcRegistryID int64 `orm:"column(src_registry_id)" json:"src_registry_id"` - SrcNamespaces string `orm:"column(src_namespaces)" json:"src_namespaces"` DestRegistryID int64 `orm:"column(dest_registry_id)" json:"dest_registry_id"` DestNamespace string `orm:"column(dest_namespace)" json:"dest_namespace"` Override bool `orm:"column(override)" json:"override"` diff --git a/src/replication/dao/policy_test.go b/src/replication/dao/policy_test.go index ba89bfa03..1d64c5a28 100644 --- a/src/replication/dao/policy_test.go +++ b/src/replication/dao/policy_test.go @@ -17,7 +17,6 @@ var ( Description: "Policy Description", Creator: "someone", SrcRegistryID: 123, - SrcNamespaces: "ns1,ns2,ns3", DestRegistryID: 456, DestNamespace: "target_ns", ReplicateDeletion: true, @@ -33,7 +32,6 @@ var ( Description: "Policy Description", Creator: "someone", SrcRegistryID: 123, - SrcNamespaces: "ns1,ns2,ns3", DestRegistryID: 456, DestNamespace: "target_ns", ReplicateDeletion: true, @@ -49,7 +47,6 @@ var ( Description: "Policy Description", Creator: "someone", SrcRegistryID: 123, - SrcNamespaces: "ns1,ns2,ns3", DestRegistryID: 456, DestNamespace: "target_ns", ReplicateDeletion: true, @@ -126,7 +123,6 @@ func TestGetPolicies(t *testing.T) { assert.Equal(t, tt.wantPolicies[i].Description, gotPolicy.Description) assert.Equal(t, tt.wantPolicies[i].Creator, gotPolicy.Creator) assert.Equal(t, tt.wantPolicies[i].SrcRegistryID, gotPolicy.SrcRegistryID) - assert.Equal(t, tt.wantPolicies[i].SrcNamespaces, gotPolicy.SrcNamespaces) assert.Equal(t, tt.wantPolicies[i].DestRegistryID, gotPolicy.DestRegistryID) assert.Equal(t, tt.wantPolicies[i].DestNamespace, gotPolicy.DestNamespace) assert.Equal(t, tt.wantPolicies[i].ReplicateDeletion, gotPolicy.ReplicateDeletion) @@ -163,7 +159,6 @@ func TestGetRepPolicy(t *testing.T) { assert.Equal(t, tt.wantPolicy.Description, gotPolicy.Description) assert.Equal(t, tt.wantPolicy.Creator, gotPolicy.Creator) assert.Equal(t, tt.wantPolicy.SrcRegistryID, gotPolicy.SrcRegistryID) - assert.Equal(t, tt.wantPolicy.SrcNamespaces, gotPolicy.SrcNamespaces) assert.Equal(t, tt.wantPolicy.DestRegistryID, gotPolicy.DestRegistryID) assert.Equal(t, tt.wantPolicy.DestNamespace, gotPolicy.DestNamespace) assert.Equal(t, tt.wantPolicy.ReplicateDeletion, gotPolicy.ReplicateDeletion) @@ -202,7 +197,6 @@ func TestGetRepPolicyByName(t *testing.T) { assert.Equal(t, tt.wantPolicy.Description, gotPolicy.Description) assert.Equal(t, tt.wantPolicy.Creator, gotPolicy.Creator) assert.Equal(t, tt.wantPolicy.SrcRegistryID, gotPolicy.SrcRegistryID) - assert.Equal(t, tt.wantPolicy.SrcNamespaces, gotPolicy.SrcNamespaces) assert.Equal(t, tt.wantPolicy.DestRegistryID, gotPolicy.DestRegistryID) assert.Equal(t, tt.wantPolicy.DestNamespace, gotPolicy.DestNamespace) assert.Equal(t, tt.wantPolicy.ReplicateDeletion, gotPolicy.ReplicateDeletion) diff --git a/src/replication/event/handler.go b/src/replication/event/handler.go index c6db7161a..3d731a4b1 100644 --- a/src/replication/event/handler.go +++ b/src/replication/event/handler.go @@ -18,6 +18,8 @@ import ( "errors" "fmt" + "github.com/goharbor/harbor/src/replication/util" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/replication/config" "github.com/goharbor/harbor/src/replication/model" @@ -55,10 +57,9 @@ func (h *handler) Handle(event *Event) error { var policies []*model.Policy var err error switch event.Type { - case EventTypeImagePush, EventTypeChartUpload: - policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace.Name) - case EventTypeImageDelete, EventTypeChartDelete: - policies, err = h.getRelatedPolicies(event.Resource.Metadata.Namespace.Name, true) + case EventTypeImagePush, EventTypeChartUpload, + EventTypeImageDelete, EventTypeChartDelete: + policies, err = h.getRelatedPolicies(event.Resource) default: return fmt.Errorf("unsupported event type %s", event.Type) } @@ -84,22 +85,15 @@ func (h *handler) Handle(event *Event) error { return nil } -func (h *handler) getRelatedPolicies(namespace string, replicateDeletion ...bool) ([]*model.Policy, error) { +func (h *handler) getRelatedPolicies(resource *model.Resource) ([]*model.Policy, error) { _, policies, err := h.policyCtl.List() if err != nil { return nil, err } result := []*model.Policy{} for _, policy := range policies { - exist := false - for _, ns := range policy.SrcNamespaces { - if ns == namespace { - exist = true - break - } - } - // contains no namespace that is specified - if !exist { + // disabled + if !policy.Enabled { continue } // has no trigger @@ -110,8 +104,16 @@ func (h *handler) getRelatedPolicies(namespace string, replicateDeletion ...bool if policy.Trigger.Type != model.TriggerTypeEventBased { continue } - // whether replicate deletion doesn't match the value specified in policy - if len(replicateDeletion) > 0 && replicateDeletion[0] != policy.Deletion { + // doesn't replicate deletion + if resource.Deleted && !policy.Deletion { + continue + } + // doesn't match the name filter + m, err := match(policy.Filters, resource) + if err != nil { + return nil, err + } + if !m { continue } result = append(result, policy) @@ -119,6 +121,26 @@ func (h *handler) getRelatedPolicies(namespace string, replicateDeletion ...bool return result, nil } +// TODO unify the match logic with other? +func match(filters []*model.Filter, resource *model.Resource) (bool, error) { + match := true + repository := resource.Metadata.Repository.Name + for _, filter := range filters { + if filter.Type != model.FilterTypeName { + continue + } + m, err := util.Match(filter.Value.(string), repository) + if err != nil { + return false, err + } + if !m { + match = false + break + } + } + return match, nil +} + // PopulateRegistries populates the source registry and destination registry properties for policy func PopulateRegistries(registryMgr registry.Manager, policy *model.Policy) error { if policy == nil { diff --git a/src/replication/event/handler_test.go b/src/replication/event/handler_test.go index 7a1b4f019..be5c7efb2 100644 --- a/src/replication/event/handler_test.go +++ b/src/replication/event/handler_test.go @@ -59,34 +59,76 @@ func (f *fakedPolicyController) Create(*model.Policy) (int64, error) { func (f *fakedPolicyController) List(...*model.PolicyQuery) (int64, []*model.Policy, error) { polices := []*model.Policy{ { - ID: 1, - SrcNamespaces: []string{"test"}, - Deletion: false, + ID: 1, + Enabled: true, + Deletion: true, Trigger: &model.Trigger{ Type: model.TriggerTypeEventBased, }, + Filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "test/*", + }, + }, }, + // nil trigger { - ID: 2, - SrcNamespaces: []string{"library"}, - Deletion: true, - Trigger: nil, + ID: 2, + Enabled: true, + Deletion: true, + Trigger: nil, + Filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "library/*", + }, + }, }, + // doesn't replicate deletion { - ID: 3, - SrcNamespaces: []string{"library"}, - Deletion: false, + ID: 3, + Enabled: true, + Deletion: false, Trigger: &model.Trigger{ Type: model.TriggerTypeEventBased, }, + Filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "library/*", + }, + }, }, + // replicate deletion { - ID: 4, - SrcNamespaces: []string{"library"}, - Deletion: true, + ID: 4, + Enabled: true, + Deletion: true, Trigger: &model.Trigger{ Type: model.TriggerTypeEventBased, }, + Filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "library/*", + }, + }, + }, + // disabled + { + ID: 5, + Enabled: false, + Deletion: true, + Trigger: &model.Trigger{ + Type: model.TriggerTypeEventBased, + }, + Filters: []*model.Filter{ + { + Type: model.FilterTypeName, + Value: "library/*", + }, + }, }, } return int64(len(polices)), polices, nil @@ -134,13 +176,26 @@ func TestGetRelatedPolicies(t *testing.T) { handler := &handler{ policyCtl: &fakedPolicyController{}, } - policies, err := handler.getRelatedPolicies("library") + policies, err := handler.getRelatedPolicies(&model.Resource{ + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "library/hello-world", + }, + }, + }) require.Nil(t, err) assert.Equal(t, 2, len(policies)) assert.Equal(t, int64(3), policies[0].ID) assert.Equal(t, int64(4), policies[1].ID) - policies, err = handler.getRelatedPolicies("library", true) + policies, err = handler.getRelatedPolicies(&model.Resource{ + Metadata: &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: "library/hello-world", + }, + }, + Deleted: true, + }) require.Nil(t, err) assert.Equal(t, 1, len(policies)) assert.Equal(t, int64(4), policies[0].ID) @@ -159,11 +214,8 @@ func TestHandle(t *testing.T) { err = handler.Handle(&Event{ Resource: &model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{}, }, @@ -176,11 +228,8 @@ func TestHandle(t *testing.T) { err = handler.Handle(&Event{ Resource: &model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{"latest"}, }, @@ -193,11 +242,8 @@ func TestHandle(t *testing.T) { err = handler.Handle(&Event{ Resource: &model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{"latest"}, }, @@ -210,11 +256,8 @@ func TestHandle(t *testing.T) { err = handler.Handle(&Event{ Resource: &model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{"latest"}, }, diff --git a/src/replication/model/namespace.go b/src/replication/model/namespace.go index 3d216695b..87ae11b60 100644 --- a/src/replication/model/namespace.go +++ b/src/replication/model/namespace.go @@ -14,6 +14,8 @@ package model +// TODO remove + // Namespace represents the full path of resource isolation unit; // if the namespace has hierarchical structure, e.g organization->team, // it should be converted to organization.team diff --git a/src/replication/model/policy.go b/src/replication/model/policy.go index a4622d200..3751d2c74 100644 --- a/src/replication/model/policy.go +++ b/src/replication/model/policy.go @@ -42,8 +42,7 @@ type Policy struct { // TODO consider to remove this property? Creator string `json:"creator"` // source - SrcRegistry *Registry `json:"src_registry"` - SrcNamespaces []string `json:"src_namespaces"` + SrcRegistry *Registry `json:"src_registry"` // destination // TODO rename to DstRegistry DestRegistry *Registry `json:"dest_registry"` diff --git a/src/replication/model/policy_test.go b/src/replication/model/policy_test.go index 4c3dd2f00..9b06a5f31 100644 --- a/src/replication/model/policy_test.go +++ b/src/replication/model/policy_test.go @@ -61,7 +61,6 @@ func TestValidOfPolicy(t *testing.T) { DestRegistry: &Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, Filters: []*Filter{ { Type: "invalid_type", @@ -80,7 +79,6 @@ func TestValidOfPolicy(t *testing.T) { DestRegistry: &Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, Filters: []*Filter{ { Type: FilterTypeName, @@ -103,7 +101,6 @@ func TestValidOfPolicy(t *testing.T) { DestRegistry: &Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, Filters: []*Filter{ { Type: FilterTypeName, @@ -126,7 +123,6 @@ func TestValidOfPolicy(t *testing.T) { DestRegistry: &Registry{ ID: 1, }, - SrcNamespaces: []string{"library"}, Filters: []*Filter{ { Type: FilterTypeName, diff --git a/src/replication/model/registry.go b/src/replication/model/registry.go index b2874029c..0d9e10390 100644 --- a/src/replication/model/registry.go +++ b/src/replication/model/registry.go @@ -109,7 +109,6 @@ type FilterStyle struct { type RegistryInfo struct { Type RegistryType `json:"type"` Description string `json:"description"` - SupportNamespace bool `json:"support_namespace"` SupportedResourceTypes []ResourceType `json:"-"` SupportedResourceFilters []*FilterStyle `json:"supported_resource_filters"` SupportedTriggers []TriggerType `json:"supported_triggers"` diff --git a/src/replication/model/resource.go b/src/replication/model/resource.go index 34dd0c911..f53be4133 100644 --- a/src/replication/model/resource.go +++ b/src/replication/model/resource.go @@ -30,7 +30,6 @@ func (r ResourceType) Valid() bool { // ResourceMetadata of resource type ResourceMetadata struct { - Namespace *Namespace `json:"namespace"` Repository *Repository `json:"repository"` Vtags []string `json:"v_tags"` // TODO the labels should be put into tag and repository level? @@ -38,18 +37,12 @@ type ResourceMetadata struct { } // GetResourceName returns the name of the resource +// TODO remove func (r *ResourceMetadata) GetResourceName() string { - name := "" - if r.Namespace != nil && len(r.Namespace.Name) > 0 { - name += r.Namespace.Name + if r.Repository == nil { + return "" } - if r.Repository != nil && len(r.Repository.Name) > 0 { - if len(name) > 0 { - name += "/" - } - name += r.Repository.Name - } - return name + return r.Repository.Name } // Repository info of the resource @@ -61,7 +54,6 @@ type Repository struct { // Resource represents the general replicating content type Resource struct { Type ResourceType `json:"type"` - URI string `json:"uri"` Metadata *ResourceMetadata `json:"metadata"` Registry *Registry `json:"registry"` ExtendedInfo map[string]interface{} `json:"extended_info"` diff --git a/src/replication/model/resource_test.go b/src/replication/model/resource_test.go index c1c2e7af3..86dac1616 100644 --- a/src/replication/model/resource_test.go +++ b/src/replication/model/resource_test.go @@ -25,7 +25,7 @@ func TestGetResourceName(t *testing.T) { assert.Equal(t, "", r.GetResourceName()) r = &ResourceMetadata{ - Namespace: &Namespace{ + Repository: &Repository{ Name: "library", }, } @@ -39,11 +39,8 @@ func TestGetResourceName(t *testing.T) { assert.Equal(t, "hello-world", r.GetResourceName()) r = &ResourceMetadata{ - Namespace: &Namespace{ - Name: "library", - }, Repository: &Repository{ - Name: "hello-world", + Name: "library/hello-world", }, } assert.Equal(t, "library/hello-world", r.GetResourceName()) diff --git a/src/replication/operation/controller.go b/src/replication/operation/controller.go index 8a4918798..10605554b 100644 --- a/src/replication/operation/controller.go +++ b/src/replication/operation/controller.go @@ -58,6 +58,9 @@ type controller struct { } func (c *controller) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) { + if !policy.Enabled { + return 0, fmt.Errorf("the policy %d is diabled", policy.ID) + } // only support one tag if the resource is specified as we append the tag name as a filter // when creating the flow in function "createFlow" if resource != nil && len(resource.Metadata.Vtags) != 1 { diff --git a/src/replication/operation/controller_test.go b/src/replication/operation/controller_test.go index 31c64d393..1c8448fbb 100644 --- a/src/replication/operation/controller_test.go +++ b/src/replication/operation/controller_test.go @@ -129,48 +129,20 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) { SupportedTriggers: []model.TriggerType{model.TriggerTypeManual}, }, nil } -func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) { - return nil, nil -} -func (f *fakedAdapter) ConvertResourceMetadata(*model.ResourceMetadata, *model.Namespace) (*model.ResourceMetadata, error) { - return &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, - Repository: &model.Repository{ - Name: "hello-world", - }, - Vtags: []string{"latest"}, - }, nil -} + func (f *fakedAdapter) PrepareForPush(*model.Resource) error { return nil } func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { return model.Healthy, nil } -func (f *fakedAdapter) GetNamespace(ns string) (*model.Namespace, error) { - var namespace *model.Namespace - if ns == "library" { - namespace = &model.Namespace{ - Name: "library", - Metadata: map[string]interface{}{ - "public": true, - }, - } - } - return namespace, nil -} func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) ([]*model.Resource, error) { return []*model.Resource{ { Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{"latest"}, }, @@ -205,11 +177,8 @@ func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter) { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "harbor", + Name: "library/harbor", }, Vtags: []string{"0.2.0"}, }, @@ -239,7 +208,6 @@ func TestStartReplication(t *testing.T) { err := adapter.RegisterFactory(model.RegistryTypeHarbor, fakedAdapterFactory) require.Nil(t, err) config.Config = &config.Configuration{} - // the resource contains Vtags whose length isn't 1 policy := &model.Policy{ SrcRegistry: &model.Registry{ Type: model.RegistryTypeHarbor, @@ -251,15 +219,18 @@ func TestStartReplication(t *testing.T) { resource := &model.Resource{ Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{"1.0", "2.0"}, }, } + // policy is disabled + _, err = ctl.StartReplication(policy, resource, model.TriggerTypeEventBased) + require.NotNil(t, err) + + policy.Enabled = true + // the resource contains Vtags whose length isn't 1 _, err = ctl.StartReplication(policy, resource, model.TriggerTypeEventBased) require.NotNil(t, err) diff --git a/src/replication/operation/flow/copy.go b/src/replication/operation/flow/copy.go index ac3395fb6..5329d9c1c 100644 --- a/src/replication/operation/flow/copy.go +++ b/src/replication/operation/flow/copy.go @@ -57,10 +57,8 @@ func (c *copyFlow) Run(interface{}) (int, error) { log.Infof("no resources need to be replicated for the execution %d, skip", c.executionID) return 0, nil } - dstResources, err := assembleDestinationResources(dstAdapter, srcResources, c.policy) - if err != nil { - return 0, err - } + + dstResources := assembleDestinationResources(srcResources, c.policy) if err = prepareForPush(dstAdapter, dstResources); err != nil { return 0, err diff --git a/src/replication/operation/flow/deletion.go b/src/replication/operation/flow/deletion.go index 337bc8bd2..ebcf98810 100644 --- a/src/replication/operation/flow/deletion.go +++ b/src/replication/operation/flow/deletion.go @@ -43,10 +43,6 @@ func NewDeletionFlow(executionMgr execution.Manager, scheduler scheduler.Schedul } func (d *deletionFlow) Run(interface{}) (int, error) { - _, dstAdapter, err := initialize(d.policy) - if err != nil { - return 0, err - } // filling the registry information for _, resource := range d.resources { resource.Registry = d.policy.SrcRegistry @@ -60,10 +56,8 @@ func (d *deletionFlow) Run(interface{}) (int, error) { log.Infof("no resources need to be replicated for the execution %d, skip", d.executionID) return 0, nil } - dstResources, err := assembleDestinationResources(dstAdapter, srcResources, d.policy) - if err != nil { - return 0, err - } + + dstResources := assembleDestinationResources(srcResources, d.policy) items, err := preprocess(d.scheduler, srcResources, dstResources) if err != nil { diff --git a/src/replication/operation/flow/deletion_test.go b/src/replication/operation/flow/deletion_test.go index 511980f6a..1e1724786 100644 --- a/src/replication/operation/flow/deletion_test.go +++ b/src/replication/operation/flow/deletion_test.go @@ -36,11 +36,8 @@ func TestRunOfDeletionFlow(t *testing.T) { resources := []*model.Resource{ { Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{"latest"}, }, diff --git a/src/replication/operation/flow/stage.go b/src/replication/operation/flow/stage.go index 4a287b64e..7fefba0da 100644 --- a/src/replication/operation/flow/stage.go +++ b/src/replication/operation/flow/stage.go @@ -87,14 +87,14 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc if !ok { return nil, fmt.Errorf("the adapter doesn't implement the ImageRegistry interface") } - res, err = reg.FetchImages(policy.SrcNamespaces, filters) + res, err = reg.FetchImages(filters) } else if typ == model.ResourceTypeChart { // charts reg, ok := adapter.(adp.ChartRegistry) if !ok { return nil, fmt.Errorf("the adapter doesn't implement the ChartRegistry interface") } - res, err = reg.FetchCharts(policy.SrcNamespaces, filters) + res, err = reg.FetchCharts(filters) } else { return nil, fmt.Errorf("unsupported resource type %s", typ) } @@ -135,8 +135,7 @@ func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*m match = false break FILTER_LOOP } - // TODO filter only the repository part? - m, err := util.Match(pattern, resource.Metadata.GetResourceName()) + m, err := util.Match(pattern, resource.Metadata.Repository.Name) if err != nil { return nil, err } @@ -184,39 +183,35 @@ func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*m } // assemble the destination resources by filling the metadata, registry and override properties -func assembleDestinationResources(adapter adp.Adapter, resources []*model.Resource, - policy *model.Policy) ([]*model.Resource, error) { +func assembleDestinationResources(resources []*model.Resource, + policy *model.Policy) []*model.Resource { var result []*model.Resource - var namespace *model.Namespace - if len(policy.DestNamespace) > 0 { - namespace = &model.Namespace{ - Name: policy.DestNamespace, - } - } for _, resource := range resources { - metadata, err := adapter.ConvertResourceMetadata(resource.Metadata, namespace) - if err != nil { - return nil, fmt.Errorf("failed to convert the resource metadata of %s: %v", resource.Metadata.GetResourceName(), err) - } res := &model.Resource{ Type: resource.Type, - Metadata: metadata, Registry: policy.DestRegistry, ExtendedInfo: resource.ExtendedInfo, Deleted: resource.Deleted, Override: policy.Override, } + res.Metadata = &model.ResourceMetadata{ + Repository: &model.Repository{ + Name: replaceNamespace(resource.Metadata.Repository.Name, policy.DestNamespace), + Metadata: resource.Metadata.Repository.Metadata, + }, + Vtags: resource.Metadata.Vtags, + } result = append(result, res) } log.Debug("assemble the destination resources completed") - return result, nil + return result } // do the prepare work for pushing/uploading the resources: create the namespace or repository func prepareForPush(adapter adp.Adapter, resources []*model.Resource) error { // TODO need to consider how to handle that both contains public/private namespace for _, resource := range resources { - name := resource.Metadata.GetResourceName() + name := resource.Metadata.Repository.Name if err := adapter.PrepareForPush(resource); err != nil { return fmt.Errorf("failed to do the prepare work for pushing/uploading %s: %v", name, err) } @@ -318,12 +313,23 @@ func getResourceName(res *model.Resource) string { return "" } if len(meta.Vtags) == 0 { - return meta.GetResourceName() + return meta.Repository.Name } if len(meta.Vtags) <= 5 { - return meta.GetResourceName() + ":[" + strings.Join(meta.Vtags, ",") + "]" + return meta.Repository.Name + ":[" + strings.Join(meta.Vtags, ",") + "]" } return fmt.Sprintf("%s:[%s ... %d in total]", meta.GetResourceName(), strings.Join(meta.Vtags[:5], ","), len(meta.Vtags)) } + +// repository:c namespace:n -> n/c +// repository:b/c namespace:n -> n/c +// repository:a/b/c namespace:n -> n/c +func replaceNamespace(repository string, namespace string) string { + if len(namespace) == 0 { + return repository + } + _, rest := util.ParseRepository(repository) + return fmt.Sprintf("%s/%s", namespace, rest) +} diff --git a/src/replication/operation/flow/stage_test.go b/src/replication/operation/flow/stage_test.go index 2c3bc425b..cfd710497 100644 --- a/src/replication/operation/flow/stage_test.go +++ b/src/replication/operation/flow/stage_test.go @@ -45,15 +45,6 @@ func (f *fakedAdapter) Info() (*model.RegistryInfo, error) { SupportedTriggers: []model.TriggerType{model.TriggerTypeManual}, }, nil } -func (f *fakedAdapter) ListNamespaces(*model.NamespaceQuery) ([]*model.Namespace, error) { - return nil, nil -} -func (f *fakedAdapter) ConvertResourceMetadata(metadata *model.ResourceMetadata, namespace *model.Namespace) (*model.ResourceMetadata, error) { - if namespace != nil { - metadata.Namespace = namespace - } - return metadata, nil -} func (f *fakedAdapter) PrepareForPush(*model.Resource) error { return nil @@ -61,28 +52,13 @@ func (f *fakedAdapter) PrepareForPush(*model.Resource) error { func (f *fakedAdapter) HealthCheck() (model.HealthStatus, error) { return model.Healthy, nil } -func (f *fakedAdapter) GetNamespace(ns string) (*model.Namespace, error) { - var namespace *model.Namespace - if ns == "library" { - namespace = &model.Namespace{ - Name: "library", - Metadata: map[string]interface{}{ - "public": true, - }, - } - } - return namespace, nil -} -func (f *fakedAdapter) FetchImages(namespace []string, filters []*model.Filter) ([]*model.Resource, error) { +func (f *fakedAdapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) { return []*model.Resource{ { Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{"latest"}, }, @@ -112,16 +88,13 @@ func (f *fakedAdapter) PullBlob(repository, digest string) (size int64, blob io. func (f *fakedAdapter) PushBlob(repository, digest string, size int64, blob io.Reader) error { return nil } -func (f *fakedAdapter) FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) { +func (f *fakedAdapter) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) { return []*model.Resource{ { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "harbor", + Name: "library/harbor", }, Vtags: []string{"0.2.0"}, }, @@ -240,11 +213,8 @@ func TestFilterResources(t *testing.T) { { Type: model.ResourceTypeRepository, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{"latest"}, // TODO test labels @@ -256,11 +226,8 @@ func TestFilterResources(t *testing.T) { { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "harbor", + Name: "library/harbor", }, Vtags: []string{"0.2.0", "0.3.0"}, // TODO test labels @@ -272,11 +239,8 @@ func TestFilterResources(t *testing.T) { { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "mysql", + Name: "library/mysql", }, Vtags: []string{"1.0"}, // TODO test labels @@ -307,23 +271,18 @@ func TestFilterResources(t *testing.T) { res, err := filterResources(resources, filters) require.Nil(t, err) assert.Equal(t, 1, len(res)) - assert.Equal(t, "library", res[0].Metadata.Namespace.Name) - assert.Equal(t, "harbor", res[0].Metadata.Repository.Name) + assert.Equal(t, "library/harbor", res[0].Metadata.Repository.Name) assert.Equal(t, 1, len(res[0].Metadata.Vtags)) assert.Equal(t, "0.2.0", res[0].Metadata.Vtags[0]) } func TestAssembleDestinationResources(t *testing.T) { - adapter := &fakedAdapter{} resources := []*model.Resource{ { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{"latest"}, }, @@ -335,12 +294,10 @@ func TestAssembleDestinationResources(t *testing.T) { DestNamespace: "test", Override: true, } - res, err := assembleDestinationResources(adapter, resources, policy) - require.Nil(t, err) + res := assembleDestinationResources(resources, policy) assert.Equal(t, 1, len(res)) assert.Equal(t, model.ResourceTypeChart, res[0].Type) - assert.Equal(t, "hello-world", res[0].Metadata.Repository.Name) - assert.Equal(t, "test", res[0].Metadata.Namespace.Name) + assert.Equal(t, "test/hello-world", res[0].Metadata.Repository.Name) assert.Equal(t, 1, len(res[0].Metadata.Vtags)) assert.Equal(t, "latest", res[0].Metadata.Vtags[0]) } @@ -351,11 +308,8 @@ func TestPreprocess(t *testing.T) { { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "library/hello-world", }, Vtags: []string{"latest"}, }, @@ -366,11 +320,8 @@ func TestPreprocess(t *testing.T) { { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "test", - }, Repository: &model.Repository{ - Name: "hello-world", + Name: "test/hello-world", }, Vtags: []string{"latest"}, }, @@ -409,3 +360,26 @@ func TestSchedule(t *testing.T) { require.Nil(t, err) assert.Equal(t, 1, n) } + +func TestReplaceNamespace(t *testing.T) { + // empty namespace + repository := "c" + namespace := "" + result := replaceNamespace(repository, namespace) + assert.Equal(t, "c", result) + // repository contains no "/" + repository = "c" + namespace = "n" + result = replaceNamespace(repository, namespace) + assert.Equal(t, "n/c", result) + // repository contains only one "/" + repository = "b/c" + namespace = "n" + result = replaceNamespace(repository, namespace) + assert.Equal(t, "n/c", result) + // repository contains more than one "/" + repository = "a/b/c" + namespace = "n" + result = replaceNamespace(repository, namespace) + assert.Equal(t, "n/c", result) +} diff --git a/src/replication/operation/scheduler/scheduler_test.go b/src/replication/operation/scheduler/scheduler_test.go index 40e049e33..94a25ca97 100644 --- a/src/replication/operation/scheduler/scheduler_test.go +++ b/src/replication/operation/scheduler/scheduler_test.go @@ -50,7 +50,7 @@ func TestStop(t *testing.T) { func generateData() ([]*ScheduleItem, error) { srcResource := &model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ + Repository: &model.Repository{ Name: "namespace1", }, Vtags: []string{"latest"}, @@ -62,7 +62,7 @@ func generateData() ([]*ScheduleItem, error) { } destResource := &model.Resource{ Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ + Repository: &model.Repository{ Name: "namespace2", }, Vtags: []string{"v1", "v2"}, diff --git a/src/replication/policy/manager/manager.go b/src/replication/policy/manager/manager.go index be2a8bbb2..cd4894963 100644 --- a/src/replication/policy/manager/manager.go +++ b/src/replication/policy/manager/manager.go @@ -18,7 +18,6 @@ import ( "encoding/json" "errors" "fmt" - "strings" "time" "github.com/goharbor/harbor/src/common/utils/log" @@ -58,19 +57,14 @@ func convertFromPersistModel(policy *persist_models.RepPolicy) (*model.Policy, e } } - // 1. parse SrcNamespaces to array - if len(policy.SrcNamespaces) > 0 { - ply.SrcNamespaces = strings.Split(policy.SrcNamespaces, ",") - } - - // 2. parse Filters + // parse Filters filters, err := parseFilters(policy.Filters) if err != nil { return nil, err } ply.Filters = filters - // 3. parse Trigger + // parse Trigger trigger, err := parseTrigger(policy.Trigger) if err != nil { return nil, err @@ -90,7 +84,6 @@ func convertToPersistModel(policy *model.Policy) (*persist_models.RepPolicy, err Name: policy.Name, Description: policy.Description, Creator: policy.Creator, - SrcNamespaces: strings.Join(policy.SrcNamespaces, ","), DestNamespace: policy.DestNamespace, Override: policy.Override, Enabled: policy.Enabled, diff --git a/src/replication/policy/manager/manager_test.go b/src/replication/policy/manager/manager_test.go index c7d27dbb9..44cbceda3 100644 --- a/src/replication/policy/manager/manager_test.go +++ b/src/replication/policy/manager/manager_test.go @@ -54,7 +54,6 @@ func Test_convertFromPersistModel(t *testing.T) { Description: "Policy Description", Creator: "someone", SrcRegistryID: 123, - SrcNamespaces: "ns1,ns2,ns3", DestRegistryID: 456, DestNamespace: "target_ns", ReplicateDeletion: true, @@ -70,7 +69,6 @@ func Test_convertFromPersistModel(t *testing.T) { SrcRegistry: &model.Registry{ ID: 123, }, - SrcNamespaces: []string{"ns1", "ns2", "ns3"}, DestRegistry: &model.Registry{ ID: 456, }, @@ -103,7 +101,6 @@ func Test_convertFromPersistModel(t *testing.T) { assert.Equal(t, tt.want.Description, got.Description) assert.Equal(t, tt.want.Creator, got.Creator) assert.Equal(t, tt.want.SrcRegistry.ID, got.SrcRegistry.ID) - assert.Equal(t, tt.want.SrcNamespaces, got.SrcNamespaces) assert.Equal(t, tt.want.DestRegistry.ID, got.DestRegistry.ID) assert.Equal(t, tt.want.DestNamespace, got.DestNamespace) assert.Equal(t, tt.want.Deletion, got.Deletion) @@ -133,7 +130,6 @@ func Test_convertToPersistModel(t *testing.T) { SrcRegistry: &model.Registry{ ID: 123, }, - SrcNamespaces: []string{"ns1", "ns2", "ns3"}, DestRegistry: &model.Registry{ ID: 456, }, @@ -149,7 +145,6 @@ func Test_convertToPersistModel(t *testing.T) { Description: "Policy Description", Creator: "someone", SrcRegistryID: 123, - SrcNamespaces: "ns1,ns2,ns3", DestRegistryID: 456, DestNamespace: "target_ns", ReplicateDeletion: true, @@ -175,7 +170,6 @@ func Test_convertToPersistModel(t *testing.T) { assert.Equal(t, tt.want.Description, got.Description) assert.Equal(t, tt.want.Creator, got.Creator) assert.Equal(t, tt.want.SrcRegistryID, got.SrcRegistryID) - assert.Equal(t, tt.want.SrcNamespaces, got.SrcNamespaces) assert.Equal(t, tt.want.DestRegistryID, got.DestRegistryID) assert.Equal(t, tt.want.DestNamespace, got.DestNamespace) assert.Equal(t, tt.want.ReplicateDeletion, got.ReplicateDeletion) diff --git a/src/replication/replication.go b/src/replication/replication.go index 12ac59ad4..ae5ee0314 100644 --- a/src/replication/replication.go +++ b/src/replication/replication.go @@ -28,9 +28,9 @@ import ( "github.com/goharbor/harbor/src/replication/registry" // register the Harbor adapter - _ "github.com/goharbor/harbor/src/replication/adapter/dockerhub" - // register the DockerHub adapter _ "github.com/goharbor/harbor/src/replication/adapter/harbor" + // register the DockerHub adapter + _ "github.com/goharbor/harbor/src/replication/adapter/dockerhub" // register the Native adapter _ "github.com/goharbor/harbor/src/replication/adapter/native" ) diff --git a/src/replication/transfer/chart/transfer_test.go b/src/replication/transfer/chart/transfer_test.go index 87ea171cf..45df2f68d 100644 --- a/src/replication/transfer/chart/transfer_test.go +++ b/src/replication/transfer/chart/transfer_test.go @@ -29,16 +29,13 @@ import ( type fakeRegistry struct{} -func (f *fakeRegistry) FetchCharts(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) { +func (f *fakeRegistry) FetchCharts(filters []*model.Filter) ([]*model.Resource, error) { return []*model.Resource{ { Type: model.ResourceTypeChart, Metadata: &model.ResourceMetadata{ - Namespace: &model.Namespace{ - Name: "library", - }, Repository: &model.Repository{ - Name: "harbor", + Name: "library/harbor", }, Vtags: []string{"0.2.0"}, }, diff --git a/src/replication/transfer/repository/transfer_test.go b/src/replication/transfer/repository/transfer_test.go index 2f698d1da..0e30db503 100644 --- a/src/replication/transfer/repository/transfer_test.go +++ b/src/replication/transfer/repository/transfer_test.go @@ -32,7 +32,7 @@ import ( type fakeRegistry struct{} -func (f *fakeRegistry) FetchImages([]string, []*model.Filter) ([]*model.Resource, error) { +func (f *fakeRegistry) FetchImages([]*model.Filter) ([]*model.Resource, error) { return nil, nil } diff --git a/src/replication/util/util.go b/src/replication/util/util.go index 375e16d60..bdd8861fd 100644 --- a/src/replication/util/util.go +++ b/src/replication/util/util.go @@ -17,6 +17,7 @@ package util import ( "net/http" "path/filepath" + "strings" "github.com/goharbor/harbor/src/common/utils/registry" ) @@ -30,3 +31,19 @@ func Match(pattern, str string) (bool, error) { func GetHTTPTransport(insecure bool) *http.Transport { return registry.GetHTTPTransport(insecure) } + +// ParseRepository parses the "repository" provided into two parts: namespace and the rest +// the string before the last "/" is the namespace part +// c -> [,c] +// b/c -> [b,c] +// a/b/c -> [a/b,c] +func ParseRepository(repository string) (string, string) { + if len(repository) == 0 { + return "", "" + } + index := strings.LastIndex(repository, "/") + if index == -1 { + return "", repository + } + return repository[:index], repository[index+1:] +} diff --git a/src/replication/util/util_test.go b/src/replication/util/util_test.go index c68af51e6..5a460c6ef 100644 --- a/src/replication/util/util_test.go +++ b/src/replication/util/util_test.go @@ -82,3 +82,26 @@ func TestGetHTTPTransport(t *testing.T) { transport = GetHTTPTransport(false) assert.False(t, transport.TLSClientConfig.InsecureSkipVerify) } + +func TestParseRepository(t *testing.T) { + // empty repository + repository := "" + namespace, rest := ParseRepository(repository) + assert.Equal(t, "", namespace) + assert.Equal(t, "", rest) + // repository contains no "/" + repository = "c" + namespace, rest = ParseRepository(repository) + assert.Equal(t, "", namespace) + assert.Equal(t, "c", rest) + // repository contains only one "/" + repository = "b/c" + namespace, rest = ParseRepository(repository) + assert.Equal(t, "b", namespace) + assert.Equal(t, "c", rest) + // repository contains more than one "/" + repository = "a/b/c" + namespace, rest = ParseRepository(repository) + assert.Equal(t, "a/b", namespace) + assert.Equal(t, "c", rest) +}