DTR replication support (#9512)

Adding DTR replication support

Signed-off-by: Greg Sidelinger <gate@ilive4code.net>

src/jobservice/job/impl/replication/replication.go

@@ -50,6 +50,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/adapter/helmhub"
 	// register the GitLab adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/gitlab"
+	// register the DTR adapter
+	_ "github.com/goharbor/harbor/src/replication/adapter/dtr"
 )
 
 // Replication implements the job interface

src/replication/adapter/dtr/adapter.go

@@ -0,0 +1,223 @@
+package dtr
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/goharbor/harbor/src/common/utils"
+	"github.com/goharbor/harbor/src/lib/log"
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/adapter/native"
+	"github.com/goharbor/harbor/src/replication/filter"
+	"github.com/goharbor/harbor/src/replication/model"
+)
+
+func init() {
+	err := adp.RegisterFactory(model.RegistryTypeDTR, new(factory))
+	if err != nil {
+		log.Errorf("failed to register factory for dtr: %v", err)
+		return
+	}
+	log.Infof("the factory of dtr adapter was registered")
+}
+
+type factory struct {
+}
+
+// Create ...
+func (f *factory) Create(r *model.Registry) (adp.Adapter, error) {
+	return newAdapter(r), nil
+}
+
+// AdapterPattern ...
+func (f *factory) AdapterPattern() *model.AdapterPattern {
+	return nil
+}
+
+var (
+	_ adp.Adapter = (*adapter)(nil)
+)
+
+type adapter struct {
+	*native.Adapter
+	registry     *model.Registry
+	url          string
+	username     string
+	token        string
+	clientDTRAPI *Client
+}
+
+func newAdapter(registry *model.Registry) *adapter {
+	return &adapter{
+		registry:     registry,
+		url:          registry.URL,
+		clientDTRAPI: NewClient(registry),
+		Adapter:      native.NewAdapter(registry),
+	}
+}
+
+// Info returns information of the registry
+func (a *adapter) Info() (*model.RegistryInfo, error) {
+	return &model.RegistryInfo{
+		Type: model.RegistryTypeAzureAcr,
+		SupportedResourceTypes: []model.ResourceType{
+			model.ResourceTypeImage,
+		},
+		SupportedResourceFilters: []*model.FilterStyle{
+			{
+				Type:  model.FilterTypeName,
+				Style: model.FilterStyleTypeText,
+			},
+			{
+				Type:  model.FilterTypeTag,
+				Style: model.FilterStyleTypeText,
+			},
+		},
+		SupportedTriggers: []model.TriggerType{
+			model.TriggerTypeManual,
+			model.TriggerTypeScheduled,
+		},
+	}, nil
+}
+
+// FetchArtifacts ...
+func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, error) {
+	var resources []*model.Resource
+
+	repositories, err := a.clientDTRAPI.getRepositories()
+	if err != nil {
+		log.Error("Failed to lookup repositories from DTR")
+		return nil, err
+	}
+	if len(repositories) == 0 {
+		return nil, nil
+	}
+	log.Debugf("%d of repositories pre filter", len(repositories))
+	repositories, err = filter.DoFilterRepositories(repositories, filters)
+	if err != nil {
+		return nil, err
+	}
+	log.Debugf("%d of repositories post filter", len(repositories))
+
+	runner := utils.NewLimitedConcurrentRunner(adp.MaxConcurrency)
+
+	for _, r := range repositories {
+		repo := r
+		runner.AddTask(func() error {
+			artifacts, err := a.listArtifacts(repo.Name, filters)
+			if err != nil {
+				return fmt.Errorf("failed to list artifacts of repository %s: %v", repo.Name, err)
+			}
+			log.Debugf("%s has %d artifacts", repo.Name, len(artifacts))
+
+			resources = append(resources, &model.Resource{
+				Type:     model.ResourceTypeImage,
+				Registry: a.registry,
+				Metadata: &model.ResourceMetadata{
+					Repository: &model.Repository{
+						Name: repo.Name,
+					},
+					Artifacts: artifacts,
+				},
+			})
+			return nil
+		})
+	}
+
+	if err = runner.Wait(); err != nil {
+		return nil, err
+	}
+
+	return resources, nil
+}
+
+// PrepareForPush creates docker repository in DTR
+func (a *adapter) PrepareForPush(resources []*model.Resource) error {
+	var dtrNamespaces []Account
+	var repos []string
+	namespaces := make(map[string]string)
+	for _, resource := range resources {
+		if resource == nil {
+			return errors.New("the resource cannot be null")
+		}
+		if resource.Metadata == nil {
+			return errors.New("the metadata of resource cannot be null")
+		}
+		if resource.Metadata.Repository == nil {
+			return errors.New("the namespace of resource cannot be null")
+		}
+		if len(resource.Metadata.Repository.Name) == 0 {
+			return errors.New("the name of namespace cannot be null")
+		}
+		path := strings.Split(resource.Metadata.Repository.Name, "/")
+		if len(path) > 0 {
+			namespaces[path[0]] = path[0]
+		}
+		if len(resource.Metadata.Repository.Name) > 0 {
+			repos = append(repos, resource.Metadata.Repository.Name)
+		}
+	}
+
+	dtrNamespaces, err := a.clientDTRAPI.getNamespaces()
+	if err != nil {
+		log.Errorf("Failed to lookup namespaces from DTR: %v", err)
+		return err
+	}
+
+	existingNamespaces := make(map[string]struct{})
+	for _, namespace := range dtrNamespaces {
+		existingNamespaces[namespace.Name] = struct{}{}
+	}
+
+	for namespace := range namespaces {
+		if _, ok := existingNamespaces[namespace]; ok {
+			log.Debugf("Namespace %s already existed in remote, skip create it", namespace)
+		} else {
+			err = a.clientDTRAPI.createNamespace(namespace)
+			if err != nil {
+				log.Errorf("Create Namespace %s error: %v", namespace, err)
+				return err
+			}
+		}
+	}
+
+	repositories, err := a.clientDTRAPI.getRepositories()
+	if err != nil {
+		log.Errorf("Failed to lookup repositories from DTR: %v", err)
+		return err
+	}
+
+	existingRepositories := make(map[string]struct{})
+	for _, repo := range repositories {
+		existingRepositories[repo.Name] = struct{}{}
+	}
+
+	for _, repo := range repos {
+		if _, ok := existingRepositories[repo]; ok {
+			log.Debugf("Repo %s already existed in remote, skip create it", repo)
+		} else {
+			err = a.clientDTRAPI.createRepository(repo)
+			if err != nil {
+				log.Errorf("Create Repository %s error: %v", repo, err)
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (a *adapter) listArtifacts(repository string, filters []*model.Filter) ([]*model.Artifact, error) {
+	tags, err := a.clientDTRAPI.getTags(repository)
+	if err != nil {
+		return nil, fmt.Errorf("List tags for repo '%s' error: %v", repository, err)
+	}
+	var artifacts []*model.Artifact
+	for _, tag := range tags {
+		artifacts = append(artifacts, &model.Artifact{
+			Tags: []string{tag},
+		})
+	}
+	return filter.DoFilterArtifacts(artifacts, filters)
+}

src/replication/adapter/dtr/adapter_test.go

@@ -0,0 +1,276 @@
+package dtr
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/goharbor/harbor/src/common/utils/test"
+	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInfo(t *testing.T) {
+	a := &adapter{}
+	info, err := a.Info()
+	assert.Nil(t, err)
+	assert.NotNil(t, info)
+	assert.EqualValues(t, 1, len(info.SupportedResourceTypes))
+	assert.EqualValues(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
+}
+
+func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Server) {
+	server := test.NewServer(
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/api/v0/repositories/mynamespace/myrepo/tags",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("X-Next-Page-Start", "")
+				w.Write([]byte(`[
+  {
+    "author": "string",
+    "createdAt": "2020-02-06T03:51:34.138Z",
+    "digest": "string",
+    "hashMismatch": true,
+    "inNotary": true,
+    "manifest": {
+      "architecture": "string",
+      "author": "string",
+      "configDigest": "string",
+      "configMediaType": "string",
+      "createdAt": "2020-02-06T03:51:34.138Z",
+      "digest": "string",
+      "dockerfile": [
+        {
+          "isEmpty": true,
+          "layerDigest": "string",
+          "line": "string",
+          "mediaType": "string",
+          "size": 0,
+          "urls": [
+            "string"
+          ]
+        }
+      ],
+      "mediaType": "string",
+      "os": "string",
+      "osVersion": "string",
+      "size": 0
+    },
+    "mirroring": {
+      "digest": "string",
+      "mirroringPolicyID": "string",
+      "remoteRepository": "string",
+      "remoteTag": "string"
+    },
+    "name": "mytag",
+    "promotion": {
+      "promotionPolicyID": "string",
+      "sourceRepository": "string",
+      "sourceTag": "string",
+      "string": "string"
+    },
+    "updatedAt": "2020-02-06T03:51:34.138Z"
+  },
+  {
+    "author": "string",
+    "createdAt": "2020-02-06T03:51:34.138Z",
+    "digest": "string",
+    "hashMismatch": true,
+    "inNotary": true,
+    "manifest": {
+      "architecture": "string",
+      "author": "string",
+      "configDigest": "string",
+      "configMediaType": "string",
+      "createdAt": "2020-02-06T03:51:34.138Z",
+      "digest": "string",
+      "dockerfile": [
+        {
+          "isEmpty": true,
+          "layerDigest": "string",
+          "line": "string",
+          "mediaType": "string",
+          "size": 0,
+          "urls": [
+            "string"
+          ]
+        }
+      ],
+      "mediaType": "string",
+      "os": "string",
+      "osVersion": "string",
+      "size": 0
+    },
+    "mirroring": {
+      "digest": "string",
+      "mirroringPolicyID": "string",
+      "remoteRepository": "string",
+      "remoteTag": "string"
+    },
+    "name": "v1.0.0",
+    "promotion": {
+      "promotionPolicyID": "string",
+      "sourceRepository": "string",
+      "sourceTag": "string",
+      "string": "string"
+    },
+    "updatedAt": "2020-02-06T03:51:34.138Z"
+  }  
+]`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/api/v0/repositories",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("X-Next-Page-Start", "")
+				w.Write([]byte(`{
+  "repositories": [
+    {
+      "enableManifestLists": true,
+      "id": "string",
+      "immutableTags": true,
+      "longDescription": "string",
+      "name": "myrepo",
+      "namespace": "mynamespace",
+      "namespaceType": "user",
+      "pulls": 0,
+      "pushes": 0,
+      "scanOnPush": true,
+      "shortDescription": "string",
+      "tagLimit": 0,
+      "visibility": "public"
+    }
+  ]
+}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/enzi/v0/accounts",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("X-Next-Page-Start", "")
+				w.Write([]byte(`{
+  "accounts": [
+    {
+      "fullName": "string",
+      "id": "string",
+      "isActive": true,
+      "isAdmin": true,
+      "isImported": true,
+      "isOrg": true,
+      "membersCount": 0,
+      "name": "mynamespace",
+      "teamsCount": 0
+    }
+  ],
+  "nextPageStart": "string",
+  "orgsCount": 0,
+  "resourceCount": 0,
+  "usersCount": 0
+}`))
+			},
+		})
+
+	registry := &model.Registry{
+		Type: model.RegistryTypeDTR,
+		URL:  server.URL,
+	}
+
+	if hasCred {
+		registry.Credential = &model.Credential{
+			AccessKey:    "admin",
+			AccessSecret: "password",
+		}
+	}
+
+	factory, err := adp.GetFactory(model.RegistryTypeDTR)
+	assert.Nil(t, err)
+	assert.NotNil(t, factory)
+	a := newAdapter(registry)
+
+	assert.Nil(t, err)
+	return a, server
+}
+
+func TestAdapter_PrepareForPush(t *testing.T) {
+	a, s := getMockAdapter(t, true, true)
+	defer s.Close()
+	resources := []*model.Resource{
+		{
+			Type: model.ResourceTypeImage,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: "mynamespace/myrepo",
+				},
+			},
+		},
+	}
+	err := a.PrepareForPush(resources)
+	assert.Nil(t, err)
+}
+
+func TestAdapter_Info(t *testing.T) {
+	a, s := getMockAdapter(t, true, true)
+	defer s.Close()
+	info, err := a.Info()
+	assert.Nil(t, err)
+	assert.NotNil(t, info)
+	assert.EqualValues(t, 1, len(info.SupportedResourceTypes))
+	assert.EqualValues(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
+}
+
+func TestAdapter_FetchArtifacts(t *testing.T) {
+	a, s := getMockAdapter(t, true, true)
+	defer s.Close()
+	filters := []*model.Filter{}
+	r, err := a.FetchArtifacts(filters)
+	assert.Nil(t, err)
+	assert.EqualValues(t, 1, len(r))
+	assert.EqualValues(t, 2, len(r[0].Metadata.Artifacts))
+
+}
+
+func TestAdapter_FetchArtifactsFiltered(t *testing.T) {
+	a, s := getMockAdapter(t, true, true)
+	defer s.Close()
+
+	testCases := []struct {
+		nameFilter string
+		tagFilter  string
+		repos      int
+		artifacts  int
+	}{
+		{"mynamespace/**", "**", 1, 2},
+		{"mynamespace/myrepo", "**", 1, 2},
+		{"mynamespace/myrepo", "v1.0.0", 1, 1},
+		{"mynamespace/myrepo", "notfound", 1, 0},
+	}
+	for _, tc := range testCases {
+
+		filters := []*model.Filter{
+			{
+				Type:  model.FilterTypeName,
+				Value: tc.nameFilter,
+			},
+			{
+				Type:  model.FilterTypeTag,
+				Value: tc.tagFilter,
+			},
+		}
+		r, err := a.FetchArtifacts(filters)
+		if err != nil {
+			t.Fatalf("could fetch artifacts for repo=%q tag=%s", tc.nameFilter, tc.tagFilter)
+		}
+		if len(r) != tc.repos {
+			t.Fatalf("wrong number of repos returned for repo=%q tag=%s, wanted %d got %d", tc.nameFilter, tc.tagFilter, tc.repos, len(r))
+		}
+		if len(r[0].Metadata.Artifacts) != tc.artifacts {
+			t.Fatalf("wrong number of artifacts returned for repo=%q tag=%s, wanted %d got %d", tc.nameFilter, tc.tagFilter, tc.artifacts, len(r))
+		}
+
+	}
+
+}

src/replication/adapter/dtr/client.go

@@ -0,0 +1,328 @@
+package dtr
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"reflect"
+	"strings"
+
+	common_http "github.com/goharbor/harbor/src/common/http"
+
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/replication/util"
+)
+
+// Client is a client to interact with DTR
+type Client struct {
+	client   *common_http.Client
+	url      string
+	username string
+	password string
+}
+
+// NewClient creates a new DTR client.
+func NewClient(registry *model.Registry) *Client {
+
+	client := &Client{
+		url:      registry.URL,
+		username: registry.Credential.AccessKey,
+		password: registry.Credential.AccessSecret,
+		client: common_http.NewClient(
+			&http.Client{
+				Transport: util.GetHTTPTransport(registry.Insecure),
+			}),
+	}
+	return client
+}
+
+// getAndIteratePagination will iterator over a paginated response from DTR
+func (c *Client) getAndIteratePagination(endpoint string, v interface{}) error {
+	urlAPI, err := url.Parse(endpoint)
+	if err != nil {
+		return err
+	}
+
+	rv := reflect.ValueOf(v)
+	if rv.Kind() != reflect.Ptr {
+		return errors.New("v should be a pointer to a slice")
+	}
+	elemType := rv.Elem().Type()
+	if elemType.Kind() != reflect.Slice {
+		return errors.New("v should be a pointer to a slice")
+	}
+
+	resources := reflect.Indirect(reflect.New(elemType))
+	for len(endpoint) > 0 {
+		req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+		if err != nil {
+			return err
+		}
+		req.SetBasicAuth(c.username, c.password)
+		resp, err := c.client.Do(req)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode < 200 || resp.StatusCode > 299 {
+			return &common_http.Error{
+				Code:    resp.StatusCode,
+				Message: string(data),
+			}
+		}
+
+		res := reflect.New(elemType)
+		if err = json.Unmarshal(data, res.Interface()); err != nil {
+			log.Errorf("Failed to parse json response: %v", string(data))
+			return err
+		}
+		resources = reflect.AppendSlice(resources, reflect.Indirect(res))
+		endpoint = ""
+
+		nextPage := resp.Header.Get("X-Next-Page-Start")
+		if len(nextPage) > 0 {
+			query := urlAPI.Query()
+			query.Set("pageStart", nextPage)
+			endpoint = urlAPI.Scheme + "://" + urlAPI.Host + urlAPI.Path + "?" + query.Encode()
+		}
+	}
+	rv.Elem().Set(resources)
+	return nil
+}
+
+// getRepositories returns a list of repositories in DTR
+func (c *Client) getRepositories() ([]*model.Repository, error) {
+	var repositories []Repository
+	var dtrRepositories Repositories
+
+	endpoint := fmt.Sprintf("%s/api/v0/repositories?pageSize=100", c.url)
+	urlAPI, err := url.Parse(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	for len(endpoint) > 0 {
+		req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+		if err != nil {
+			return nil, err
+		}
+		req.SetBasicAuth(c.username, c.password)
+		resp, err := c.client.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		if resp.StatusCode < 200 || resp.StatusCode > 299 {
+			return nil, &common_http.Error{
+				Code:    resp.StatusCode,
+				Message: string(data),
+			}
+		}
+
+		if err = json.Unmarshal(data, &dtrRepositories); err != nil {
+			log.Errorf("Failed to parse json response")
+			log.Errorf("%v", err)
+			log.Errorf("%s", string(data))
+
+			return nil, err
+		}
+
+		// merge the arrays
+		repositories = append(repositories, dtrRepositories.Repositories...)
+		endpoint = ""
+
+		nextPage := resp.Header.Get("X-Next-Page-Start")
+		if len(nextPage) > 0 {
+			query := urlAPI.Query()
+			query.Set("pageStart", nextPage)
+			endpoint = urlAPI.Scheme + "://" + urlAPI.Host + urlAPI.Path + "?" + query.Encode()
+		}
+	}
+
+	result := []*model.Repository{}
+
+	for _, repository := range repositories {
+		log.Debugf("Processing DTR repo %s", repository.Name)
+		result = append(result, &model.Repository{
+			Name: fmt.Sprintf("%s/%s", repository.Namespace, repository.Name),
+		})
+	}
+	return result, nil
+}
+
+// getTags looks up a repositories tags in DTR
+func (c *Client) getTags(repository string) ([]string, error) {
+	var tags []*Tag
+	// This assumes repository is of form namespace/repo
+	urlAPI := fmt.Sprintf("%s/api/v0/repositories/%s/tags?pageSize=100", c.url, repository)
+	log.Debugf("Looking up tags for %s at %s", repository, urlAPI)
+	if err := c.getAndIteratePagination(urlAPI, &tags); err != nil {
+		log.Debugf("Failed looking up tags for %s at %s", repository, urlAPI)
+		return nil, err
+	}
+
+	var result []string
+	for _, tag := range tags {
+		result = append(result, tag.Name)
+	}
+	return result, nil
+}
+
+// getNamespaces returns DTR namespaces.  DTR also calles these orgs and accounts depending on where you look
+func (c *Client) getNamespaces() ([]Account, error) {
+	var accounts []Account
+	var response Accounts
+
+	endpoint := fmt.Sprintf("%s/enzi/v0/accounts?limit=100", c.url)
+	urlAPI, err := url.Parse(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	for len(endpoint) > 0 {
+		req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+		if err != nil {
+			return nil, err
+		}
+		req.SetBasicAuth(c.username, c.password)
+		resp, err := c.client.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		if resp.StatusCode < 200 || resp.StatusCode > 299 {
+			return nil, &common_http.Error{
+				Code:    resp.StatusCode,
+				Message: string(data),
+			}
+		}
+
+		if err = json.Unmarshal(data, &response); err != nil {
+			log.Errorf("Failed to parse json response")
+			log.Errorf("%v", err)
+			log.Errorf("%s", string(data))
+
+			return nil, err
+		}
+
+		accounts = append(accounts, response.Accounts...)
+		endpoint = ""
+
+		nextPage := resp.Header.Get("X-Next-Page-Start")
+		if len(nextPage) > 0 {
+			query := urlAPI.Query()
+			query.Set("start", nextPage)
+			endpoint = urlAPI.Scheme + "://" + urlAPI.Host + urlAPI.Path + "?" + query.Encode()
+		}
+	}
+
+	return accounts, nil
+}
+
+// createRepository creates a repository in DTR.  The namespace/org/account must already exist.
+func (c *Client) createRepository(repository string) error {
+	var namespace string
+	var repositoryName string
+
+	path := strings.Split(repository, "/")
+	if len(path) > 1 {
+		namespace = path[0]
+		repositoryName = path[1]
+	} else {
+		return errors.New("repository did not contain a namespace")
+	}
+
+	repo := newDefaultDTRRepository(repositoryName)
+	body, err := json.Marshal(repo)
+	if err != nil {
+		return err
+	}
+
+	urlAPI := fmt.Sprintf("%s/api/v0/repositories/%s", c.url, namespace)
+	log.Debugf("Creating repo %s in DTR at %s", repositoryName, urlAPI)
+	req, err := http.NewRequest(http.MethodPost, urlAPI, bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.SetBasicAuth(c.username, c.password)
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusCreated {
+		return nil
+	}
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return &common_http.Error{
+		Code:    resp.StatusCode,
+		Message: string(b),
+	}
+}
+
+// createNamespace creates a namespace in DTR
+// This actually hits the enzi API which appears to map to the UCP
+// accounts API.  The DTR v0 api has no official way to create a
+// namespace as of 2.7.1
+// this operation needs admin access
+func (c *Client) createNamespace(namespace string) error {
+	ns := newDefaultDTRNamespace(namespace)
+	body, err := json.Marshal(ns)
+	if err != nil {
+		return err
+	}
+
+	urlAPI := fmt.Sprintf("%s/enzi/v0/accounts", c.url)
+	log.Debugf("Creating namespace %s in DTR at %s", namespace, urlAPI)
+	req, err := http.NewRequest(http.MethodPost, urlAPI, bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.SetBasicAuth(c.username, c.password)
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusCreated {
+		return nil
+	}
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return &common_http.Error{
+		Code:    resp.StatusCode,
+		Message: string(b),
+	}
+}

src/replication/adapter/dtr/client_test.go

@@ -0,0 +1,196 @@
+package dtr
+
+import (
+	"net/http"
+	"testing"
+
+	common_http "github.com/goharbor/harbor/src/common/http"
+	"github.com/goharbor/harbor/src/common/utils/test"
+	"github.com/goharbor/harbor/src/replication/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestProjects(t *testing.T) {
+	server := test.NewServer(
+		&test.RequestHandlerMapping{
+			Method:  http.MethodPost,
+			Pattern: "/api/v0/repositories",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(201)
+				w.Write([]byte(`{
+  "enableManifestLists": true,
+  "immutableTags": true,
+  "longDescription": "string",
+  "name": "mynamespace/myrepo",
+  "scanOnPush": true,
+  "shortDescription": "string",
+  "tagLimit": 0,
+  "visibility": "public"
+}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/api/v0/repositories/mynamespace/myrepo/tags",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("X-Next-Page-Start", "")
+				w.Write([]byte(`[
+  {
+    "author": "string",
+    "createdAt": "2020-02-06T03:51:34.138Z",
+    "digest": "string",
+    "hashMismatch": true,
+    "inNotary": true,
+    "manifest": {
+      "architecture": "string",
+      "author": "string",
+      "configDigest": "string",
+      "configMediaType": "string",
+      "createdAt": "2020-02-06T03:51:34.138Z",
+      "digest": "string",
+      "dockerfile": [
+        {
+          "isEmpty": true,
+          "layerDigest": "string",
+          "line": "string",
+          "mediaType": "string",
+          "size": 0,
+          "urls": [
+            "string"
+          ]
+        }
+      ],
+      "mediaType": "string",
+      "os": "string",
+      "osVersion": "string",
+      "size": 0
+    },
+    "mirroring": {
+      "digest": "string",
+      "mirroringPolicyID": "string",
+      "remoteRepository": "string",
+      "remoteTag": "string"
+    },
+    "name": "mytag",
+    "promotion": {
+      "promotionPolicyID": "string",
+      "sourceRepository": "string",
+      "sourceTag": "string",
+      "string": "string"
+    },
+    "updatedAt": "2020-02-06T03:51:34.138Z"
+  }
+]`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/api/v0/repositories/mynamespace/missingimage/tags",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusNotFound)
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/api/v0/repositories",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("X-Next-Page-Start", "")
+				w.Write([]byte(`{
+  "repositories": [
+    {
+      "enableManifestLists": true,
+      "id": "string",
+      "immutableTags": true,
+      "longDescription": "string",
+      "name": "myrepo",
+      "namespace": "mynamespace",
+      "namespaceType": "user",
+      "pulls": 0,
+      "pushes": 0,
+      "scanOnPush": true,
+      "shortDescription": "string",
+      "tagLimit": 0,
+      "visibility": "public"
+    }
+  ]
+}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/enzi/v0/accounts",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("X-Next-Page-Start", "")
+				w.Write([]byte(`{
+  "accounts": [
+    {
+      "fullName": "string",
+      "id": "string",
+      "isActive": true,
+      "isAdmin": true,
+      "isImported": true,
+      "isOrg": true,
+      "membersCount": 0,
+      "name": "mynamespace",
+      "teamsCount": 0
+    }
+  ],
+  "nextPageStart": "string",
+  "orgsCount": 0,
+  "resourceCount": 0,
+  "usersCount": 0
+}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodPost,
+			Pattern: "/enzi/v0/accounts",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(201)
+				w.Write([]byte(`{
+  "fullName": "string",
+  "isActive": true,
+  "isAdmin": false,
+  "isOrg": true,
+  "name": "mynamespace",
+  "password": "string",
+  "searchLDAP": false
+}`))
+			},
+		})
+	client := &Client{
+		url:      server.URL,
+		username: "test",
+		client: common_http.NewClient(
+			&http.Client{
+				Transport: util.GetHTTPTransport(true),
+			}),
+	}
+
+	repositories, e := client.getRepositories()
+	require.Nil(t, e)
+	assert.Equal(t, 1, len(repositories))
+	assert.Equal(t, "mynamespace/myrepo", repositories[0].Name)
+
+	namespaces, e := client.getNamespaces()
+	require.Nil(t, e)
+	assert.Equal(t, 1, len(namespaces))
+	assert.Equal(t, "mynamespace", namespaces[0].Name)
+
+	tags, e := client.getTags("mynamespace/myrepo")
+	require.Nil(t, e)
+	assert.Equal(t, 1, len(tags))
+	assert.Equal(t, "mytag", tags[0])
+
+	// List tags for missign image
+	_, e = client.getTags("mynamespace/missingimage")
+	require.NotNil(t, e)
+
+	e = client.createRepository("mynamespace/myrepo")
+	require.Nil(t, e)
+
+	e = client.createNamespace("mynamespace")
+	require.Nil(t, e)
+
+}

src/replication/adapter/dtr/types.go

@@ -0,0 +1,96 @@
+package dtr
+
+// Accounts describes the DTR accounts API response
+type Accounts struct {
+	Accounts []Account `json:"accounts"`
+}
+
+// Account describes the DTR account API response
+// DTR calls them namespaces/orgs however they are
+// accessed under an accounts API, keep that straight.
+type Account struct {
+	Name       string `json:"name,omitempty"`
+	ID         string `json:"id,omitempty"`
+	FullName   string `json:"fullName,omitempty"`
+	IsOrg      bool   `json:"isOrg,omitempty"`
+	IsAdmin    bool   `json:"isAdmin,omitempty"`
+	IsActive   bool   `json:"isActive,omitempty"`
+	IsImported bool   `json:"isImported,omitempty"`
+}
+
+// namespaceCreate describes the format for a new namespace in DTR
+type namespaceCreate struct {
+	FullName   string `json:"fullName"`
+	IsOrg      bool   `json:"isOrg"`
+	IsAdmin    bool   `json:"isAdmin,omitempty"`
+	IsActive   bool   `json:"isActive,omitempty"`
+	Name       string `json:"name"`
+	Password   string `json:"password,omitempty"`
+	SearchLDAP bool   `json:"searchLDAP,omitempty"`
+}
+
+// newDefaultDTRNamespace is the defaults values for a new namespace
+func newDefaultDTRNamespace(name string) *namespaceCreate {
+	return &namespaceCreate{
+		FullName: name,
+		IsOrg:    true,
+		Name:     name,
+	}
+}
+
+// Repositories describes the DTR repositories API response
+type Repositories struct {
+	Repositories []Repository `json:"repositories"`
+}
+
+// Repository describes a repository in DTR
+type Repository struct {
+	EnableManifestLists bool   `json:"enableManifestLists,omitempty"`
+	ID                  string `json:"id,omitempty"`
+	ImmutableTags       bool   `json:"immutableTags,omitempty"`
+	LongDescription     string `json:"longDescription,omitempty"`
+	Name                string `json:"name"`
+	Namespace           string `json:"namespace"`
+	NamespaceType       string `json:"namespaceType,omitempty"`
+	Pulls               int64  `json:"pulls,omitempty"`
+	Pushes              int64  `json:"pushes,omitempty"`
+	ScanOnPush          bool   `json:"scanOnPush,omitempty"`
+	ShortDescription    string `json:"shortDescription,omitempty"`
+	TagLimit            int64  `json:"tagLimit,omitempty"`
+	Visibility          string `json:"visibility,omitempty"`
+}
+
+// repositoryCreate describes the format for a new repository in DTR
+type repositoryCreate struct {
+	EnableManifestLists bool   `json:"enableManifestLists,omitempty"`
+	ImmutableTags       bool   `json:"immutableTags,omitempty"`
+	LongDescription     string `json:"longDescription,omitempty"`
+	Name                string `json:"name"`
+	ScanOnPush          bool   `json:"scanOnPush,omitempty"`
+	ShortDescription    string `json:"shortDescription,omitempty"`
+	TagLimit            int64  `json:"tagLimit,omitempty"`
+	Visibility          string `json:"visibility"`
+}
+
+// newDefaultDTRRepository is the defaults values for a new repository
+func newDefaultDTRRepository(name string) *repositoryCreate {
+	return &repositoryCreate{
+		EnableManifestLists: true,
+		ImmutableTags:       false,
+		Name:                name,
+		ScanOnPush:          false,
+		TagLimit:            0,
+		Visibility:          "private",
+	}
+}
+
+// Tag describes the DTR tag API response
+type Tag struct {
+	Author       string `json:"author,omitempty"`
+	CreatedAt    string `json:"createdAt,omitempty"`
+	Digest       string `json:"digest,omitempty"`
+	HashMismatch bool   `json:"hashMismatch"`
+	InNotary     bool   `json:"inNotary,omitempty"`
+	Name         string `json:"name"`
+	UpdatedAt    string `json:"updatedAt,omitempty"`
+}

src/replication/model/registry.go

@@ -31,6 +31,7 @@ const (
 	RegistryTypeJfrogArtifactory RegistryType = "jfrog-artifactory"
 	RegistryTypeQuay             RegistryType = "quay"
 	RegistryTypeGitLab           RegistryType = "gitlab"
+	RegistryTypeDTR              RegistryType = "dtr"
 
 	RegistryTypeHelmHub RegistryType = "helm-hub"
 

src/replication/replication.go

@@ -51,6 +51,8 @@ import (
 	_ "github.com/goharbor/harbor/src/replication/adapter/helmhub"
 	// register the GitLab adapter
 	_ "github.com/goharbor/harbor/src/replication/adapter/gitlab"
+	// register the DTR adapter
+	_ "github.com/goharbor/harbor/src/replication/adapter/dtr"
 )
 
 var (