Merge branch 'master' of https://github.com/goharbor/harbor into project-quota-dev

src/common/dao/group/usergroup.go

@@ -15,10 +15,8 @@
 package group
 
 import (
-	"strings"
 	"time"
 
-	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/common/utils"
 
 	"github.com/goharbor/harbor/src/common/dao"
@@ -139,20 +137,3 @@ func OnBoardUserGroup(g *models.UserGroup, keyAttribute string, combinedKeyAttri
 
 	return nil
 }
-
-// GetGroupDNQueryCondition get the part of IN ('XXX', 'XXX') condition
-func GetGroupDNQueryCondition(userGroupList []*models.UserGroup) string {
-	result := make([]string, 0)
-	count := 0
-	for _, userGroup := range userGroupList {
-		if userGroup.GroupType == common.LdapGroupType {
-			result = append(result, "'"+userGroup.LdapGroupDN+"'")
-			count++
-		}
-	}
-	// No LDAP Group found
-	if count == 0 {
-		return ""
-	}
-	return strings.Join(result, ",")
-}

src/common/dao/group/usergroup_test.go

@@ -47,6 +47,8 @@ func TestMain(m *testing.M) {
 		initSqls := []string{
 			"insert into harbor_user (username, email, password, realname)  values ('member_test_01', 'member_test_01@example.com', '123456', 'member_test_01')",
 			"insert into project (name, owner_id) values ('member_test_01', 1)",
+			`insert into project (name, owner_id) values ('group_project2', 1)`,
+			`insert into project (name, owner_id) values ('group_project_private', 1)`,
 			"insert into user_group (group_name, group_type, ldap_group_dn) values ('test_group_01', 1, 'cn=harbor_users,ou=sample,ou=vmware,dc=harbor,dc=com')",
 			"update project set owner_id = (select user_id from harbor_user where username = 'member_test_01') where name = 'member_test_01'",
 			"insert into project_member (project_id, entity_id, entity_type, role) values ( (select project_id from project where name = 'member_test_01') , (select user_id from harbor_user where username = 'member_test_01'), 'u', 1)",
@@ -55,6 +57,8 @@ func TestMain(m *testing.M) {
 
 		clearSqls := []string{
 			"delete from project where name='member_test_01'",
+			"delete from project where name='group_project2'",
+			"delete from project where name='group_project_private'",
 			"delete from harbor_user where username='member_test_01' or username='pm_sample'",
 			"delete from user_group",
 			"delete from project_member",
@@ -175,7 +179,7 @@ func TestUpdateUserGroup(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			fmt.Printf("id=%v", createdUserGroupID)
+			fmt.Printf("id=%v\n", createdUserGroupID)
 			if err := UpdateUserGroupName(tt.args.id, tt.args.groupName); (err != nil) != tt.wantErr {
 				t.Errorf("UpdateUserGroup() error = %v, wantErr %v", err, tt.wantErr)
 				userGroup, err := GetUserGroup(tt.args.id)
@@ -249,40 +253,6 @@ func TestOnBoardUserGroup(t *testing.T) {
 	}
 }
 
-func TestGetGroupDNQueryCondition(t *testing.T) {
-	userGroupList := []*models.UserGroup{
-		{
-			GroupName:   "sample1",
-			GroupType:   1,
-			LdapGroupDN: "cn=sample1_users,ou=groups,dc=example,dc=com",
-		},
-		{
-			GroupName:   "sample2",
-			GroupType:   1,
-			LdapGroupDN: "cn=sample2_users,ou=groups,dc=example,dc=com",
-		},
-		{
-			GroupName:   "sample3",
-			GroupType:   0,
-			LdapGroupDN: "cn=sample3_users,ou=groups,dc=example,dc=com",
-		},
-	}
-
-	groupQueryConditions := GetGroupDNQueryCondition(userGroupList)
-	expectedConditions := `'cn=sample1_users,ou=groups,dc=example,dc=com','cn=sample2_users,ou=groups,dc=example,dc=com'`
-	if groupQueryConditions != expectedConditions {
-		t.Errorf("Failed to GetGroupDNQueryCondition, expected %v, actual %v", expectedConditions, groupQueryConditions)
-	}
-	var userGroupList2 []*models.UserGroup
-	groupQueryCondition2 := GetGroupDNQueryCondition(userGroupList2)
-	if len(groupQueryCondition2) > 0 {
-		t.Errorf("Failed to GetGroupDNQueryCondition, expected %v, actual %v", "", groupQueryCondition2)
-	}
-	groupQueryCondition3 := GetGroupDNQueryCondition(nil)
-	if len(groupQueryCondition3) > 0 {
-		t.Errorf("Failed to GetGroupDNQueryCondition, expected %v, actual %v", "", groupQueryCondition3)
-	}
-}
 func TestGetGroupProjects(t *testing.T) {
 	userID, err := dao.Register(models.User{
 		Username: "grouptestu09",
@@ -322,7 +292,6 @@ func TestGetGroupProjects(t *testing.T) {
 	})
 	defer project.DeleteProjectMemberByID(pmid)
 	type args struct {
-		groupDNCondition string
 		query *models.ProjectQueryParam
 	}
 	member := &models.MemberQuery{
@@ -335,19 +304,17 @@ func TestGetGroupProjects(t *testing.T) {
 		wantErr  bool
 	}{
 		{"Query with group DN",
-			args{"'cn=harbor_users,ou=groups,dc=example,dc=com'",
+			args{&models.ProjectQueryParam{
-				&models.ProjectQueryParam{
 				Member: member,
 			}},
 			1, false},
 		{"Query without group DN",
-			args{"",
+			args{&models.ProjectQueryParam{}},
-				&models.ProjectQueryParam{}},
 			1, false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := dao.GetGroupProjects(tt.args.groupDNCondition, tt.args.query)
+			got, err := dao.GetGroupProjects([]int{groupID}, tt.args.query)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("GetGroupProjects() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -392,7 +359,6 @@ func TestGetTotalGroupProjects(t *testing.T) {
 	})
 	defer project.DeleteProjectMemberByID(pmid)
 	type args struct {
-		groupDNCondition string
 		query *models.ProjectQueryParam
 	}
 	tests := []struct {
@@ -401,18 +367,16 @@ func TestGetTotalGroupProjects(t *testing.T) {
 		wantSize int
 		wantErr  bool
 	}{
-		{"Query with group DN",
+		{"Query with group ID",
-			args{"'cn=harbor_users,ou=groups,dc=example,dc=com'",
+			args{&models.ProjectQueryParam{}},
-				&models.ProjectQueryParam{}},
 			1, false},
-		{"Query without group DN",
+		{"Query without group ID",
-			args{"",
+			args{&models.ProjectQueryParam{}},
-				&models.ProjectQueryParam{}},
 			1, false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := dao.GetTotalGroupProjects(tt.args.groupDNCondition, tt.args.query)
+			got, err := dao.GetTotalGroupProjects([]int{groupID}, tt.args.query)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("GetGroupProjects() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -423,3 +387,44 @@ func TestGetTotalGroupProjects(t *testing.T) {
 		})
 	}
 }
+func TestGetRolesByLDAPGroup(t *testing.T) {
+
+	userGroupList, err := QueryUserGroup(models.UserGroup{LdapGroupDN: "cn=harbor_users,ou=sample,ou=vmware,dc=harbor,dc=com", GroupType: 1})
+	if err != nil || len(userGroupList) < 1 {
+		t.Errorf("failed to query user group, err %v", err)
+	}
+	project, err := dao.GetProjectByName("member_test_01")
+	if err != nil {
+		t.Errorf("Error occurred when Get project by name: %v", err)
+	}
+	privateProject, err := dao.GetProjectByName("group_project_private")
+	if err != nil {
+		t.Errorf("Error occurred when Get project by name: %v", err)
+	}
+
+	type args struct {
+		projectID int64
+		groupIDs  []int
+	}
+	tests := []struct {
+		name     string
+		args     args
+		wantSize int
+		wantErr  bool
+	}{
+		{"Check normal", args{projectID: project.ProjectID, groupIDs: []int{userGroupList[0].ID}}, 1, false},
+		{"Check non exist", args{projectID: privateProject.ProjectID, groupIDs: []int{9999}}, 0, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := dao.GetRolesByGroupID(tt.args.projectID, tt.args.groupIDs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("TestGetRolesByLDAPGroup() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if len(got) != tt.wantSize {
+				t.Errorf("TestGetRolesByLDAPGroup() = %v, want %v", len(got), tt.wantSize)
+			}
+		})
+	}
+}

src/common/dao/project.go

@@ -156,18 +156,19 @@ func GetProjects(query *models.ProjectQueryParam) ([]*models.Project, error) {
 
 // GetGroupProjects - Get user's all projects, including user is the user member of this project
 // and the user is in the group which is a group member of this project.
-func GetGroupProjects(groupDNCondition string, query *models.ProjectQueryParam) ([]*models.Project, error) {
+func GetGroupProjects(groupIDs []int, query *models.ProjectQueryParam) ([]*models.Project, error) {
 	sql, params := projectQueryConditions(query)
 	sql = `select distinct p.project_id, p.name, p.owner_id, 
 				p.creation_time, p.update_time ` + sql
-	if len(groupDNCondition) > 0 {
+	groupIDCondition := JoinNumberConditions(groupIDs)
+	if len(groupIDs) > 0 {
 		sql = fmt.Sprintf(
 			`%s union select distinct p.project_id, p.name, p.owner_id, p.creation_time, p.update_time  
 		     from project p 
 		     left join project_member pm on p.project_id = pm.project_id
-		     left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g' and ug.group_type = 1
+		     left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g' 
-			 where ug.ldap_group_dn in ( %s ) order by name`,
+			 where ug.id in ( %s ) order by name`,
-			sql, groupDNCondition)
+			sql, groupIDCondition)
 	}
 	sqlStr, queryParams := CreatePagination(query, sql, params)
 	log.Debugf("query sql:%v", sql)
@@ -178,10 +179,11 @@ func GetGroupProjects(groupDNCondition string, query *models.ProjectQueryParam)
 
 // GetTotalGroupProjects - Get the total count of projects, including  user is the member of this project and the
 // user is in the group, which is the group member of this project.
-func GetTotalGroupProjects(groupDNCondition string, query *models.ProjectQueryParam) (int, error) {
+func GetTotalGroupProjects(groupIDs []int, query *models.ProjectQueryParam) (int, error) {
 	var sql string
 	sqlCondition, params := projectQueryConditions(query)
-	if len(groupDNCondition) == 0 {
+	groupIDCondition := JoinNumberConditions(groupIDs)
+	if len(groupIDs) == 0 {
 		sql = `select count(1) ` + sqlCondition
 	} else {
 		sql = fmt.Sprintf(
@@ -189,9 +191,9 @@ func GetTotalGroupProjects(groupDNCondition string, query *models.ProjectQueryPa
 			   from ( select  p.project_id %s  union select  p.project_id  
 			   from project p 
 			   left join project_member pm on p.project_id = pm.project_id
-			   left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g' and ug.group_type = 1
+			   left join user_group ug on ug.id = pm.entity_id and pm.entity_type = 'g' 
-			   where ug.ldap_group_dn in ( %s )) t`,
+			   where ug.id in ( %s )) t`,
-			sqlCondition, groupDNCondition)
+			sqlCondition, groupIDCondition)
 	}
 	log.Debugf("query sql:%v", sql)
 	var count int
@@ -291,24 +293,24 @@ func DeleteProject(id int64) error {
 	return err
 }
 
-// GetRolesByLDAPGroup - Get Project roles of the
+// GetRolesByGroupID - Get Project roles of the
-// specified group DN is a member of current project
+// specified group is a member of current project
-func GetRolesByLDAPGroup(projectID int64, groupDNCondition string) ([]int, error) {
+func GetRolesByGroupID(projectID int64, groupIDs []int) ([]int, error) {
 	var roles []int
-	if len(groupDNCondition) == 0 {
+	if len(groupIDs) == 0 {
 		return roles, nil
 	}
+	groupIDCondition := JoinNumberConditions(groupIDs)
 	o := GetOrmer()
-	// Because an LDAP user can be memberof multiple groups,
 	// the role is in descent order (1-admin, 2-developer, 3-guest, 4-master), use min to select the max privilege role.
 	sql := fmt.Sprintf(
 		`select min(pm.role) from project_member pm 
 		left join user_group ug on pm.entity_type = 'g' and pm.entity_id = ug.id 
-		where ug.ldap_group_dn in ( %s ) and pm.project_id = ? `,
+		where ug.id in ( %s ) and pm.project_id = ?`,
-		groupDNCondition)
+		groupIDCondition)
 	log.Debugf("sql:%v", sql)
 	if _, err := o.Raw(sql, projectID).QueryRows(&roles); err != nil {
-		log.Warningf("Error in GetRolesByLDAPGroup, error: %v", err)
+		log.Warningf("Error in GetRolesByGroupID, error: %v", err)
 		return nil, err
 	}
 	// If there is no row selected, the min returns an empty row, to avoid return 0 as role

src/common/dao/project/projectmember.go

@@ -148,16 +148,3 @@ func SearchMemberByName(projectID int64, entityName string) ([]*models.Member, e
 	_, err := o.Raw(sql, queryParam).QueryRows(&members)
 	return members, err
 }
-
-// GetRolesByGroup -- Query group roles
-func GetRolesByGroup(projectID int64, groupDNCondition string) []int {
-	var roles []int
-	o := dao.GetOrmer()
-	sql := `select role from project_member pm 
-	left join user_group ug on pm.project_id = ?
-	where ug.group_type = 1 and ug.ldap_group_dn in (` + groupDNCondition + `)`
-	if _, err := o.Raw(sql, projectID).QueryRows(&roles); err != nil {
-		return roles
-	}
-	return roles
-}

src/common/dao/project/projectmember_test.go

@@ -305,30 +305,3 @@ func PrepareGroupTest() {
 	}
 	dao.PrepareTestData(clearSqls, initSqls)
 }
-func TestGetRolesByGroup(t *testing.T) {
-	PrepareGroupTest()
-
-	project, err := dao.GetProjectByName("group_project")
-	if err != nil {
-		t.Errorf("Error occurred when GetProjectByName : %v", err)
-	}
-	type args struct {
-		projectID        int64
-		groupDNCondition string
-	}
-	tests := []struct {
-		name string
-		args args
-		want []int
-	}{
-		{"Query group with role", args{project.ProjectID, "'cn=harbor_user,dc=example,dc=com'"}, []int{2}},
-		{"Query group no role", args{project.ProjectID, "'cn=another_user,dc=example,dc=com'"}, []int{}},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := GetRolesByGroup(tt.args.projectID, tt.args.groupDNCondition); !dao.ArrayEqual(got, tt.want) {
-				t.Errorf("GetRolesByGroup() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}

src/common/dao/project_test.go

@@ -118,36 +118,6 @@ func Test_projectQueryConditions(t *testing.T) {
 	}
 }
 
-func TestGetGroupProjects(t *testing.T) {
-	prepareGroupTest()
-	query := &models.ProjectQueryParam{Member: &models.MemberQuery{Name: "sample_group"}}
-	type args struct {
-		groupDNCondition string
-		query            *models.ProjectQueryParam
-	}
-	tests := []struct {
-		name     string
-		args     args
-		wantSize int
-		wantErr  bool
-	}{
-		{"Verify correct sql", args{groupDNCondition: "'cn=harbor_user,dc=example,dc=com'", query: query}, 1, false},
-		{"Verify missed sql", args{groupDNCondition: "'cn=another_user,dc=example,dc=com'", query: query}, 0, false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := GetGroupProjects(tt.args.groupDNCondition, tt.args.query)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("GetGroupProjects() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if len(got) != tt.wantSize {
-				t.Errorf("GetGroupProjects() = %v, want %v", got, tt.wantSize)
-			}
-		})
-	}
-}
-
 func prepareGroupTest() {
 	initSqls := []string{
 		`insert into user_group (group_name, group_type, ldap_group_dn) values ('harbor_group_01', 1, 'cn=harbor_user,dc=example,dc=com')`,
@@ -169,73 +139,6 @@ func prepareGroupTest() {
 	PrepareTestData(clearSqls, initSqls)
 }
 
-func TestGetTotalGroupProjects(t *testing.T) {
-	prepareGroupTest()
-	query := &models.ProjectQueryParam{Member: &models.MemberQuery{Name: "sample_group"}}
-	type args struct {
-		groupDNCondition string
-		query            *models.ProjectQueryParam
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    int
-		wantErr bool
-	}{
-		{"Verify correct sql", args{groupDNCondition: "'cn=harbor_user,dc=example,dc=com'", query: query}, 1, false},
-		{"Verify missed sql", args{groupDNCondition: "'cn=another_user,dc=example,dc=com'", query: query}, 0, false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := GetTotalGroupProjects(tt.args.groupDNCondition, tt.args.query)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("GetTotalGroupProjects() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if got != tt.want {
-				t.Errorf("GetTotalGroupProjects() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestGetRolesByLDAPGroup(t *testing.T) {
-	prepareGroupTest()
-	project, err := GetProjectByName("group_project")
-	if err != nil {
-		t.Errorf("Error occurred when Get project by name: %v", err)
-	}
-	privateProject, err := GetProjectByName("group_project_private")
-	if err != nil {
-		t.Errorf("Error occurred when Get project by name: %v", err)
-	}
-	type args struct {
-		projectID        int64
-		groupDNCondition string
-	}
-	tests := []struct {
-		name     string
-		args     args
-		wantSize int
-		wantErr  bool
-	}{
-		{"Check normal", args{project.ProjectID, "'cn=harbor_user,dc=example,dc=com'"}, 1, false},
-		{"Check non exist", args{privateProject.ProjectID, "'cn=not_harbor_user,dc=example,dc=com'"}, 0, false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := GetRolesByLDAPGroup(tt.args.projectID, tt.args.groupDNCondition)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("TestGetRolesByLDAPGroup() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if len(got) != tt.wantSize {
-				t.Errorf("TestGetRolesByLDAPGroup() = %v, want %v", len(got), tt.wantSize)
-			}
-		})
-	}
-}
-
 func TestProjetExistsByName(t *testing.T) {
 	name := "project_exist_by_name_test"
 	exist := ProjectExistsByName(name)

src/common/dao/utils.go

@@ -0,0 +1,11 @@
+package dao
+
+import (
+	"fmt"
+	"strings"
+)
+
+// JoinNumberConditions - To join number condition into string,used in sql query
+func JoinNumberConditions(ids []int) string {
+	return strings.Trim(strings.Replace(fmt.Sprint(ids), " ", ",", -1), "[]")
+}

src/common/dao/utils_test.go

@@ -0,0 +1,24 @@
+package dao
+
+import "testing"
+
+func TestJoinNumberConditions(t *testing.T) {
+	type args struct {
+		ids []int
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{name: "normal test", args: args{[]int{1, 2, 3}}, want: "1,2,3"},
+		{name: "dummy test", args: args{[]int{}}, want: ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := JoinNumberConditions(tt.args.ids); got != tt.want {
+				t.Errorf("JoinNumberConditions() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

src/common/models/project.go

@@ -130,7 +130,7 @@ type ProjectQueryParam struct {
 type MemberQuery struct {
 	Name     string // the username of member
 	Role     int    // the role of the member has to the project
-	GroupList []*UserGroup // the group list of current user
+	GroupIDs []int  // the group ID of current user belongs to
 }
 
 // Pagination ...

src/common/models/user.go

@@ -40,7 +40,7 @@ type User struct {
 	Salt         string    `orm:"column(salt)" json:"-"`
 	CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
 	UpdateTime   time.Time `orm:"column(update_time);auto_now" json:"update_time"`
-	GroupList    []*UserGroup `orm:"-" json:"-"`
+	GroupIDs     []int     `orm:"-" json:"-"`
 	OIDCUserMeta *OIDCUser `orm:"-" json:"oidc_user_meta,omitempty"`
 }
 

src/common/security/local/context.go

@@ -17,7 +17,6 @@ package local
 import (
 	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/dao/group"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/rbac"
 	"github.com/goharbor/harbor/src/common/rbac/project"
@@ -140,12 +139,11 @@ func (s *SecurityContext) GetRolesByGroup(projectIDOrName interface{}) []int {
 	user := s.user
 	project, err := s.pm.Get(projectIDOrName)
 	// No user, group or project info
-	if err != nil || project == nil || user == nil || len(user.GroupList) == 0 {
+	if err != nil || project == nil || user == nil || len(user.GroupIDs) == 0 {
 		return roles
 	}
-	// Get role by LDAP group
+	// Get role by Group ID
-	groupDNConditions := group.GetGroupDNQueryCondition(user.GroupList)
+	roles, err = dao.GetRolesByGroupID(project.ProjectID, user.GroupIDs)
-	roles, err = dao.GetRolesByLDAPGroup(project.ProjectID, groupDNConditions)
 	if err != nil {
 		return nil
 	}
@@ -158,7 +156,7 @@ func (s *SecurityContext) GetMyProjects() ([]*models.Project, error) {
 		&models.ProjectQueryParam{
 			Member: &models.MemberQuery{
 				Name:     s.GetUsername(),
-				GroupList: s.user.GroupList,
+				GroupIDs: s.user.GroupIDs,
 			},
 		})
 	if err != nil {

src/common/security/local/context_test.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/common/dao/group"
 	"github.com/goharbor/harbor/src/common/dao/project"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/rbac"
@@ -253,9 +254,16 @@ func TestHasPushPullPermWithGroup(t *testing.T) {
 	if err != nil {
 		t.Errorf("Error occurred when GetUser: %v", err)
 	}
-	developer.GroupList = []*models.UserGroup{
+
-		{GroupName: "test_group", GroupType: 1, LdapGroupDN: "cn=harbor_user,dc=example,dc=com"},
+	userGroups, err := group.QueryUserGroup(models.UserGroup{GroupType: common.LdapGroupType, LdapGroupDN: "cn=harbor_user,dc=example,dc=com"})
+	if err != nil {
+		t.Errorf("Failed to query user group %v", err)
 	}
+	if len(userGroups) < 1 {
+		t.Errorf("Failed to retrieve user group")
+	}
+
+	developer.GroupIDs = []int{userGroups[0].ID}
 
 	resource := rbac.NewProjectNamespace(project.Name).Resource(rbac.ResourceRepository)
 
@@ -332,9 +340,15 @@ func TestSecurityContext_GetRolesByGroup(t *testing.T) {
 	if err != nil {
 		t.Errorf("Error occurred when GetUser: %v", err)
 	}
-	developer.GroupList = []*models.UserGroup{
+	userGroups, err := group.QueryUserGroup(models.UserGroup{GroupType: common.LdapGroupType, LdapGroupDN: "cn=harbor_user,dc=example,dc=com"})
-		{GroupName: "test_group", GroupType: 1, LdapGroupDN: "cn=harbor_user,dc=example,dc=com"},
+	if err != nil {
+		t.Errorf("Failed to query user group %v", err)
 	}
+	if len(userGroups) < 1 {
+		t.Errorf("Failed to retrieve user group")
+	}
+
+	developer.GroupIDs = []int{userGroups[0].ID}
 	type fields struct {
 		user *models.User
 		pm   promgr.ProjectManager

src/core/auth/ldap/ldap.go

@@ -20,11 +20,11 @@ import (
 	"strings"
 
 	"github.com/goharbor/harbor/src/common"
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/common/dao/group"
 	"github.com/goharbor/harbor/src/common/utils"
 	goldap "gopkg.in/ldap.v2"
 
-	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/dao/group"
 	"github.com/goharbor/harbor/src/common/models"
 	ldapUtils "github.com/goharbor/harbor/src/common/utils/ldap"
 	"github.com/goharbor/harbor/src/common/utils/log"
@@ -79,7 +79,7 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) {
 	u.Username = ldapUsers[0].Username
 	u.Email = strings.TrimSpace(ldapUsers[0].Email)
 	u.Realname = ldapUsers[0].Realname
-	userGroups := make([]*models.UserGroup, 0)
+	ugIDs := []int{}
 
 	dn := ldapUsers[0].DN
 	if err = ldapSession.Bind(dn, m.Password); err != nil {
@@ -95,6 +95,7 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) {
 	for _, groupDN := range ldapUsers[0].GroupDNList {
 
 		groupDN = utils.TrimLower(groupDN)
+		// Attach LDAP group admin
 		if len(groupAdminDN) > 0 && groupAdminDN == groupDN {
 			u.HasAdminRole = true
 		}
@@ -103,16 +104,16 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) {
 			GroupType:   1,
 			LdapGroupDN: groupDN,
 		}
-		userGroupList, err := group.QueryUserGroup(userGroupQuery)
+		userGroups, err := group.QueryUserGroup(userGroupQuery)
 		if err != nil {
 			continue
 		}
-		if len(userGroupList) == 0 {
+		if len(userGroups) == 0 {
 			continue
 		}
-		userGroups = append(userGroups, userGroupList[0])
+		ugIDs = append(ugIDs, userGroups[0].ID)
 	}
-	u.GroupList = userGroups
+	u.GroupIDs = ugIDs
 
 	return &u, nil
 }

src/core/promgr/pmsdriver/local/local.go

@@ -20,7 +20,6 @@ import (
 	"time"
 
 	"github.com/goharbor/harbor/src/common/dao"
-	"github.com/goharbor/harbor/src/common/dao/group"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/common/utils"
 	errutil "github.com/goharbor/harbor/src/common/utils/error"
@@ -132,19 +131,16 @@ func (d *driver) Update(projectIDOrName interface{},
 func (d *driver) List(query *models.ProjectQueryParam) (*models.ProjectQueryResult, error) {
 	var total int64
 	var projects []*models.Project
-	var groupDNCondition string
+	var groupIDs []int
-
-	// List with LDAP group projects
 	if query != nil && query.Member != nil {
-		groupDNCondition = group.GetGroupDNQueryCondition(query.Member.GroupList)
+		groupIDs = query.Member.GroupIDs
 	}
-
+	count, err := dao.GetTotalGroupProjects(groupIDs, query)
-	count, err := dao.GetTotalGroupProjects(groupDNCondition, query)
 	if err != nil {
 		return nil, err
 	}
 	total = int64(count)
-	projects, err = dao.GetGroupProjects(groupDNCondition, query)
+	projects, err = dao.GetGroupProjects(groupIDs, query)
 	if err != nil {
 		return nil, err
 	}

src/replication/adapter/adapter.go

@@ -17,10 +17,18 @@ package adapter
 import (
 	"errors"
 	"fmt"
+	"io"
 
+	"github.com/docker/distribution"
+	"github.com/goharbor/harbor/src/replication/filter"
 	"github.com/goharbor/harbor/src/replication/model"
 )
 
+// const definition
+const (
+	UserAgentReplication = "harbor-replication-service"
+)
+
 var registry = map[model.RegistryType]Factory{}
 
 // Factory creates a specific Adapter according to the params
@@ -37,6 +45,81 @@ type Adapter interface {
 	HealthCheck() (model.HealthStatus, error)
 }
 
+// ImageRegistry defines the capabilities that an image registry should have
+type ImageRegistry interface {
+	FetchImages(filters []*model.Filter) ([]*model.Resource, error)
+	ManifestExist(repository, reference string) (exist bool, digest string, err error)
+	PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error)
+	PushManifest(repository, reference, mediaType string, payload []byte) error
+	// the "reference" can be "tag" or "digest", the function needs to handle both
+	DeleteManifest(repository, reference string) error
+	BlobExist(repository, digest string) (exist bool, err error)
+	PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error)
+	PushBlob(repository, digest string, size int64, blob io.Reader) error
+}
+
+// ChartRegistry defines the capabilities that a chart registry should have
+type ChartRegistry interface {
+	FetchCharts(filters []*model.Filter) ([]*model.Resource, error)
+	ChartExist(name, version string) (bool, error)
+	DownloadChart(name, version string) (io.ReadCloser, error)
+	UploadChart(name, version string, chart io.Reader) error
+	DeleteChart(name, version string) error
+}
+
+// Repository defines an repository object, it can be image repository, chart repository and etc.
+type Repository struct {
+	ResourceType string `json:"resource_type"`
+	Name         string `json:"name"`
+}
+
+// GetName returns the name
+func (r *Repository) GetName() string {
+	return r.Name
+}
+
+// GetFilterableType returns the filterable type
+func (r *Repository) GetFilterableType() filter.FilterableType {
+	return filter.FilterableTypeRepository
+}
+
+// GetResourceType returns the resource type
+func (r *Repository) GetResourceType() string {
+	return r.ResourceType
+}
+
+// GetLabels returns the labels
+func (r *Repository) GetLabels() []string {
+	return nil
+}
+
+// VTag defines an vTag object, it can be image tag, chart version and etc.
+type VTag struct {
+	ResourceType string   `json:"resource_type"`
+	Name         string   `json:"name"`
+	Labels       []string `json:"labels"`
+}
+
+// GetFilterableType returns the filterable type
+func (v *VTag) GetFilterableType() filter.FilterableType {
+	return filter.FilterableTypeVTag
+}
+
+// GetResourceType returns the resource type
+func (v *VTag) GetResourceType() string {
+	return v.ResourceType
+}
+
+// GetName returns the name
+func (v *VTag) GetName() string {
+	return v.Name
+}
+
+// GetLabels returns the labels
+func (v *VTag) GetLabels() []string {
+	return v.Labels
+}
+
 // RegisterFactory registers one adapter factory to the registry
 func RegisterFactory(t model.RegistryType, factory Factory) error {
 	if len(t) == 0 {

src/replication/adapter/awsecr/adapter.go

@@ -16,6 +16,9 @@ package awsecr
 
 import (
 	"errors"
+	"net/http"
+	"regexp"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -24,9 +27,8 @@ import (
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/common/utils/registry"
 	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/adapter/native"
 	"github.com/goharbor/harbor/src/replication/model"
-	"net/http"
-	"regexp"
 )
 
 func init() {
@@ -45,13 +47,13 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
 		return nil, err
 	}
 	authorizer := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure)
-	reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer)
+	dockerRegistry, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
 	if err != nil {
 		return nil, err
 	}
 	return &adapter{
 		registry: registry,
-		DefaultImageRegistry: reg,
+		Adapter:  dockerRegistry,
 		region:   region,
 	}, nil
 }
@@ -66,7 +68,7 @@ func parseRegion(url string) (string, error) {
 }
 
 type adapter struct {
-	*adp.DefaultImageRegistry
+	*native.Adapter
 	registry      *model.Registry
 	region        string
 	forceEndpoint *string

src/replication/adapter/awsecr/adapter_test.go

@@ -2,8 +2,6 @@ package awsecr
 
 import (
 	"fmt"
-	"github.com/goharbor/harbor/src/common/utils/test"
-	"github.com/stretchr/testify/assert"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -11,8 +9,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/goharbor/harbor/src/common/utils/test"
 	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/adapter/native"
 	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestAdapter_NewAdapter(t *testing.T) {
@@ -130,13 +131,13 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser
 			AccessSecret: "ppp",
 		}
 	}
-	reg, err := adp.NewDefaultImageRegistry(registry)
+	dockerRegistryAdapter, err := native.NewAdapter(registry)
 	if err != nil {
 		panic(err)
 	}
 	return &adapter{
 		registry:      registry,
-		DefaultImageRegistry: reg,
+		Adapter:       dockerRegistryAdapter,
 		region:        "test-region",
 		forceEndpoint: &server.URL,
 	}, server

src/replication/adapter/awsecr/image_registry.go

@@ -1,113 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package awsecr
-
-import (
-	adp "github.com/goharbor/harbor/src/replication/adapter"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/util"
-)
-
-var _ adp.ImageRegistry = adapter{}
-
-func (a adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
-	nameFilterPattern := ""
-	tagFilterPattern := ""
-	for _, filter := range filters {
-		switch filter.Type {
-		case model.FilterTypeName:
-			nameFilterPattern = filter.Value.(string)
-		case model.FilterTypeTag:
-			tagFilterPattern = filter.Value.(string)
-		}
-	}
-	repositories, err := a.filterRepositories(nameFilterPattern)
-	if err != nil {
-		return nil, err
-	}
-
-	var resources []*model.Resource
-	for _, repository := range repositories {
-		tags, err := a.filterTags(repository, tagFilterPattern)
-		if err != nil {
-			return nil, err
-		}
-		if len(tags) == 0 {
-			continue
-		}
-		resources = append(resources, &model.Resource{
-			Type:     model.ResourceTypeImage,
-			Registry: a.registry,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: repository,
-				},
-				Vtags: tags,
-			},
-		})
-	}
-
-	return resources, nil
-}
-
-func (a adapter) filterRepositories(pattern string) ([]string, error) {
-	// if the pattern is a specific repository name, just returns the parsed repositories
-	// and will check the existence later when filtering the tags
-	if repositories, ok := util.IsSpecificPath(pattern); ok {
-		return repositories, nil
-	}
-	// search repositories from catalog api
-	repositories, err := a.Catalog()
-	if err != nil {
-		return nil, err
-	}
-	// if the pattern is null, just return the result of catalog API
-	if len(pattern) == 0 {
-		return repositories, nil
-	}
-	result := []string{}
-	for _, repository := range repositories {
-		match, err := util.Match(pattern, repository)
-		if err != nil {
-			return nil, err
-		}
-		if match {
-			result = append(result, repository)
-		}
-	}
-	return result, nil
-}
-
-func (a adapter) filterTags(repository, pattern string) ([]string, error) {
-	tags, err := a.ListTag(repository)
-	if err != nil {
-		return nil, err
-	}
-	if len(pattern) == 0 {
-		return tags, nil
-	}
-
-	var result []string
-	for _, tag := range tags {
-		match, err := util.Match(pattern, tag)
-		if err != nil {
-			return nil, err
-		}
-		if match {
-			result = append(result, tag)
-		}
-	}
-	return result, nil
-}

src/replication/adapter/azurecr/adapter.go

@@ -24,24 +24,26 @@ func init() {
 }
 
 func factory(registry *model.Registry) (adp.Adapter, error) {
-	client, err := getClient(registry)
+	if registry.Credential == nil || len(registry.Credential.AccessKey) == 0 ||
-	if err != nil {
+		len(registry.Credential.AccessSecret) == 0 {
-		return nil, err
+		return nil, fmt.Errorf("credential is necessary for registry %s", registry.URL)
 	}
 
-	reg, err := native.NewWithClient(registry, client)
+	authorizer := auth.NewBasicAuthCredential(registry.Credential.AccessKey,
+		registry.Credential.AccessSecret)
+	dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
 	if err != nil {
 		return nil, err
 	}
 
 	return &adapter{
 		registry: registry,
-		Native:   reg,
+		Adapter:  dockerRegistryAdapter,
 	}, nil
 }
 
 type adapter struct {
-	*native.Native
+	*native.Adapter
 	registry *model.Registry
 }
 
@@ -72,11 +74,6 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
 	}, nil
 }
 
-// PrepareForPush no preparation needed for Azure container registry
-func (a *adapter) PrepareForPush(resources []*model.Resource) error {
-	return nil
-}
-
 // HealthCheck checks health status of a registry
 func (a *adapter) HealthCheck() (model.HealthStatus, error) {
 	err := a.PingGet()

src/replication/adapter/chart_registry.go

@@ -1,30 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package adapter
-
-import (
-	"io"
-
-	"github.com/goharbor/harbor/src/replication/model"
-)
-
-// ChartRegistry defines the capabilities that a chart registry should have
-type ChartRegistry interface {
-	FetchCharts(filters []*model.Filter) ([]*model.Resource, error)
-	ChartExist(name, version string) (bool, error)
-	DownloadChart(name, version string) (io.ReadCloser, error)
-	UploadChart(name, version string, chart io.Reader) error
-	DeleteChart(name, version string) error
-}

src/replication/adapter/dockerhub/adapter.go

@@ -12,6 +12,7 @@ import (
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/common/utils/registry/auth"
 	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/adapter/native"
 	"github.com/goharbor/harbor/src/replication/model"
 	"github.com/goharbor/harbor/src/replication/util"
 )
@@ -47,7 +48,7 @@ func factory(registry *model.Registry) (adp.Adapter, error) {
 		Transport: util.GetHTTPTransport(registry.Insecure),
 	}, credential)
 
-	reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(&model.Registry{
+	dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(&model.Registry{
 		Name:       registry.Name,
 		URL:        registryURL, // specify the URL of Docker Hub registry service
 		Credential: registry.Credential,
@@ -60,12 +61,12 @@ func factory(registry *model.Registry) (adp.Adapter, error) {
 	return &adapter{
 		client:   client,
 		registry: registry,
-		DefaultImageRegistry: reg,
+		Adapter:  dockerRegistryAdapter,
 	}, nil
 }
 
 type adapter struct {
-	*adp.DefaultImageRegistry
+	*native.Adapter
 	registry *model.Registry
 	client   *Client
 }

src/replication/adapter/googlegcr/adapter.go

@@ -15,12 +15,14 @@
 package googlegcr
 
 import (
+	"net/http"
+
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/common/utils/registry/auth"
 	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/adapter/native"
 	"github.com/goharbor/harbor/src/replication/model"
 	"github.com/goharbor/harbor/src/replication/util"
-	"net/http"
 )
 
 func init() {
@@ -44,19 +46,19 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
 		Transport: util.GetHTTPTransport(registry.Insecure),
 	}, credential)
 
-	reg, err := adp.NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer)
+	dockerRegistryAdapter, err := native.NewAdapterWithCustomizedAuthorizer(registry, authorizer)
 	if err != nil {
 		return nil, err
 	}
 
 	return &adapter{
 		registry: registry,
-		DefaultImageRegistry: reg,
+		Adapter:  dockerRegistryAdapter,
 	}, nil
 }
 
 type adapter struct {
-	*adp.DefaultImageRegistry
+	*native.Adapter
 	registry *model.Registry
 }
 

src/replication/adapter/googlegcr/image_registry.go

@@ -1,113 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package googlegcr
-
-import (
-	adp "github.com/goharbor/harbor/src/replication/adapter"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/util"
-)
-
-var _ adp.ImageRegistry = adapter{}
-
-func (a adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
-	nameFilterPattern := ""
-	tagFilterPattern := ""
-	for _, filter := range filters {
-		switch filter.Type {
-		case model.FilterTypeName:
-			nameFilterPattern = filter.Value.(string)
-		case model.FilterTypeTag:
-			tagFilterPattern = filter.Value.(string)
-		}
-	}
-	repositories, err := a.filterRepositories(nameFilterPattern)
-	if err != nil {
-		return nil, err
-	}
-
-	var resources []*model.Resource
-	for _, repository := range repositories {
-		tags, err := a.filterTags(repository, tagFilterPattern)
-		if err != nil {
-			return nil, err
-		}
-		if len(tags) == 0 {
-			continue
-		}
-		resources = append(resources, &model.Resource{
-			Type:     model.ResourceTypeImage,
-			Registry: a.registry,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: repository,
-				},
-				Vtags: tags,
-			},
-		})
-	}
-
-	return resources, nil
-}
-
-func (a adapter) filterRepositories(pattern string) ([]string, error) {
-	// if the pattern is a specific repository name, just returns the parsed repositories
-	// and will check the existence later when filtering the tags
-	if repositories, ok := util.IsSpecificPath(pattern); ok {
-		return repositories, nil
-	}
-	// search repositories from catalog api
-	repositories, err := a.Catalog()
-	if err != nil {
-		return nil, err
-	}
-	// if the pattern is null, just return the result of catalog API
-	if len(pattern) == 0 {
-		return repositories, nil
-	}
-	result := []string{}
-	for _, repository := range repositories {
-		match, err := util.Match(pattern, repository)
-		if err != nil {
-			return nil, err
-		}
-		if match {
-			result = append(result, repository)
-		}
-	}
-	return result, nil
-}
-
-func (a adapter) filterTags(repository, pattern string) ([]string, error) {
-	tags, err := a.ListTag(repository)
-	if err != nil {
-		return nil, err
-	}
-	if len(pattern) == 0 {
-		return tags, nil
-	}
-
-	var result []string
-	for _, tag := range tags {
-		match, err := util.Match(pattern, tag)
-		if err != nil {
-			return nil, err
-		}
-		if match {
-			result = append(result, tag)
-		}
-	}
-	return result, nil
-}

src/replication/adapter/harbor/adapter.go

@@ -27,6 +27,7 @@ import (
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/common/utils/registry/auth"
 	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/adapter/native"
 	"github.com/goharbor/harbor/src/replication/model"
 	"github.com/goharbor/harbor/src/replication/util"
 )
@@ -42,7 +43,7 @@ func init() {
 }
 
 type adapter struct {
-	*adp.DefaultImageRegistry
+	*native.Adapter
 	registry *model.Registry
 	url      string
 	client   *common_http.Client
@@ -67,7 +68,7 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
 		modifiers = append(modifiers, authorizer)
 	}
 
-	reg, err := adp.NewDefaultImageRegistry(registry)
+	dockerRegistryAdapter, err := native.NewAdapter(registry)
 	if err != nil {
 		return nil, err
 	}
@@ -78,7 +79,7 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
 			&http.Client{
 				Transport: transport,
 			}, modifiers...),
-		DefaultImageRegistry: reg,
+		Adapter: dockerRegistryAdapter,
 	}, nil
 }
 

src/replication/adapter/huawei/huawei_adapter.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/goharbor/harbor/src/common/utils/log"
 	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/replication/adapter/native"
 	"github.com/goharbor/harbor/src/replication/model"
 	"github.com/goharbor/harbor/src/replication/util"
 )
@@ -27,7 +28,7 @@ func init() {
 
 // Adapter is for images replications between harbor and Huawei image repository(SWR)
 type adapter struct {
-	*adp.DefaultImageRegistry
+	*native.Adapter
 	registry *model.Registry
 }
 
@@ -232,13 +233,13 @@ func (a *adapter) HealthCheck() (model.HealthStatus, error) {
 
 // AdapterFactory is the factory for huawei adapter
 func AdapterFactory(registry *model.Registry) (adp.Adapter, error) {
-	reg, err := adp.NewDefaultImageRegistry(registry)
+	dockerRegistryAdapter, err := native.NewAdapter(registry)
 	if err != nil {
 		return nil, err
 	}
 	return &adapter{
 		registry: registry,
-		DefaultImageRegistry: reg,
+		Adapter:  dockerRegistryAdapter,
 	}, nil
 
 }

src/replication/adapter/image_registry.go

@@ -1,325 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package adapter
-
-import (
-	"errors"
-	"io"
-	"net/http"
-	"strings"
-	"sync"
-
-	"github.com/goharbor/harbor/src/replication/filter"
-
-	"github.com/docker/distribution"
-	"github.com/docker/distribution/manifest/schema1"
-	"github.com/goharbor/harbor/src/common/http/modifier"
-	common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
-	"github.com/goharbor/harbor/src/common/utils/log"
-	registry_pkg "github.com/goharbor/harbor/src/common/utils/registry"
-	"github.com/goharbor/harbor/src/common/utils/registry/auth"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/util"
-)
-
-// const definition
-const (
-	UserAgentReplication = "harbor-replication-service"
-)
-
-// ImageRegistry defines the capabilities that an image registry should have
-type ImageRegistry interface {
-	FetchImages(filters []*model.Filter) ([]*model.Resource, error)
-	ManifestExist(repository, reference string) (exist bool, digest string, err error)
-	PullManifest(repository, reference string, accepttedMediaTypes []string) (manifest distribution.Manifest, digest string, err error)
-	PushManifest(repository, reference, mediaType string, payload []byte) error
-	// the "reference" can be "tag" or "digest", the function needs to handle both
-	DeleteManifest(repository, reference string) error
-	BlobExist(repository, digest string) (exist bool, err error)
-	PullBlob(repository, digest string) (size int64, blob io.ReadCloser, err error)
-	PushBlob(repository, digest string, size int64, blob io.Reader) error
-}
-
-// Repository defines an repository object, it can be image repository, chart repository and etc.
-type Repository struct {
-	ResourceType string `json:"resource_type"`
-	Name         string `json:"name"`
-}
-
-// GetName returns the name
-func (r *Repository) GetName() string {
-	return r.Name
-}
-
-// GetFilterableType returns the filterable type
-func (r *Repository) GetFilterableType() filter.FilterableType {
-	return filter.FilterableTypeRepository
-}
-
-// GetResourceType returns the resource type
-func (r *Repository) GetResourceType() string {
-	return r.ResourceType
-}
-
-// GetLabels returns the labels
-func (r *Repository) GetLabels() []string {
-	return nil
-}
-
-// VTag defines an vTag object, it can be image tag, chart version and etc.
-type VTag struct {
-	ResourceType string   `json:"resource_type"`
-	Name         string   `json:"name"`
-	Labels       []string `json:"labels"`
-}
-
-// GetFilterableType returns the filterable type
-func (v *VTag) GetFilterableType() filter.FilterableType {
-	return filter.FilterableTypeVTag
-}
-
-// GetResourceType returns the resource type
-func (v *VTag) GetResourceType() string {
-	return v.ResourceType
-}
-
-// GetName returns the name
-func (v *VTag) GetName() string {
-	return v.Name
-}
-
-// GetLabels returns the labels
-func (v *VTag) GetLabels() []string {
-	return v.Labels
-}
-
-// DefaultImageRegistry provides a default implementation for interface ImageRegistry
-type DefaultImageRegistry struct {
-	sync.RWMutex
-	*registry_pkg.Registry
-	registry *model.Registry
-	client   *http.Client
-	clients  map[string]*registry_pkg.Repository
-}
-
-// NewDefaultRegistryWithClient returns an instance of DefaultImageRegistry
-func NewDefaultRegistryWithClient(registry *model.Registry, client *http.Client) (*DefaultImageRegistry, error) {
-	reg, err := registry_pkg.NewRegistry(registry.URL, client)
-	if err != nil {
-		return nil, err
-	}
-
-	return &DefaultImageRegistry{
-		Registry: reg,
-		client:   client,
-		registry: registry,
-		clients:  map[string]*registry_pkg.Repository{},
-	}, nil
-}
-
-// NewDefaultImageRegistry returns an instance of DefaultImageRegistry
-func NewDefaultImageRegistry(registry *model.Registry) (*DefaultImageRegistry, error) {
-	var authorizer modifier.Modifier
-	if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
-		var cred modifier.Modifier
-		if registry.Credential.Type == model.CredentialTypeSecret {
-			cred = common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret)
-		} else {
-			cred = auth.NewBasicAuthCredential(
-				registry.Credential.AccessKey,
-				registry.Credential.AccessSecret)
-		}
-		authorizer = auth.NewStandardTokenAuthorizer(&http.Client{
-			Transport: util.GetHTTPTransport(registry.Insecure),
-		}, cred, registry.TokenServiceURL)
-	}
-	return NewDefaultImageRegistryWithCustomizedAuthorizer(registry, authorizer)
-}
-
-// NewDefaultImageRegistryWithCustomizedAuthorizer returns an instance of DefaultImageRegistry with the customized authorizer
-func NewDefaultImageRegistryWithCustomizedAuthorizer(registry *model.Registry, authorizer modifier.Modifier) (*DefaultImageRegistry, error) {
-	transport := util.GetHTTPTransport(registry.Insecure)
-	modifiers := []modifier.Modifier{
-		&auth.UserAgentModifier{
-			UserAgent: UserAgentReplication,
-		},
-	}
-	if authorizer != nil {
-		modifiers = append(modifiers, authorizer)
-	}
-	client := &http.Client{
-		Transport: registry_pkg.NewTransport(transport, modifiers...),
-	}
-	reg, err := registry_pkg.NewRegistry(registry.URL, client)
-	if err != nil {
-		return nil, err
-	}
-	return &DefaultImageRegistry{
-		Registry: reg,
-		client:   client,
-		registry: registry,
-		clients:  map[string]*registry_pkg.Repository{},
-	}, nil
-}
-
-func (d *DefaultImageRegistry) getClient(repository string) (*registry_pkg.Repository, error) {
-	d.RLock()
-	client, exist := d.clients[repository]
-	d.RUnlock()
-	if exist {
-		return client, nil
-	}
-
-	return d.create(repository)
-}
-
-func (d *DefaultImageRegistry) create(repository string) (*registry_pkg.Repository, error) {
-	d.Lock()
-	defer d.Unlock()
-	// double check
-	client, exist := d.clients[repository]
-	if exist {
-		return client, nil
-	}
-
-	client, err := registry_pkg.NewRepository(repository, d.registry.URL, d.client)
-	if err != nil {
-		return nil, err
-	}
-	d.clients[repository] = client
-	return client, nil
-}
-
-// HealthCheck checks health status of a registry
-func (d *DefaultImageRegistry) HealthCheck() (model.HealthStatus, error) {
-	var err error
-	if d.registry.Credential == nil ||
-		(len(d.registry.Credential.AccessKey) == 0 && len(d.registry.Credential.AccessSecret) == 0) {
-		err = d.PingSimple()
-	} else {
-		err = d.Ping()
-	}
-	if err != nil {
-		log.Errorf("failed to ping registry %s: %v", d.registry.URL, err)
-		return model.Unhealthy, nil
-	}
-	return model.Healthy, nil
-}
-
-// FetchImages ...
-func (d *DefaultImageRegistry) FetchImages(namespaces []string, filters []*model.Filter) ([]*model.Resource, error) {
-	return nil, errors.New("not implemented")
-}
-
-// ManifestExist ...
-func (d *DefaultImageRegistry) ManifestExist(repository, reference string) (bool, string, error) {
-	client, err := d.getClient(repository)
-	if err != nil {
-		return false, "", err
-	}
-	digest, exist, err := client.ManifestExist(reference)
-	return exist, digest, err
-}
-
-// PullManifest ...
-func (d *DefaultImageRegistry) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) {
-	client, err := d.getClient(repository)
-	if err != nil {
-		return nil, "", err
-	}
-	digest, mediaType, payload, err := client.PullManifest(reference, accepttedMediaTypes)
-	if err != nil {
-		return nil, "", err
-	}
-	if strings.Contains(mediaType, "application/json") {
-		mediaType = schema1.MediaTypeManifest
-	}
-	manifest, _, err := registry_pkg.UnMarshal(mediaType, payload)
-	if err != nil {
-		return nil, "", err
-	}
-	return manifest, digest, nil
-}
-
-// PushManifest ...
-func (d *DefaultImageRegistry) PushManifest(repository, reference, mediaType string, payload []byte) error {
-	client, err := d.getClient(repository)
-	if err != nil {
-		return err
-	}
-	_, err = client.PushManifest(reference, mediaType, payload)
-	return err
-}
-
-// DeleteManifest ...
-func (d *DefaultImageRegistry) DeleteManifest(repository, reference string) error {
-	client, err := d.getClient(repository)
-	if err != nil {
-		return err
-	}
-	digest := reference
-	if !isDigest(digest) {
-		dgt, exist, err := client.ManifestExist(reference)
-		if err != nil {
-			return err
-		}
-		if !exist {
-			log.Debugf("the manifest of %s:%s doesn't exist", repository, reference)
-			return nil
-		}
-		digest = dgt
-	}
-	return client.DeleteManifest(digest)
-}
-
-// BlobExist ...
-func (d *DefaultImageRegistry) BlobExist(repository, digest string) (bool, error) {
-	client, err := d.getClient(repository)
-	if err != nil {
-		return false, err
-	}
-	return client.BlobExist(digest)
-}
-
-// PullBlob ...
-func (d *DefaultImageRegistry) PullBlob(repository, digest string) (int64, io.ReadCloser, error) {
-	client, err := d.getClient(repository)
-	if err != nil {
-		return 0, nil, err
-	}
-	return client.PullBlob(digest)
-}
-
-// PushBlob ...
-func (d *DefaultImageRegistry) PushBlob(repository, digest string, size int64, blob io.Reader) error {
-	client, err := d.getClient(repository)
-	if err != nil {
-		return err
-	}
-	return client.PushBlob(digest, size, blob)
-}
-
-func isDigest(str string) bool {
-	return strings.Contains(str, ":")
-}
-
-// ListTag ...
-func (d *DefaultImageRegistry) ListTag(repository string) ([]string, error) {
-	client, err := d.getClient(repository)
-	if err != nil {
-		return []string{}, err
-	}
-	return client.ListTag()
-}

src/replication/adapter/image_registry_test.go

@@ -1,46 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package adapter
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-// TODO add UT
-
-func TestIsDigest(t *testing.T) {
-	cases := []struct {
-		str      string
-		isDigest bool
-	}{
-		{
-			str:      "",
-			isDigest: false,
-		},
-		{
-			str:      "latest",
-			isDigest: false,
-		},
-		{
-			str:      "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf",
-			isDigest: true,
-		},
-	}
-	for _, c := range cases {
-		assert.Equal(t, c.isDigest, isDigest(c.str))
-	}
-}

src/replication/adapter/native/adapter.go

@@ -15,16 +15,26 @@
 package native
 
 import (
+	"io"
 	"net/http"
+	"strings"
+	"sync"
 
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/goharbor/harbor/src/common/http/modifier"
+	common_http_auth "github.com/goharbor/harbor/src/common/http/modifier/auth"
 	"github.com/goharbor/harbor/src/common/utils/log"
+	registry_pkg "github.com/goharbor/harbor/src/common/utils/registry"
+	"github.com/goharbor/harbor/src/common/utils/registry/auth"
 	adp "github.com/goharbor/harbor/src/replication/adapter"
 	"github.com/goharbor/harbor/src/replication/model"
+	"github.com/goharbor/harbor/src/replication/util"
 )
 
 func init() {
 	if err := adp.RegisterFactory(model.RegistryTypeDockerRegistry, func(registry *model.Registry) (adp.Adapter, error) {
-		return newAdapter(registry)
+		return NewAdapter(registry)
 	}); err != nil {
 		log.Errorf("failed to register factory for %s: %v", model.RegistryTypeDockerRegistry, err)
 		return
@@ -32,39 +42,65 @@ func init() {
 	log.Infof("the factory for adapter %s registered", model.RegistryTypeDockerRegistry)
 }
 
-func newAdapter(registry *model.Registry) (*Native, error) {
+var _ adp.Adapter = &Adapter{}
-	reg, err := adp.NewDefaultImageRegistry(registry)
-	if err != nil {
-		return nil, err
-	}
-	return &Native{
-		registry:             registry,
-		DefaultImageRegistry: reg,
-	}, nil
-}
 
-// NewWithClient ...
+// Adapter implements an adapter for Docker registry. It can be used to all registries
-func NewWithClient(registry *model.Registry, client *http.Client) (*Native, error) {
+// that implement the registry V2 API
-	reg, err := adp.NewDefaultRegistryWithClient(registry, client)
+type Adapter struct {
-	if err != nil {
+	sync.RWMutex
-		return nil, err
+	*registry_pkg.Registry
-	}
-	return &Native{
-		registry:             registry,
-		DefaultImageRegistry: reg,
-	}, nil
-}
-
-// Native is adapter to native docker registry
-type Native struct {
-	*adp.DefaultImageRegistry
 	registry *model.Registry
+	client   *http.Client
+	clients  map[string]*registry_pkg.Repository // client for repositories
 }
 
-var _ adp.Adapter = Native{}
+// NewAdapter returns an instance of the Adapter
+func NewAdapter(registry *model.Registry) (*Adapter, error) {
+	var authorizer modifier.Modifier
+	if registry.Credential != nil && len(registry.Credential.AccessSecret) != 0 {
+		var cred modifier.Modifier
+		if registry.Credential.Type == model.CredentialTypeSecret {
+			cred = common_http_auth.NewSecretAuthorizer(registry.Credential.AccessSecret)
+		} else {
+			cred = auth.NewBasicAuthCredential(
+				registry.Credential.AccessKey,
+				registry.Credential.AccessSecret)
+		}
+		authorizer = auth.NewStandardTokenAuthorizer(&http.Client{
+			Transport: util.GetHTTPTransport(registry.Insecure),
+		}, cred, registry.TokenServiceURL)
+	}
+	return NewAdapterWithCustomizedAuthorizer(registry, authorizer)
+}
 
-// Info ...
+// NewAdapterWithCustomizedAuthorizer returns an instance of the Adapter with the customized authorizer
-func (Native) Info() (info *model.RegistryInfo, err error) {
+func NewAdapterWithCustomizedAuthorizer(registry *model.Registry, authorizer modifier.Modifier) (*Adapter, error) {
+	transport := util.GetHTTPTransport(registry.Insecure)
+	modifiers := []modifier.Modifier{
+		&auth.UserAgentModifier{
+			UserAgent: adp.UserAgentReplication,
+		},
+	}
+	if authorizer != nil {
+		modifiers = append(modifiers, authorizer)
+	}
+	client := &http.Client{
+		Transport: registry_pkg.NewTransport(transport, modifiers...),
+	}
+	reg, err := registry_pkg.NewRegistry(registry.URL, client)
+	if err != nil {
+		return nil, err
+	}
+	return &Adapter{
+		Registry: reg,
+		registry: registry,
+		client:   client,
+		clients:  map[string]*registry_pkg.Repository{},
+	}, nil
+}
+
+// Info returns the basic information about the adapter
+func (a *Adapter) Info() (info *model.RegistryInfo, err error) {
 	return &model.RegistryInfo{
 		Type: model.RegistryTypeDockerRegistry,
 		SupportedResourceTypes: []model.ResourceType{
@@ -87,5 +123,250 @@ func (Native) Info() (info *model.RegistryInfo, err error) {
 	}, nil
 }
 
-// PrepareForPush nothing need to do.
+// PrepareForPush does nothing
-func (Native) PrepareForPush([]*model.Resource) error { return nil }
+func (a *Adapter) PrepareForPush([]*model.Resource) error {
+	return nil
+}
+
+// HealthCheck checks health status of a registry
+func (a *Adapter) HealthCheck() (model.HealthStatus, error) {
+	var err error
+	if a.registry.Credential == nil ||
+		(len(a.registry.Credential.AccessKey) == 0 && len(a.registry.Credential.AccessSecret) == 0) {
+		err = a.PingSimple()
+	} else {
+		err = a.Ping()
+	}
+	if err != nil {
+		log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
+		return model.Unhealthy, nil
+	}
+	return model.Healthy, nil
+}
+
+// FetchImages ...
+func (a *Adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
+	repositories, err := a.getRepositories(filters)
+	if err != nil {
+		return nil, err
+	}
+	if len(repositories) == 0 {
+		return nil, nil
+	}
+	for _, filter := range filters {
+		if err = filter.DoFilter(&repositories); err != nil {
+			return nil, err
+		}
+	}
+
+	var resources []*model.Resource
+	for _, repository := range repositories {
+		vTags, err := a.getVTags(repository.Name)
+		if err != nil {
+			return nil, err
+		}
+		if len(vTags) == 0 {
+			continue
+		}
+		for _, filter := range filters {
+			if err = filter.DoFilter(&vTags); err != nil {
+				return nil, err
+			}
+		}
+		if len(vTags) == 0 {
+			continue
+		}
+		tags := []string{}
+		for _, vTag := range vTags {
+			tags = append(tags, vTag.Name)
+		}
+		resources = append(resources, &model.Resource{
+			Type:     model.ResourceTypeImage,
+			Registry: a.registry,
+			Metadata: &model.ResourceMetadata{
+				Repository: &model.Repository{
+					Name: repository.Name,
+				},
+				Vtags: tags,
+			},
+		})
+	}
+
+	return resources, nil
+}
+
+func (a *Adapter) getRepositories(filters []*model.Filter) ([]*adp.Repository, error) {
+	pattern := ""
+	for _, filter := range filters {
+		if filter.Type == model.FilterTypeName {
+			pattern = filter.Value.(string)
+			break
+		}
+	}
+	var repositories []string
+	var err error
+	// if the pattern of repository name filter is a specific repository name, just returns
+	// the parsed repositories and will check the existence later when filtering the tags
+	if paths, ok := util.IsSpecificPath(pattern); ok {
+		repositories = paths
+	} else {
+		// search repositories from catalog API
+		repositories, err = a.Catalog()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	result := []*adp.Repository{}
+	for _, repository := range repositories {
+		result = append(result, &adp.Repository{
+			ResourceType: string(model.ResourceTypeImage),
+			Name:         repository,
+		})
+	}
+	return result, nil
+}
+
+func (a *Adapter) getVTags(repository string) ([]*adp.VTag, error) {
+	tags, err := a.ListTag(repository)
+	if err != nil {
+		return nil, err
+	}
+	var result []*adp.VTag
+	for _, tag := range tags {
+		result = append(result, &adp.VTag{
+			ResourceType: string(model.ResourceTypeImage),
+			Name:         tag,
+		})
+	}
+	return result, nil
+}
+
+// ManifestExist ...
+func (a *Adapter) ManifestExist(repository, reference string) (bool, string, error) {
+	client, err := a.getClient(repository)
+	if err != nil {
+		return false, "", err
+	}
+	digest, exist, err := client.ManifestExist(reference)
+	return exist, digest, err
+}
+
+// PullManifest ...
+func (a *Adapter) PullManifest(repository, reference string, accepttedMediaTypes []string) (distribution.Manifest, string, error) {
+	client, err := a.getClient(repository)
+	if err != nil {
+		return nil, "", err
+	}
+	digest, mediaType, payload, err := client.PullManifest(reference, accepttedMediaTypes)
+	if err != nil {
+		return nil, "", err
+	}
+	if strings.Contains(mediaType, "application/json") {
+		mediaType = schema1.MediaTypeManifest
+	}
+	manifest, _, err := registry_pkg.UnMarshal(mediaType, payload)
+	if err != nil {
+		return nil, "", err
+	}
+	return manifest, digest, nil
+}
+
+// PushManifest ...
+func (a *Adapter) PushManifest(repository, reference, mediaType string, payload []byte) error {
+	client, err := a.getClient(repository)
+	if err != nil {
+		return err
+	}
+	_, err = client.PushManifest(reference, mediaType, payload)
+	return err
+}
+
+// DeleteManifest ...
+func (a *Adapter) DeleteManifest(repository, reference string) error {
+	client, err := a.getClient(repository)
+	if err != nil {
+		return err
+	}
+	digest := reference
+	if !isDigest(digest) {
+		dgt, exist, err := client.ManifestExist(reference)
+		if err != nil {
+			return err
+		}
+		if !exist {
+			log.Debugf("the manifest of %s:%s doesn't exist", repository, reference)
+			return nil
+		}
+		digest = dgt
+	}
+	return client.DeleteManifest(digest)
+}
+
+// BlobExist ...
+func (a *Adapter) BlobExist(repository, digest string) (bool, error) {
+	client, err := a.getClient(repository)
+	if err != nil {
+		return false, err
+	}
+	return client.BlobExist(digest)
+}
+
+// PullBlob ...
+func (a *Adapter) PullBlob(repository, digest string) (int64, io.ReadCloser, error) {
+	client, err := a.getClient(repository)
+	if err != nil {
+		return 0, nil, err
+	}
+	return client.PullBlob(digest)
+}
+
+// PushBlob ...
+func (a *Adapter) PushBlob(repository, digest string, size int64, blob io.Reader) error {
+	client, err := a.getClient(repository)
+	if err != nil {
+		return err
+	}
+	return client.PushBlob(digest, size, blob)
+}
+
+func isDigest(str string) bool {
+	return strings.Contains(str, ":")
+}
+
+// ListTag ...
+func (a *Adapter) ListTag(repository string) ([]string, error) {
+	client, err := a.getClient(repository)
+	if err != nil {
+		return []string{}, err
+	}
+	return client.ListTag()
+}
+
+func (a *Adapter) getClient(repository string) (*registry_pkg.Repository, error) {
+	a.RLock()
+	client, exist := a.clients[repository]
+	a.RUnlock()
+	if exist {
+		return client, nil
+	}
+
+	return a.create(repository)
+}
+
+func (a *Adapter) create(repository string) (*registry_pkg.Repository, error) {
+	a.Lock()
+	defer a.Unlock()
+	// double check
+	client, exist := a.clients[repository]
+	if exist {
+		return client, nil
+	}
+
+	client, err := registry_pkg.NewRepository(repository, a.registry.URL, a.client)
+	if err != nil {
+		return nil, err
+	}
+	a.clients[repository] = client
+	return client, nil
+}

src/replication/adapter/native/adapter_test.go

@@ -15,11 +15,15 @@
 package native
 
 import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
-	adp "github.com/goharbor/harbor/src/replication/adapter"
+	"github.com/goharbor/harbor/src/common/utils/test"
 	"github.com/goharbor/harbor/src/replication/model"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func Test_newAdapter(t *testing.T) {
@@ -33,7 +37,7 @@ func Test_newAdapter(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := newAdapter(tt.registry)
+			got, err := NewAdapter(tt.registry)
 			if tt.wantErr {
 				assert.NotNil(t, err)
 				assert.Nil(t, got)
@@ -47,14 +51,11 @@ func Test_newAdapter(t *testing.T) {
 
 func Test_native_Info(t *testing.T) {
 	var registry = &model.Registry{URL: "abc"}
-	var reg, _ = adp.NewDefaultImageRegistry(registry)
+	adapter, err := NewAdapter(registry)
-	var adapter = Native{
+	require.Nil(t, err)
-		DefaultImageRegistry: reg,
-		registry:             registry,
-	}
 	assert.NotNil(t, adapter)
 
-	var info, err = adapter.Info()
+	info, err := adapter.Info()
 	assert.Nil(t, err)
 	assert.NotNil(t, info)
 	assert.Equal(t, model.RegistryTypeDockerRegistry, info.Type)
@@ -66,13 +67,279 @@ func Test_native_Info(t *testing.T) {
 
 func Test_native_PrepareForPush(t *testing.T) {
 	var registry = &model.Registry{URL: "abc"}
-	var reg, _ = adp.NewDefaultImageRegistry(registry)
+	adapter, err := NewAdapter(registry)
-	var adapter = Native{
+	require.Nil(t, err)
-		DefaultImageRegistry: reg,
-		registry:             registry,
-	}
 	assert.NotNil(t, adapter)
 
-	var err = adapter.PrepareForPush(nil)
+	err = adapter.PrepareForPush(nil)
 	assert.Nil(t, err)
 }
+
+func mockNativeRegistry() (mock *httptest.Server) {
+	return test.NewServer(
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/_catalog",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte(`{"repositories":["test/a1","test/b2","test/c3/3level"]}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/test/a1/tags/list",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte(`{"name":"test/a1","tags":["tag11"]}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/test/b2/tags/list",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte(`{"name":"test/b2","tags":["tag11","tag2","tag13"]}`))
+			},
+		},
+		&test.RequestHandlerMapping{
+			Method:  http.MethodGet,
+			Pattern: "/v2/test/c3/3level/tags/list",
+			Handler: func(w http.ResponseWriter, r *http.Request) {
+				w.Write([]byte(`{"name":"test/c3/3level","tags":["tag4"]}`))
+			},
+		},
+	)
+}
+func Test_native_FetchImages(t *testing.T) {
+	var mock = mockNativeRegistry()
+	defer mock.Close()
+	fmt.Println("mockNativeRegistry URL: ", mock.URL)
+
+	var registry = &model.Registry{
+		Type:     model.RegistryTypeDockerRegistry,
+		URL:      mock.URL,
+		Insecure: true,
+	}
+	adapter, err := NewAdapter(registry)
+	assert.Nil(t, err)
+	assert.NotNil(t, adapter)
+
+	tests := []struct {
+		name    string
+		filters []*model.Filter
+		want    []*model.Resource
+		wantErr bool
+	}{
+		{
+			name: "repository not exist",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeName,
+					Value: "b1",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "tag not exist",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeTag,
+					Value: "this_tag_not_exist_in_the_mock_server",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name:    "no filters",
+			filters: []*model.Filter{},
+			want: []*model.Resource{
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/a1"},
+						Vtags:      []string{"tag11"},
+					},
+				},
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/b2"},
+						Vtags:      []string{"tag11", "tag2", "tag13"},
+					},
+				},
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/c3/3level"},
+						Vtags:      []string{"tag4"},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "only special repository",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeName,
+					Value: "test/a1",
+				},
+			},
+			want: []*model.Resource{
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/a1"},
+						Vtags:      []string{"tag11"},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "only special tag",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeTag,
+					Value: "tag11",
+				},
+			},
+			want: []*model.Resource{
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/a1"},
+						Vtags:      []string{"tag11"},
+					},
+				},
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/b2"},
+						Vtags:      []string{"tag11"},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "special repository and special tag",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeName,
+					Value: "test/b2",
+				},
+				{
+					Type:  model.FilterTypeTag,
+					Value: "tag2",
+				},
+			},
+			want: []*model.Resource{
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/b2"},
+						Vtags:      []string{"tag2"},
+					},
+				},
+			},
+
+			wantErr: false,
+		},
+		{
+			name: "only wildcard repository",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeName,
+					Value: "test/b*",
+				},
+			},
+			want: []*model.Resource{
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/b2"},
+						Vtags:      []string{"tag11", "tag2", "tag13"},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "only wildcard tag",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeTag,
+					Value: "tag1*",
+				},
+			},
+			want: []*model.Resource{
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/a1"},
+						Vtags:      []string{"tag11"},
+					},
+				},
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/b2"},
+						Vtags:      []string{"tag11", "tag13"},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "wildcard repository and wildcard tag",
+			filters: []*model.Filter{
+				{
+					Type:  model.FilterTypeName,
+					Value: "test/b*",
+				},
+				{
+					Type:  model.FilterTypeTag,
+					Value: "tag1*",
+				},
+			},
+			want: []*model.Resource{
+				{
+					Metadata: &model.ResourceMetadata{
+						Repository: &model.Repository{Name: "test/b2"},
+						Vtags:      []string{"tag11", "tag13"},
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var resources, err = adapter.FetchImages(tt.filters)
+			if tt.wantErr {
+				require.Len(t, resources, 0)
+				require.NotNil(t, err)
+			} else {
+				require.Equal(t, len(tt.want), len(resources))
+				for i, resource := range resources {
+					require.NotNil(t, resource.Metadata)
+					assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository)
+					assert.Equal(t, tt.want[i].Metadata.Vtags, resource.Metadata.Vtags)
+				}
+			}
+		})
+	}
+}
+
+func TestIsDigest(t *testing.T) {
+	cases := []struct {
+		str      string
+		isDigest bool
+	}{
+		{
+			str:      "",
+			isDigest: false,
+		},
+		{
+			str:      "latest",
+			isDigest: false,
+		},
+		{
+			str:      "sha256:fea8895f450959fa676bcc1df0611ea93823a735a01205fd8622846041d0c7cf",
+			isDigest: true,
+		},
+	}
+	for _, c := range cases {
+		assert.Equal(t, c.isDigest, isDigest(c.str))
+	}
+}

src/replication/adapter/native/image_registry.go

@@ -1,114 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package native
-
-import (
-	adp "github.com/goharbor/harbor/src/replication/adapter"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/goharbor/harbor/src/replication/util"
-)
-
-var _ adp.ImageRegistry = Native{}
-
-// FetchImages ...
-func (n Native) FetchImages(filters []*model.Filter) ([]*model.Resource, error) {
-	nameFilterPattern := ""
-	tagFilterPattern := ""
-	for _, filter := range filters {
-		switch filter.Type {
-		case model.FilterTypeName:
-			nameFilterPattern = filter.Value.(string)
-		case model.FilterTypeTag:
-			tagFilterPattern = filter.Value.(string)
-		}
-	}
-	repositories, err := n.filterRepositories(nameFilterPattern)
-	if err != nil {
-		return nil, err
-	}
-
-	var resources []*model.Resource
-	for _, repository := range repositories {
-		tags, err := n.filterTags(repository, tagFilterPattern)
-		if err != nil {
-			return nil, err
-		}
-		if len(tags) == 0 {
-			continue
-		}
-		resources = append(resources, &model.Resource{
-			Type:     model.ResourceTypeImage,
-			Registry: n.registry,
-			Metadata: &model.ResourceMetadata{
-				Repository: &model.Repository{
-					Name: repository,
-				},
-				Vtags: tags,
-			},
-		})
-	}
-
-	return resources, nil
-}
-
-func (n Native) filterRepositories(pattern string) ([]string, error) {
-	// if the pattern is a specific repository name, just returns the parsed repositories
-	// and will check the existence later when filtering the tags
-	if repositories, ok := util.IsSpecificPath(pattern); ok {
-		return repositories, nil
-	}
-	// search repositories from catalog api
-	repositories, err := n.Catalog()
-	if err != nil {
-		return nil, err
-	}
-	// if the pattern is null, just return the result of catalog API
-	if len(pattern) == 0 {
-		return repositories, nil
-	}
-	result := []string{}
-	for _, repository := range repositories {
-		match, err := util.Match(pattern, repository)
-		if err != nil {
-			return nil, err
-		}
-		if match {
-			result = append(result, repository)
-		}
-	}
-	return result, nil
-}
-
-func (n Native) filterTags(repository, pattern string) ([]string, error) {
-	tags, err := n.ListTag(repository)
-	if err != nil {
-		return nil, err
-	}
-	if len(pattern) == 0 {
-		return tags, nil
-	}
-
-	var result []string
-	for _, tag := range tags {
-		match, err := util.Match(pattern, tag)
-		if err != nil {
-			return nil, err
-		}
-		if match {
-			result = append(result, tag)
-		}
-	}
-	return result, nil
-}

src/replication/adapter/native/image_registry_test.go

@@ -1,279 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package native
-
-import (
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/goharbor/harbor/src/common/utils/test"
-	adp "github.com/goharbor/harbor/src/replication/adapter"
-	"github.com/goharbor/harbor/src/replication/model"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func mockNativeRegistry() (mock *httptest.Server) {
-	return test.NewServer(
-		&test.RequestHandlerMapping{
-			Method:  http.MethodGet,
-			Pattern: "/v2/_catalog",
-			Handler: func(w http.ResponseWriter, r *http.Request) {
-				w.Write([]byte(`{"repositories":["test/a1","test/b2","test/c3/3level"]}`))
-			},
-		},
-		&test.RequestHandlerMapping{
-			Method:  http.MethodGet,
-			Pattern: "/v2/test/a1/tags/list",
-			Handler: func(w http.ResponseWriter, r *http.Request) {
-				w.Write([]byte(`{"name":"test/a1","tags":["tag11"]}`))
-			},
-		},
-		&test.RequestHandlerMapping{
-			Method:  http.MethodGet,
-			Pattern: "/v2/test/b2/tags/list",
-			Handler: func(w http.ResponseWriter, r *http.Request) {
-				w.Write([]byte(`{"name":"test/b2","tags":["tag11","tag2","tag13"]}`))
-			},
-		},
-		&test.RequestHandlerMapping{
-			Method:  http.MethodGet,
-			Pattern: "/v2/test/c3/3level/tags/list",
-			Handler: func(w http.ResponseWriter, r *http.Request) {
-				w.Write([]byte(`{"name":"test/c3/3level","tags":["tag4"]}`))
-			},
-		},
-	)
-}
-func Test_native_FetchImages(t *testing.T) {
-	var mock = mockNativeRegistry()
-	defer mock.Close()
-	fmt.Println("mockNativeRegistry URL: ", mock.URL)
-
-	var registry = &model.Registry{
-		Type:     model.RegistryTypeDockerRegistry,
-		URL:      mock.URL,
-		Insecure: true,
-	}
-	var reg, err = adp.NewDefaultImageRegistry(registry)
-	assert.NotNil(t, reg)
-	assert.Nil(t, err)
-	var adapter = Native{
-		DefaultImageRegistry: reg,
-		registry:             registry,
-	}
-	assert.NotNil(t, adapter)
-
-	tests := []struct {
-		name    string
-		filters []*model.Filter
-		want    []*model.Resource
-		wantErr bool
-	}{
-		{
-			name: "repository not exist",
-			filters: []*model.Filter{
-				{
-					Type:  model.FilterTypeName,
-					Value: "b1",
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "tag not exist",
-			filters: []*model.Filter{
-				{
-					Type:  model.FilterTypeTag,
-					Value: "this_tag_not_exist_in_the_mock_server",
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name:    "no filters",
-			filters: []*model.Filter{},
-			want: []*model.Resource{
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/a1"},
-						Vtags:      []string{"tag11"},
-					},
-				},
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/b2"},
-						Vtags:      []string{"tag11", "tag2", "tag13"},
-					},
-				},
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/c3/3level"},
-						Vtags:      []string{"tag4"},
-					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "only special repository",
-			filters: []*model.Filter{
-				{
-					Type:  model.FilterTypeName,
-					Value: "test/a1",
-				},
-			},
-			want: []*model.Resource{
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/a1"},
-						Vtags:      []string{"tag11"},
-					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "only special tag",
-			filters: []*model.Filter{
-				{
-					Type:  model.FilterTypeTag,
-					Value: "tag11",
-				},
-			},
-			want: []*model.Resource{
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/a1"},
-						Vtags:      []string{"tag11"},
-					},
-				},
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/b2"},
-						Vtags:      []string{"tag11"},
-					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "special repository and special tag",
-			filters: []*model.Filter{
-				{
-					Type:  model.FilterTypeName,
-					Value: "test/b2",
-				},
-				{
-					Type:  model.FilterTypeTag,
-					Value: "tag2",
-				},
-			},
-			want: []*model.Resource{
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/b2"},
-						Vtags:      []string{"tag2"},
-					},
-				},
-			},
-
-			wantErr: false,
-		},
-		{
-			name: "only wildcard repository",
-			filters: []*model.Filter{
-				{
-					Type:  model.FilterTypeName,
-					Value: "test/b*",
-				},
-			},
-			want: []*model.Resource{
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/b2"},
-						Vtags:      []string{"tag11", "tag2", "tag13"},
-					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "only wildcard tag",
-			filters: []*model.Filter{
-				{
-					Type:  model.FilterTypeTag,
-					Value: "tag1*",
-				},
-			},
-			want: []*model.Resource{
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/a1"},
-						Vtags:      []string{"tag11"},
-					},
-				},
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/b2"},
-						Vtags:      []string{"tag11", "tag13"},
-					},
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "wildcard repository and wildcard tag",
-			filters: []*model.Filter{
-				{
-					Type:  model.FilterTypeName,
-					Value: "test/b*",
-				},
-				{
-					Type:  model.FilterTypeTag,
-					Value: "tag1*",
-				},
-			},
-			want: []*model.Resource{
-				{
-					Metadata: &model.ResourceMetadata{
-						Repository: &model.Repository{Name: "test/b2"},
-						Vtags:      []string{"tag11", "tag13"},
-					},
-				},
-			},
-			wantErr: false,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			var resources, err = adapter.FetchImages(tt.filters)
-			if tt.wantErr {
-				require.Len(t, resources, 0)
-				require.NotNil(t, err)
-			} else {
-				require.Equal(t, len(tt.want), len(resources))
-				for i, resource := range resources {
-					require.NotNil(t, resource.Metadata)
-					assert.Equal(t, tt.want[i].Metadata.Repository, resource.Metadata.Repository)
-					assert.Equal(t, tt.want[i].Metadata.Vtags, resource.Metadata.Vtags)
-				}
-			}
-		})
-	}
-}

tests/ldap_test.ldif

@@ -84,6 +84,8 @@ member: cn=user023,ou=people,dc=example,dc=com
 member: cn=user024,ou=people,dc=example,dc=com
 member: cn=user025,ou=people,dc=example,dc=com
 member: cn=user026,ou=people,dc=example,dc=com
+member: cn=user027,ou=people,dc=example,dc=com
+member: cn=user028,ou=people,dc=example,dc=com
 objectclass: groupOfNames
 objectclass: top
 
@@ -678,6 +680,42 @@ uid: user026
 uidnumber: 5030
 userpassword: {MD5}jMx5MaPXabPyX7F0SoFxYQ==
 
+dn: cn=user027,ou=people,dc=example,dc=com
+cn: user027
+gidnumber: 10000
+givenname: user027
+homedirectory: /home/user027
+loginshell: /bin/bash
+mail: user027@example.com
+objectclass: top
+objectclass: posixAccount
+objectclass: shadowAccount
+objectclass: inetOrgPerson
+objectclass: organizationalPerson
+objectclass: person
+sn: user027
+uid: user027
+uidnumber: 5031
+userpassword: {MD5}jMx5MaPXabPyX7F0SoFxYQ==
+
+dn: cn=user028,ou=people,dc=example,dc=com
+cn: user028
+gidnumber: 10000
+givenname: user028
+homedirectory: /home/user028
+loginshell: /bin/bash
+mail: user028@example.com
+objectclass: top
+objectclass: posixAccount
+objectclass: shadowAccount
+objectclass: inetOrgPerson
+objectclass: organizationalPerson
+objectclass: person
+sn: user028
+uid: user028
+uidnumber: 5032
+userpassword: {MD5}jMx5MaPXabPyX7F0SoFxYQ==
+
 dn: cn=guest_user,ou=people,dc=example,dc=com
 cn: guest_user
 gidnumber: 10000

tests/resources/Harbor-Pages/ToolKit.robot

@@ -77,8 +77,8 @@ Multi-delete Member
     [Arguments]    @{obj}
     :For  ${obj}  in  @{obj}
     \    Retry Element Click  //clr-dg-row[contains(.,'${obj}')]//label
-    Retry Element Click  ${member_action_xpath}
+    Retry Double Keywords When Error  Retry Element Click  ${member_action_xpath}  Retry Wait Until Page Contains Element  ${delete_action_xpath}
-    Retry Element Click  ${delete_action_xpath}
+    Retry Double Keywords When Error  Retry Element Click  ${delete_action_xpath}  Retry Wait Until Page Contains Element  ${delete_btn}
     Retry Double Keywords When Error  Retry Element Click  ${delete_btn}  Retry Wait Until Page Not Contains Element  ${delete_btn}