From eb5193e0ef0cd7911196008c4097c1e35c258b42 Mon Sep 17 00:00:00 2001 From: "stonezdj(Daojun Zhang)" Date: Fri, 9 Aug 2024 15:24:25 +0800 Subject: [PATCH] Parallel attach ldap group (#20705) Parallel attach LDAP group Add configure LDAP group attach parallel UI Change the /c/login timeout from 60 (nginx default) to 900 seconds in nginx.conf Signed-off-by: stonezdj --- api/v2.0/swagger.yaml | 8 ++ .../templates/nginx/nginx.http.conf.jinja | 3 + src/common/const.go | 1 + src/core/auth/ldap/ldap.go | 95 +++++++++++++++++-- src/lib/config/metadata/metadatalist.go | 1 + src/lib/config/models/model.go | 1 + src/lib/config/userconfig.go | 1 + .../config/auth/config-auth.component.html | 31 ++++++ .../config/auth/config-auth.component.ts | 7 +- .../app/base/left-side-nav/config/config.ts | 1 + src/portal/src/i18n/lang/en-us-lang.json | 4 +- src/portal/src/i18n/lang/zh-cn-lang.json | 5 +- 12 files changed, 144 insertions(+), 14 deletions(-) diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 681d0e354..07beb01e9 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -8989,6 +8989,9 @@ definitions: ldap_group_search_scope: $ref: '#/definitions/IntegerConfigItem' description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'' + ldap_group_attach_parallel: + $ref: '#/definitions/BoolConfigItem' + description: Attach LDAP user group information in parallel. ldap_scope: $ref: '#/definitions/IntegerConfigItem' description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE' @@ -9179,6 +9182,11 @@ definitions: description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'' x-omitempty: true x-isnullable: true + ldap_group_attach_parallel: + type: boolean + description: Attach LDAP user group information in parallel, the parallel worker count is 5 + x-omitempty: true + x-isnullable: true ldap_scope: type: integer description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE' diff --git a/make/photon/prepare/templates/nginx/nginx.http.conf.jinja b/make/photon/prepare/templates/nginx/nginx.http.conf.jinja index 6022b3ac9..7e55e72de 100644 --- a/make/photon/prepare/templates/nginx/nginx.http.conf.jinja +++ b/make/photon/prepare/templates/nginx/nginx.http.conf.jinja @@ -101,6 +101,9 @@ http { proxy_buffering off; proxy_request_buffering off; + + proxy_send_timeout 900; + proxy_read_timeout 900; } location /api/ { diff --git a/src/common/const.go b/src/common/const.go index aaa3c3fbe..224a2e4f3 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -134,6 +134,7 @@ const ( OIDCGroupType = 3 LDAPGroupAdminDn = "ldap_group_admin_dn" LDAPGroupMembershipAttribute = "ldap_group_membership_attribute" + LDAPGroupAttachParallel = "ldap_group_attach_parallel" DefaultRegistryControllerEndpoint = "http://registryctl:8080" DefaultPortalURL = "http://portal:8080" DefaultRegistryCtlURL = "http://registryctl:8080" diff --git a/src/core/auth/ldap/ldap.go b/src/core/auth/ldap/ldap.go index 38fa5a6f9..56533b287 100644 --- a/src/core/auth/ldap/ldap.go +++ b/src/core/auth/ldap/ldap.go @@ -21,6 +21,7 @@ import ( "strings" goldap "github.com/go-ldap/ldap/v3" + "golang.org/x/sync/errgroup" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/models" @@ -38,6 +39,10 @@ import ( ugModel "github.com/goharbor/harbor/src/pkg/usergroup/model" ) +const ( + workerCount = 5 +) + // Auth implements AuthenticateHelper interface to authenticate against LDAP type Auth struct { auth.DefaultAuthenticateHelper @@ -117,24 +122,94 @@ func (l *Auth) attachLDAPGroup(ctx context.Context, ldapUsers []model.User, u *m return } userGroups := make([]ugModel.UserGroup, 0) + if groupCfg.AttachParallel { + log.Debug("Attach LDAP group in parallel") + l.attachGroupParallel(ctx, ldapUsers, u) + return + } + // Attach LDAP group sequencially for _, dn := range ldapUsers[0].GroupDNList { - lGroups, err := sess.SearchGroupByDN(dn) - if err != nil { - log.Warningf("Can not get the ldap group name with DN %v, error %v", dn, err) - continue + if lgroup, exist := verifyGroupInLDAP(dn, sess); exist { + userGroups = append(userGroups, ugModel.UserGroup{GroupName: lgroup.Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType}) } - if len(lGroups) == 0 { - log.Warningf("Can not get the ldap group name with DN %v", dn) - continue - } - userGroups = append(userGroups, ugModel.UserGroup{GroupName: lGroups[0].Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType}) } u.GroupIDs, err = ugCtl.Ctl.Populate(ctx, userGroups) if err != nil { - log.Warningf("Failed to fetch ldap group configuration:%v", err) + log.Warningf("Failed to populate ldap group, error: %v", err) } } +func (l *Auth) attachGroupParallel(ctx context.Context, ldapUsers []model.User, u *models.User) { + userGroupsList := make([][]ugModel.UserGroup, workerCount) + gdsList := make([][]string, workerCount) + // Divide the groupDNs into workerCount parts + for index, dn := range ldapUsers[0].GroupDNList { + idx := index % workerCount + gdsList[idx] = append(gdsList[idx], dn) + } + g := new(errgroup.Group) + g.SetLimit(workerCount) + + for i := 0; i < workerCount; i++ { + curIndex := i + g.Go(func() error { + userGroups := make([]ugModel.UserGroup, 0) + groups := gdsList[curIndex] + if len(groups) == 0 { + return nil + } + // use different ldap session for each go routine + ldapSession, err := ldapCtl.Ctl.Session(ctx) + if err != nil { + return err + } + if err = ldapSession.Open(); err != nil { + return err + } + defer ldapSession.Close() + log.Debugf("Current worker index is %v", curIndex) + // verify and populate group + for _, dn := range groups { + if lgroup, exist := verifyGroupInLDAP(dn, ldapSession); exist { + userGroups = append(userGroups, ugModel.UserGroup{GroupName: lgroup.Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType}) + } + } + userGroupsList[curIndex] = userGroups + + return nil + }) + } + if err := g.Wait(); err != nil { + log.Warningf("failed to verify and populate ldap group parallel, error %v", err) + } + ugs := make([]ugModel.UserGroup, 0) + for _, userGroups := range userGroupsList { + ugs = append(ugs, userGroups...) + } + + groupIDsList, err := ugCtl.Ctl.Populate(ctx, ugs) + if err != nil { + log.Warningf("Failed to populate user groups :%v", err) + } + u.GroupIDs = groupIDsList +} + +func verifyGroupInLDAP(groupDN string, sess *ldap.Session) (*model.Group, bool) { + if _, err := goldap.ParseDN(groupDN); err != nil { + return nil, false + } + lGroups, err := sess.SearchGroupByDN(groupDN) + if err != nil { + log.Warningf("Can not get the ldap group name with DN %v, error %v", groupDN, err) + return nil, false + } + if len(lGroups) == 0 { + log.Warningf("Can not get the ldap group name with DN %v", groupDN) + return nil, false + } + return &lGroups[0], true +} + func (l *Auth) syncUserInfoFromDB(ctx context.Context, u *models.User) { // Retrieve SysAdminFlag from DB so that it transfer to session dbUser, err := l.userMgr.GetByName(ctx, u.Username) diff --git a/src/lib/config/metadata/metadatalist.go b/src/lib/config/metadata/metadatalist.go index dd2a7f67c..aab4919fd 100644 --- a/src/lib/config/metadata/metadatalist.go +++ b/src/lib/config/metadata/metadatalist.go @@ -96,6 +96,7 @@ var ( {Name: common.LDAPURL, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_URL", DefaultValue: "", ItemType: &NonEmptyStringType{}, Editable: false, Description: `The URL of LDAP server`}, {Name: common.LDAPVerifyCert, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_VERIFY_CERT", DefaultValue: "true", ItemType: &BoolType{}, Editable: false, Description: `Whether verify your OIDC server certificate, disable it if your OIDC server is hosted via self-hosted certificate.`}, {Name: common.LDAPGroupMembershipAttribute, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_MEMBERSHIP_ATTRIBUTE", DefaultValue: "memberof", ItemType: &StringType{}, Editable: true, Description: `The user attribute to identify the group membership`}, + {Name: common.LDAPGroupAttachParallel, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_ATTACH_PARALLEL", DefaultValue: "false", ItemType: &BoolType{}, Editable: true, Description: `Attach LDAP group information to Harbor in parallel`}, {Name: common.MaxJobWorkers, Scope: SystemScope, Group: BasicGroup, EnvKey: "MAX_JOB_WORKERS", DefaultValue: "10", ItemType: &IntType{}, Editable: false}, {Name: common.ScanAllPolicy, Scope: UserScope, Group: BasicGroup, EnvKey: "", DefaultValue: "", ItemType: &MapType{}, Editable: false, Description: `The policy to scan images`}, diff --git a/src/lib/config/models/model.go b/src/lib/config/models/model.go index 51a9c0c2c..25b41388a 100644 --- a/src/lib/config/models/model.go +++ b/src/lib/config/models/model.go @@ -94,6 +94,7 @@ type GroupConf struct { SearchScope int `json:"ldap_group_search_scope"` AdminDN string `json:"ldap_group_admin_dn,omitempty"` MembershipAttribute string `json:"ldap_group_membership_attribute,omitempty"` + AttachParallel bool `json:"ldap_group_attach_parallel,omitempty"` } type GDPRSetting struct { diff --git a/src/lib/config/userconfig.go b/src/lib/config/userconfig.go index a0fd5f90a..4012097c9 100644 --- a/src/lib/config/userconfig.go +++ b/src/lib/config/userconfig.go @@ -81,6 +81,7 @@ func LDAPGroupConf(ctx context.Context) (*cfgModels.GroupConf, error) { SearchScope: mgr.Get(ctx, common.LDAPGroupSearchScope).GetInt(), AdminDN: mgr.Get(ctx, common.LDAPGroupAdminDn).GetString(), MembershipAttribute: mgr.Get(ctx, common.LDAPGroupMembershipAttribute).GetString(), + AttachParallel: mgr.Get(ctx, common.LDAPGroupAttachParallel).GetBool(), }, nil } diff --git a/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html b/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html index b64ba7cf5..2b1401d1b 100644 --- a/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html +++ b/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html @@ -498,6 +498,37 @@ + + + + + +