diff --git a/Makefile b/Makefile index 0dfd8b6e44..45efb936f2 100644 --- a/Makefile +++ b/Makefile @@ -165,7 +165,6 @@ GOIMAGEBUILD_CORE=$(GOIMAGEBUILDCMD) $(GOFLAGS) ${GOTAGS} --ldflags "-w -s $(COR GOBUILDPATH_CORE=$(GOBUILDPATHINCONTAINER)/src/core GOBUILDPATH_JOBSERVICE=$(GOBUILDPATHINCONTAINER)/src/jobservice GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATHINCONTAINER)/src/registryctl -GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATHINCONTAINER)/src/cmd/migrate-patch GOBUILDPATH_STANDALONE_DB_MIGRATOR=$(GOBUILDPATHINCONTAINER)/src/cmd/standalone-db-migrator GOBUILDPATH_EXPORTER=$(GOBUILDPATHINCONTAINER)/src/cmd/exporter GOBUILDMAKEPATH=make @@ -182,7 +181,6 @@ JOBSERVICEBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_JOBSERVICE) JOBSERVICEBINARYNAME=harbor_jobservice REGISTRYCTLBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_REGISTRYCTL) REGISTRYCTLBINARYNAME=harbor_registryctl -MIGRATEPATCHBINARYNAME=migrate-patch STANDALONE_DB_MIGRATOR_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR) STANDALONE_DB_MIGRATOR_BINARYNAME=migrate @@ -548,7 +546,6 @@ cleanbinary: if [ -f $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ] ; then rm $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ; fi if [ -f $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ] ; then rm $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ; fi if [ -f $(REGISTRYCTLBINARYPATH)/$(REGISTRYCTLBINARYNAME) ] ; then rm $(REGISTRYCTLBINARYPATH)/$(REGISTRYCTLBINARYNAME) ; fi - if [ -f $(MIGRATEPATCHBINARYPATH)/$(MIGRATEPATCHBINARYNAME) ] ; then rm $(MIGRATEPATCHBINARYPATH)/$(MIGRATEPATCHBINARYNAME) ; fi rm -rf make/photon/*/binary/ cleanbaseimage: diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index c9e1e8a50f..c995f705e8 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -109,7 +109,7 @@ paths: operationId: searchLdapUser summary: Search available ldap users. description: | - This endpoint searches the available ldap users based on related configuration parameters. Support searched by input ladp configuration, load configuration from the system and specific filter. + This endpoint searches the available ldap users based on related configuration parameters. Support searched by input ldap configuration, load configuration from the system and specific filter. parameters: - $ref: '#/parameters/requestId' - name: username @@ -1548,6 +1548,88 @@ paths: $ref: '#/responses/409' '500': $ref: '#/responses/500' + /projects/{project_name_or_id}/artifacts: + get: + summary: List artifacts + description: List artifacts of the specified project + tags: + - project + operationId: listArtifactsOfProject + parameters: + - $ref: '#/parameters/requestId' + - $ref: '#/parameters/isResourceName' + - $ref: '#/parameters/projectNameOrId' + - $ref: '#/parameters/query' + - $ref: '#/parameters/sort' + - $ref: '#/parameters/page' + - $ref: '#/parameters/pageSize' + - $ref: '#/parameters/acceptVulnerabilities' + - name: with_tag + in: query + description: Specify whether the tags are included inside the returning artifacts + type: boolean + required: false + default: true + - name: with_label + in: query + description: Specify whether the labels are included inside the returning artifacts + type: boolean + required: false + default: false + - name: with_scan_overview + in: query + description: Specify whether the scan overview is included inside the returning artifacts + type: boolean + required: false + default: false + - name: with_sbom_overview + in: query + description: Specify whether the SBOM overview is included in returning artifacts, when this option is true, the SBOM overview will be included in the response + type: boolean + required: false + default: false + - name: with_immutable_status + in: query + description: Specify whether the immutable status is included inside the tags of the returning artifacts. Only works when setting "with_immutable_status=true" + type: boolean + required: false + default: false + - name: with_accessory + in: query + description: Specify whether the accessories are included of the returning artifacts. Only works when setting "with_accessory=true" + type: boolean + required: false + default: false + - name: latest_in_repository + in: query + description: Specify whether only the latest pushed artifact of each repository is included inside the returning artifacts. Only works when either artifact_type or media_type is included in the query. + type: boolean + required: false + default: false + responses: + '200': + description: Success + headers: + X-Total-Count: + description: The total count of artifacts + type: integer + Link: + description: Link refers to the previous page and next page + type: string + schema: + type: array + items: + $ref: '#/definitions/Artifact' + '400': + $ref: '#/responses/400' + '401': + $ref: '#/responses/401' + '403': + $ref: '#/responses/403' + '404': + $ref: '#/responses/404' + '500': + $ref: '#/responses/500' '/projects/{project_name_or_id}/scanner': get: summary: Get project level scanner @@ -6586,6 +6668,9 @@ definitions: manifest_media_type: type: string description: The manifest media type of the artifact + artifact_type: + type: string + description: The artifact_type in the manifest of the artifact project_id: type: integer format: int64 @@ -6594,6 +6679,9 @@ definitions: type: integer format: int64 description: The ID of the repository that the artifact belongs to + repository_name: + type: string + description: The name of the repository that the artifact belongs to digest: type: string description: The digest of the artifact @@ -7252,6 +7340,10 @@ definitions: type: string description: 'The ID of the tag retention policy for the project' x-nullable: true + proxy_speed_kb: + type: string + description: 'The bandwidth limit of proxy cache, in Kbps (kilobits per second). It limits the communication between Harbor and the upstream registry, not the client and the Harbor.' + x-nullable: true ProjectSummary: type: object properties: @@ -7754,6 +7846,9 @@ definitions: type: array items: $ref: '#/definitions/RobotPermission' + creator: + type: string + description: The creator of the robot creation_time: type: string format: date-time @@ -8897,6 +8992,9 @@ definitions: ldap_group_search_scope: $ref: '#/definitions/IntegerConfigItem' description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'' + ldap_group_attach_parallel: + $ref: '#/definitions/BoolConfigItem' + description: Attach LDAP user group information in parallel. ldap_scope: $ref: '#/definitions/IntegerConfigItem' description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE' @@ -9087,6 +9185,11 @@ definitions: description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'' x-omitempty: true x-isnullable: true + ldap_group_attach_parallel: + type: boolean + description: Attach LDAP user group information in parallel, the parallel worker count is 5 + x-omitempty: true + x-isnullable: true ldap_scope: type: integer description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE' diff --git a/make/migrations/postgresql/0150_2.12.0_schema.up.sql b/make/migrations/postgresql/0150_2.12.0_schema.up.sql new file mode 100644 index 0000000000..82f1061722 --- /dev/null +++ b/make/migrations/postgresql/0150_2.12.0_schema.up.sql @@ -0,0 +1,5 @@ +/* +Add new column creator for robot table to add a new column to record the creator of the robot +*/ +ALTER TABLE robot ADD COLUMN IF NOT EXISTS creator varchar(255); +UPDATE robot SET creator = 'unknown' WHERE creator IS NULL; diff --git a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja index 96a32af245..0ccf1ace52 100644 --- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja +++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja @@ -1,4 +1,3 @@ -version: '2.3' services: log: image: goharbor/harbor-log:{{version}} diff --git a/make/photon/prepare/templates/nginx/nginx.http.conf.jinja b/make/photon/prepare/templates/nginx/nginx.http.conf.jinja index 6022b3ac94..7e55e72ded 100644 --- a/make/photon/prepare/templates/nginx/nginx.http.conf.jinja +++ b/make/photon/prepare/templates/nginx/nginx.http.conf.jinja @@ -101,6 +101,9 @@ http { proxy_buffering off; proxy_request_buffering off; + + proxy_send_timeout 900; + proxy_read_timeout 900; } location /api/ { diff --git a/src/.mockery.yaml b/src/.mockery.yaml index c0d0b35a6c..7c5e856971 100644 --- a/src/.mockery.yaml +++ b/src/.mockery.yaml @@ -9,6 +9,17 @@ packages: Controller: config: dir: testing/controller/artifact + github.com/goharbor/harbor/src/controller/artifact/processor: + interfaces: + Processor: + config: + dir: testing/pkg/processor + github.com/goharbor/harbor/src/controller/artifact/annotation: + interfaces: + Parser: + config: + dir: testing/pkg/parser + outpkg: parser github.com/goharbor/harbor/src/controller/blob: interfaces: Controller: @@ -188,6 +199,11 @@ packages: Manager: config: dir: testing/pkg/artifact + github.com/goharbor/harbor/src/pkg/artifactrash: + interfaces: + Manager: + config: + dir: testing/pkg/artifactrash github.com/goharbor/harbor/src/pkg/blob: interfaces: Manager: @@ -218,6 +234,11 @@ packages: Handler: config: dir: testing/pkg/scan + github.com/goharbor/harbor/src/pkg/scan/postprocessors: + interfaces: + NativeScanReportConverter: + config: + dir: testing/pkg/scan/postprocessors github.com/goharbor/harbor/src/pkg/scan/report: interfaces: Manager: @@ -238,7 +259,7 @@ packages: dir: pkg/scheduler outpkg: scheduler mockname: mockDAO - filename: mock_dao_test.go + filename: mock_dao_test.go inpackage: True Scheduler: config: @@ -342,6 +363,14 @@ packages: DAO: config: dir: testing/pkg/immutable/dao + github.com/goharbor/harbor/src/pkg/immutable/match: + interfaces: + ImmutableTagMatcher: + config: + dir: testing/pkg/immutable + filename: matcher.go + outpkg: immutable + mockname: FakeMatcher github.com/goharbor/harbor/src/pkg/ldap: interfaces: Manager: @@ -505,20 +534,36 @@ packages: Manager: config: dir: testing/pkg/securityhub - - - - - - - - - - - - - - - - - + github.com/goharbor/harbor/src/pkg/tag: + interfaces: + Manager: + config: + dir: testing/pkg/tag + github.com/goharbor/harbor/src/pkg/p2p/preheat/policy: + interfaces: + Manager: + config: + dir: testing/pkg/p2p/preheat/policy + github.com/goharbor/harbor/src/pkg/p2p/preheat/instance: + interfaces: + Manager: + config: + dir: testing/pkg/p2p/preheat/instance + github.com/goharbor/harbor/src/pkg/chart: + interfaces: + Operator: + config: + dir: testing/pkg/chart + # registryctl related mocks + github.com/goharbor/harbor/src/registryctl/client: + interfaces: + Client: + config: + dir: testing/registryctl + outpkg: registryctl + # remote interfaces + github.com/docker/distribution: + interfaces: + Manifest: + config: + dir: testing/pkg/distribution diff --git a/src/cmd/migrate-patch/README.md b/src/cmd/migrate-patch/README.md deleted file mode 100644 index 9c9b70568f..0000000000 --- a/src/cmd/migrate-patch/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Migrate Patch -This is a simple program to fix the breakage that was introduced by migrate in notary. -## Usage -```sh -patch -database -``` diff --git a/src/cmd/migrate-patch/main.go b/src/cmd/migrate-patch/main.go deleted file mode 100644 index eb728b3a11..0000000000 --- a/src/cmd/migrate-patch/main.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "database/sql" - "flag" - "log" - "strings" - "time" - - _ "github.com/jackc/pgx/v4/stdlib" // registry pgx driver -) - -var dbURL string - -const pgSQLAlterStmt string = `ALTER TABLE schema_migrations ADD COLUMN "dirty" boolean NOT NULL DEFAULT false` -const pgSQLCheckColStmt string = `SELECT T1.C1, T2.C2 FROM -(SELECT COUNT(*) AS C1 FROM information_schema.tables WHERE table_name='schema_migrations') T1, -(SELECT COUNT(*) AS C2 FROM information_schema.columns WHERE table_name='schema_migrations' and column_name='dirty') T2` -const pgSQLDelRows string = `DELETE FROM schema_migrations t WHERE t.version < ( SELECT MAX(version) FROM schema_migrations )` - -func init() { - urlUsage := `The URL to the target database (driver://url). Currently it only supports postgres` - flag.StringVar(&dbURL, "database", "", urlUsage) -} - -func main() { - flag.Parse() - log.Printf("Updating database.") - if !strings.HasPrefix(dbURL, "postgres://") { - log.Fatalf("Invalid URL: '%s'\n", dbURL) - } - db, err := sql.Open("pgx", dbURL) - if err != nil { - log.Fatalf("Failed to connect to Database, error: %v\n", err) - } - defer db.Close() - - c := make(chan struct{}) - go func() { - defer close(c) - - err := db.Ping() - for ; err != nil; err = db.Ping() { - log.Println("Failed to Ping DB, sleep for 1 second.") - time.Sleep(1 * time.Second) - } - }() - select { - case <-c: - case <-time.After(30 * time.Second): - log.Fatal("Failed to connect DB after 30 seconds, time out. \n") - } - - row := db.QueryRow(pgSQLCheckColStmt) - var tblCount, colCount int - if err := row.Scan(&tblCount, &colCount); err != nil { - log.Fatalf("Failed to check schema_migrations table, error: %v \n", err) - } - if tblCount == 0 { - log.Println("schema_migrations table does not exist, skip.") - return - } - if colCount > 0 { - log.Println("schema_migrations table does not require update, skip.") - return - } - if _, err := db.Exec(pgSQLDelRows); err != nil { - log.Fatalf("Failed to clean up table, error: %v", err) - } - if _, err := db.Exec(pgSQLAlterStmt); err != nil { - log.Fatalf("Failed to update database, error: %v \n", err) - } - log.Println("Done updating database.") -} diff --git a/src/common/const.go b/src/common/const.go index aaa3c3fbe0..224a2e4f35 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -134,6 +134,7 @@ const ( OIDCGroupType = 3 LDAPGroupAdminDn = "ldap_group_admin_dn" LDAPGroupMembershipAttribute = "ldap_group_membership_attribute" + LDAPGroupAttachParallel = "ldap_group_attach_parallel" DefaultRegistryControllerEndpoint = "http://registryctl:8080" DefaultPortalURL = "http://portal:8080" DefaultRegistryCtlURL = "http://registryctl:8080" diff --git a/src/controller/artifact/controller.go b/src/controller/artifact/controller.go index 45f6e4f59c..4bcea401f1 100644 --- a/src/controller/artifact/controller.go +++ b/src/controller/artifact/controller.go @@ -118,6 +118,8 @@ type Controller interface { Walk(ctx context.Context, root *Artifact, walkFn func(*Artifact) error, option *Option) error // HasUnscannableLayer check artifact with digest if has unscannable layer HasUnscannableLayer(ctx context.Context, dgst string) (bool, error) + // ListWithLatest list the artifacts when the latest_in_repository in the query was set + ListWithLatest(ctx context.Context, query *q.Query, option *Option) (artifacts []*Artifact, err error) } // NewController creates an instance of the default artifact controller @@ -171,16 +173,18 @@ func (c *controller) Ensure(ctx context.Context, repository, digest string, opti } } } - // fire event - e := &metadata.PushArtifactEventMetadata{ - Ctx: ctx, - Artifact: artifact, - } + if created { + // fire event for create + e := &metadata.PushArtifactEventMetadata{ + Ctx: ctx, + Artifact: artifact, + } - if option != nil && len(option.Tags) > 0 { - e.Tag = option.Tags[0] + if option != nil && len(option.Tags) > 0 { + e.Tag = option.Tags[0] + } + notification.AddEvent(ctx, e) } - notification.AddEvent(ctx, e) return created, artifact.ID, nil } @@ -782,3 +786,16 @@ func (c *controller) HasUnscannableLayer(ctx context.Context, dgst string) (bool } return false, nil } + +// ListWithLatest ... +func (c *controller) ListWithLatest(ctx context.Context, query *q.Query, option *Option) (artifacts []*Artifact, err error) { + arts, err := c.artMgr.ListWithLatest(ctx, query) + if err != nil { + return nil, err + } + var res []*Artifact + for _, art := range arts { + res = append(res, c.assembleArtifact(ctx, art, option)) + } + return res, nil +} diff --git a/src/controller/artifact/controller_test.go b/src/controller/artifact/controller_test.go index 0e6ed458a2..d4b6743db7 100644 --- a/src/controller/artifact/controller_test.go +++ b/src/controller/artifact/controller_test.go @@ -67,7 +67,7 @@ type controllerTestSuite struct { ctl *controller repoMgr *repotesting.Manager artMgr *arttesting.Manager - artrashMgr *artrashtesting.FakeManager + artrashMgr *artrashtesting.Manager blobMgr *blob.Manager tagCtl *tagtesting.FakeController labelMgr *label.Manager @@ -80,7 +80,7 @@ type controllerTestSuite struct { func (c *controllerTestSuite) SetupTest() { c.repoMgr = &repotesting.Manager{} c.artMgr = &arttesting.Manager{} - c.artrashMgr = &artrashtesting.FakeManager{} + c.artrashMgr = &artrashtesting.Manager{} c.blobMgr = &blob.Manager{} c.tagCtl = &tagtesting.FakeController{} c.labelMgr = &label.Manager{} @@ -323,6 +323,44 @@ func (c *controllerTestSuite) TestList() { c.Equal(0, len(artifacts[0].Accessories)) } +func (c *controllerTestSuite) TestListWithLatest() { + query := &q.Query{} + option := &Option{ + WithTag: true, + WithAccessory: true, + } + c.artMgr.On("ListWithLatest", mock.Anything, mock.Anything).Return([]*artifact.Artifact{ + { + ID: 1, + RepositoryID: 1, + }, + }, nil) + c.tagCtl.On("List").Return([]*tag.Tag{ + { + Tag: model_tag.Tag{ + ID: 1, + RepositoryID: 1, + ArtifactID: 1, + Name: "latest", + }, + }, + }, nil) + c.repoMgr.On("Get", mock.Anything, mock.Anything).Return(&repomodel.RepoRecord{ + Name: "library/hello-world", + }, nil) + c.repoMgr.On("List", mock.Anything, mock.Anything).Return([]*repomodel.RepoRecord{ + {RepositoryID: 1, Name: "library/hello-world"}, + }, nil) + c.accMgr.On("List", mock.Anything, mock.Anything).Return([]accessorymodel.Accessory{}, nil) + artifacts, err := c.ctl.ListWithLatest(nil, query, option) + c.Require().Nil(err) + c.Require().Len(artifacts, 1) + c.Equal(int64(1), artifacts[0].ID) + c.Require().Len(artifacts[0].Tags, 1) + c.Equal(int64(1), artifacts[0].Tags[0].ID) + c.Equal(0, len(artifacts[0].Accessories)) +} + func (c *controllerTestSuite) TestGet() { c.artMgr.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(&artifact.Artifact{ ID: 1, @@ -476,7 +514,7 @@ func (c *controllerTestSuite) TestDeleteDeeply() { }, }, nil) c.repoMgr.On("Get", mock.Anything, mock.Anything).Return(&repomodel.RepoRecord{}, nil) - c.artrashMgr.On("Create").Return(0, nil) + c.artrashMgr.On("Create", mock.Anything, mock.Anything).Return(int64(0), nil) c.accMgr.On("List", mock.Anything, mock.Anything).Return([]accessorymodel.Accessory{}, nil) err = c.ctl.deleteDeeply(orm.NewContext(nil, &ormtesting.FakeOrmer{}), 1, false, false) c.Require().Nil(err) @@ -534,7 +572,7 @@ func (c *controllerTestSuite) TestDeleteDeeply() { c.blobMgr.On("List", mock.Anything, mock.Anything).Return(nil, nil) c.blobMgr.On("CleanupAssociationsForProject", mock.Anything, mock.Anything, mock.Anything).Return(nil) c.repoMgr.On("Get", mock.Anything, mock.Anything).Return(&repomodel.RepoRecord{}, nil) - c.artrashMgr.On("Create").Return(0, nil) + c.artrashMgr.On("Create", mock.Anything, mock.Anything).Return(int64(0), nil) err = c.ctl.deleteDeeply(orm.NewContext(nil, &ormtesting.FakeOrmer{}), 1, true, true) c.Require().Nil(err) diff --git a/src/controller/artifact/model.go b/src/controller/artifact/model.go index 6bf731dd75..d24369c986 100644 --- a/src/controller/artifact/model.go +++ b/src/controller/artifact/model.go @@ -102,8 +102,9 @@ type AdditionLink struct { // Option is used to specify the properties returned when listing/getting artifacts type Option struct { - WithTag bool - TagOption *tag.Option // only works when WithTag is set to true - WithLabel bool - WithAccessory bool + WithTag bool + TagOption *tag.Option // only works when WithTag is set to true + WithLabel bool + WithAccessory bool + LatestInRepository bool } diff --git a/src/controller/artifact/processor/chart/chart_test.go b/src/controller/artifact/processor/chart/chart_test.go index e8e208dd99..637da7bf6f 100644 --- a/src/controller/artifact/processor/chart/chart_test.go +++ b/src/controller/artifact/processor/chart/chart_test.go @@ -64,12 +64,12 @@ type processorTestSuite struct { suite.Suite processor *processor regCli *registry.Client - chartOptr *chart.FakeOpertaor + chartOptr *chart.Operator } func (p *processorTestSuite) SetupTest() { p.regCli = ®istry.Client{} - p.chartOptr = &chart.FakeOpertaor{} + p.chartOptr = &chart.Operator{} p.processor = &processor{ chartOperator: p.chartOptr, } @@ -106,7 +106,7 @@ func (p *processorTestSuite) TestAbstractAddition() { p.Require().Nil(err) p.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil) p.regCli.On("PullBlob", mock.Anything, mock.Anything).Return(int64(0), io.NopCloser(strings.NewReader(chartYaml)), nil) - p.chartOptr.On("GetDetails").Return(chartDetails, nil) + p.chartOptr.On("GetDetails", mock.Anything).Return(chartDetails, nil) // values.yaml addition, err := p.processor.AbstractAddition(nil, artifact, AdditionTypeValues) diff --git a/src/controller/event/handler/auditlog/auditlog.go b/src/controller/event/handler/auditlog/auditlog.go index 6c6b5c3e96..ef7cdbdda5 100644 --- a/src/controller/event/handler/auditlog/auditlog.go +++ b/src/controller/event/handler/auditlog/auditlog.go @@ -45,7 +45,8 @@ func (h *Handler) Handle(ctx context.Context, value interface{}) error { switch v := value.(type) { case *event.PushArtifactEvent, *event.DeleteArtifactEvent, *event.DeleteRepositoryEvent, *event.CreateProjectEvent, *event.DeleteProjectEvent, - *event.DeleteTagEvent, *event.CreateTagEvent: + *event.DeleteTagEvent, *event.CreateTagEvent, + *event.CreateRobotEvent, *event.DeleteRobotEvent: addAuditLog = true case *event.PullArtifactEvent: addAuditLog = !config.PullAuditLogDisable(ctx) diff --git a/src/controller/event/handler/init.go b/src/controller/event/handler/init.go index aa28103da2..841847a5cc 100644 --- a/src/controller/event/handler/init.go +++ b/src/controller/event/handler/init.go @@ -65,6 +65,8 @@ func init() { _ = notifier.Subscribe(event.TopicDeleteRepository, &auditlog.Handler{}) _ = notifier.Subscribe(event.TopicCreateTag, &auditlog.Handler{}) _ = notifier.Subscribe(event.TopicDeleteTag, &auditlog.Handler{}) + _ = notifier.Subscribe(event.TopicCreateRobot, &auditlog.Handler{}) + _ = notifier.Subscribe(event.TopicDeleteRobot, &auditlog.Handler{}) // internal _ = notifier.Subscribe(event.TopicPullArtifact, &internal.ArtifactEventHandler{}) diff --git a/src/controller/event/metadata/robot.go b/src/controller/event/metadata/robot.go new file mode 100644 index 0000000000..a4b325b34d --- /dev/null +++ b/src/controller/event/metadata/robot.go @@ -0,0 +1,73 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +import ( + "context" + "fmt" + "time" + + "github.com/goharbor/harbor/src/common/security" + event2 "github.com/goharbor/harbor/src/controller/event" + "github.com/goharbor/harbor/src/lib/config" + "github.com/goharbor/harbor/src/pkg/notifier/event" + "github.com/goharbor/harbor/src/pkg/robot/model" +) + +// CreateRobotEventMetadata is the metadata from which the create robot event can be resolved +type CreateRobotEventMetadata struct { + Ctx context.Context + Robot *model.Robot +} + +// Resolve to the event from the metadata +func (c *CreateRobotEventMetadata) Resolve(event *event.Event) error { + data := &event2.CreateRobotEvent{ + EventType: event2.TopicCreateRobot, + Robot: c.Robot, + OccurAt: time.Now(), + } + cx, exist := security.FromContext(c.Ctx) + if exist { + data.Operator = cx.GetUsername() + } + data.Robot.Name = fmt.Sprintf("%s%s", config.RobotPrefix(c.Ctx), data.Robot.Name) + event.Topic = event2.TopicCreateRobot + event.Data = data + return nil +} + +// DeleteRobotEventMetadata is the metadata from which the delete robot event can be resolved +type DeleteRobotEventMetadata struct { + Ctx context.Context + Robot *model.Robot +} + +// Resolve to the event from the metadata +func (d *DeleteRobotEventMetadata) Resolve(event *event.Event) error { + data := &event2.DeleteRobotEvent{ + EventType: event2.TopicDeleteRobot, + Robot: d.Robot, + OccurAt: time.Now(), + } + cx, exist := security.FromContext(d.Ctx) + if exist { + data.Operator = cx.GetUsername() + } + data.Robot.Name = fmt.Sprintf("%s%s", config.RobotPrefix(d.Ctx), data.Robot.Name) + event.Topic = event2.TopicDeleteRobot + event.Data = data + return nil +} diff --git a/src/controller/event/metadata/robot_test.go b/src/controller/event/metadata/robot_test.go new file mode 100644 index 0000000000..b7a2be3d77 --- /dev/null +++ b/src/controller/event/metadata/robot_test.go @@ -0,0 +1,83 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/goharbor/harbor/src/common" + event2 "github.com/goharbor/harbor/src/controller/event" + "github.com/goharbor/harbor/src/lib/config" + _ "github.com/goharbor/harbor/src/pkg/config/inmemory" + "github.com/goharbor/harbor/src/pkg/notifier/event" + "github.com/goharbor/harbor/src/pkg/robot/model" +) + +type robotEventTestSuite struct { + suite.Suite +} + +func (t *tagEventTestSuite) TestResolveOfCreateRobotEventMetadata() { + cfg := map[string]interface{}{ + common.RobotPrefix: "robot$", + } + config.InitWithSettings(cfg) + + e := &event.Event{} + metadata := &CreateRobotEventMetadata{ + Ctx: context.Background(), + Robot: &model.Robot{ + ID: 1, + Name: "test", + }, + } + err := metadata.Resolve(e) + t.Require().Nil(err) + t.Equal(event2.TopicCreateRobot, e.Topic) + t.Require().NotNil(e.Data) + data, ok := e.Data.(*event2.CreateRobotEvent) + t.Require().True(ok) + t.Equal(int64(1), data.Robot.ID) + t.Equal("robot$test", data.Robot.Name) +} + +func (t *tagEventTestSuite) TestResolveOfDeleteRobotEventMetadata() { + cfg := map[string]interface{}{ + common.RobotPrefix: "robot$", + } + config.InitWithSettings(cfg) + + e := &event.Event{} + metadata := &DeleteRobotEventMetadata{ + Ctx: context.Background(), + Robot: &model.Robot{ + ID: 1, + }, + } + err := metadata.Resolve(e) + t.Require().Nil(err) + t.Equal(event2.TopicDeleteRobot, e.Topic) + t.Require().NotNil(e.Data) + data, ok := e.Data.(*event2.DeleteRobotEvent) + t.Require().True(ok) + t.Equal(int64(1), data.Robot.ID) +} + +func TestRobotEventTestSuite(t *testing.T) { + suite.Run(t, &robotEventTestSuite{}) +} diff --git a/src/controller/event/topic.go b/src/controller/event/topic.go index 08e133e1a4..60dd8107a4 100644 --- a/src/controller/event/topic.go +++ b/src/controller/event/topic.go @@ -23,6 +23,7 @@ import ( "github.com/goharbor/harbor/src/pkg/artifact" "github.com/goharbor/harbor/src/pkg/audit/model" proModels "github.com/goharbor/harbor/src/pkg/project/models" + robotModel "github.com/goharbor/harbor/src/pkg/robot/model" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" ) @@ -47,6 +48,8 @@ const ( TopicReplication = "REPLICATION" TopicArtifactLabeled = "ARTIFACT_LABELED" TopicTagRetention = "TAG_RETENTION" + TopicCreateRobot = "CREATE_ROBOT" + TopicDeleteRobot = "DELETE_ROBOT" ) // CreateProjectEvent is the creating project event @@ -369,3 +372,53 @@ func (r *RetentionEvent) String() string { return fmt.Sprintf("TaskID-%d Status-%s Deleted-%s OccurAt-%s", r.TaskID, r.Status, candidates, r.OccurAt.Format("2006-01-02 15:04:05")) } + +// CreateRobotEvent is the creating robot event +type CreateRobotEvent struct { + EventType string + Robot *robotModel.Robot + Operator string + OccurAt time.Time +} + +// ResolveToAuditLog ... +func (c *CreateRobotEvent) ResolveToAuditLog() (*model.AuditLog, error) { + auditLog := &model.AuditLog{ + ProjectID: c.Robot.ProjectID, + OpTime: c.OccurAt, + Operation: rbac.ActionCreate.String(), + Username: c.Operator, + ResourceType: "robot", + Resource: c.Robot.Name} + return auditLog, nil +} + +func (c *CreateRobotEvent) String() string { + return fmt.Sprintf("Name-%s Operator-%s OccurAt-%s", + c.Robot.Name, c.Operator, c.OccurAt.Format("2006-01-02 15:04:05")) +} + +// DeleteRobotEvent is the deleting robot event +type DeleteRobotEvent struct { + EventType string + Robot *robotModel.Robot + Operator string + OccurAt time.Time +} + +// ResolveToAuditLog ... +func (c *DeleteRobotEvent) ResolveToAuditLog() (*model.AuditLog, error) { + auditLog := &model.AuditLog{ + ProjectID: c.Robot.ProjectID, + OpTime: c.OccurAt, + Operation: rbac.ActionDelete.String(), + Username: c.Operator, + ResourceType: "robot", + Resource: c.Robot.Name} + return auditLog, nil +} + +func (c *DeleteRobotEvent) String() string { + return fmt.Sprintf("Name-%s Operator-%s OccurAt-%s", + c.Robot.Name, c.Operator, c.OccurAt.Format("2006-01-02 15:04:05")) +} diff --git a/src/controller/p2p/preheat/controllor_test.go b/src/controller/p2p/preheat/controllor_test.go index b06af2672e..c57802dd98 100644 --- a/src/controller/p2p/preheat/controllor_test.go +++ b/src/controller/p2p/preheat/controllor_test.go @@ -31,8 +31,8 @@ type preheatSuite struct { suite.Suite ctx context.Context controller Controller - fakeInstanceMgr *instance.FakeManager - fakePolicyMgr *pmocks.FakeManager + fakeInstanceMgr *instance.Manager + fakePolicyMgr *pmocks.Manager fakeScheduler *smocks.Scheduler mockInstanceServer *httptest.Server fakeExecutionMgr *tmocks.ExecutionManager @@ -40,8 +40,8 @@ type preheatSuite struct { func TestPreheatSuite(t *testing.T) { t.Log("Start TestPreheatSuite") - fakeInstanceMgr := &instance.FakeManager{} - fakePolicyMgr := &pmocks.FakeManager{} + fakeInstanceMgr := &instance.Manager{} + fakePolicyMgr := &pmocks.Manager{} fakeScheduler := &smocks.Scheduler{} fakeExecutionMgr := &tmocks.ExecutionManager{} diff --git a/src/controller/p2p/preheat/enforcer_test.go b/src/controller/p2p/preheat/enforcer_test.go index b789bd3ab5..9d6ed5f243 100644 --- a/src/controller/p2p/preheat/enforcer_test.go +++ b/src/controller/p2p/preheat/enforcer_test.go @@ -70,7 +70,7 @@ func (suite *EnforcerTestSuite) SetupSuite() { suite.server.StartTLS() fakePolicies := mockPolicies() - fakePolicyManager := &policy.FakeManager{} + fakePolicyManager := &policy.Manager{} fakePolicyManager.On("Get", context.TODO(), mock.AnythingOfType("int64")). @@ -130,7 +130,7 @@ func (suite *EnforcerTestSuite) SetupSuite() { }, }, nil) - fakeInstanceMgr := &instance.FakeManager{} + fakeInstanceMgr := &instance.Manager{} fakeInstanceMgr.On("Get", context.TODO(), mock.AnythingOfType("int64"), diff --git a/src/controller/proxy/controller.go b/src/controller/proxy/controller.go index 38d55397c3..7138b6f0b0 100644 --- a/src/controller/proxy/controller.go +++ b/src/controller/proxy/controller.go @@ -264,7 +264,7 @@ func (c *controller) HeadManifest(_ context.Context, art lib.ArtifactInfo, remot func (c *controller) ProxyBlob(ctx context.Context, p *proModels.Project, art lib.ArtifactInfo) (int64, io.ReadCloser, error) { remoteRepo := getRemoteRepo(art) log.Debugf("The blob doesn't exist, proxy the request to the target server, url:%v", remoteRepo) - rHelper, err := NewRemoteHelper(ctx, p.RegistryID) + rHelper, err := NewRemoteHelper(ctx, p.RegistryID, WithSpeed(p.ProxyCacheSpeed())) if err != nil { return 0, nil, err } diff --git a/src/controller/proxy/options.go b/src/controller/proxy/options.go new file mode 100644 index 0000000000..2f81bfc3ff --- /dev/null +++ b/src/controller/proxy/options.go @@ -0,0 +1,37 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxy + +type Option func(*Options) + +type Options struct { + // Speed is the data transfer speed for proxy cache from Harbor to upstream registry, no limit by default. + Speed int32 +} + +func NewOptions(opts ...Option) *Options { + o := &Options{} + for _, opt := range opts { + opt(o) + } + + return o +} + +func WithSpeed(speed int32) Option { + return func(o *Options) { + o.Speed = speed + } +} diff --git a/src/controller/proxy/options_test.go b/src/controller/proxy/options_test.go new file mode 100644 index 0000000000..2b0a4ef801 --- /dev/null +++ b/src/controller/proxy/options_test.go @@ -0,0 +1,33 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxy + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewOptions(t *testing.T) { + // test default options + o := NewOptions() + assert.Equal(t, int32(0), o.Speed) + + // test with options + // with speed + withSpeed := WithSpeed(1024) + o = NewOptions(withSpeed) + assert.Equal(t, int32(1024), o.Speed) +} diff --git a/src/controller/proxy/remote.go b/src/controller/proxy/remote.go index ac7f23f28b..4143b81709 100644 --- a/src/controller/proxy/remote.go +++ b/src/controller/proxy/remote.go @@ -21,6 +21,7 @@ import ( "github.com/docker/distribution" + "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/pkg/reg" "github.com/goharbor/harbor/src/pkg/reg/adapter" "github.com/goharbor/harbor/src/pkg/reg/model" @@ -43,13 +44,16 @@ type remoteHelper struct { regID int64 registry adapter.ArtifactRegistry registryMgr reg.Manager + opts *Options } // NewRemoteHelper create a remote interface -func NewRemoteHelper(ctx context.Context, regID int64) (RemoteInterface, error) { +func NewRemoteHelper(ctx context.Context, regID int64, opts ...Option) (RemoteInterface, error) { r := &remoteHelper{ regID: regID, - registryMgr: reg.Mgr} + registryMgr: reg.Mgr, + opts: NewOptions(opts...), + } if err := r.init(ctx); err != nil { return nil, err } @@ -83,7 +87,14 @@ func (r *remoteHelper) init(ctx context.Context) error { } func (r *remoteHelper) BlobReader(repo, dig string) (int64, io.ReadCloser, error) { - return r.registry.PullBlob(repo, dig) + sz, bReader, err := r.registry.PullBlob(repo, dig) + if err != nil { + return 0, nil, err + } + if r.opts != nil && r.opts.Speed > 0 { + bReader = lib.NewReader(bReader, r.opts.Speed) + } + return sz, bReader, err } func (r *remoteHelper) Manifest(repo string, ref string) (distribution.Manifest, string, error) { diff --git a/src/controller/replication/transfer/image/transfer.go b/src/controller/replication/transfer/image/transfer.go index 55bda96469..09c1bea27e 100644 --- a/src/controller/replication/transfer/image/transfer.go +++ b/src/controller/replication/transfer/image/transfer.go @@ -30,6 +30,7 @@ import ( common_http "github.com/goharbor/harbor/src/common/http" trans "github.com/goharbor/harbor/src/controller/replication/transfer" + "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/pkg/reg/adapter" "github.com/goharbor/harbor/src/pkg/reg/model" @@ -380,7 +381,7 @@ func (t *transfer) copyBlobByMonolithic(srcRepo, dstRepo, digest string, sizeFro return err } if speed > 0 { - data = trans.NewReader(data, speed) + data = lib.NewReader(data, speed) } defer data.Close() // get size 0 from PullBlob, use size from distribution.Descriptor instead. @@ -435,7 +436,7 @@ func (t *transfer) copyBlobByChunk(srcRepo, dstRepo, digest string, sizeFromDesc } if speed > 0 { - data = trans.NewReader(data, speed) + data = lib.NewReader(data, speed) } // failureEnd will only be used for adjusting content range when issue happened during push the chunk. var failureEnd int64 diff --git a/src/controller/robot/controller.go b/src/controller/robot/controller.go index 28eef53e40..79ae3576cc 100644 --- a/src/controller/robot/controller.go +++ b/src/controller/robot/controller.go @@ -23,12 +23,14 @@ import ( rbac_project "github.com/goharbor/harbor/src/common/rbac/project" "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/controller/event/metadata" "github.com/goharbor/harbor/src/lib/config" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/lib/retry" "github.com/goharbor/harbor/src/pkg" + "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/permission/types" "github.com/goharbor/harbor/src/pkg/project" "github.com/goharbor/harbor/src/pkg/rbac" @@ -121,7 +123,8 @@ func (d *controller) Create(ctx context.Context, r *Robot) (int64, string, error if r.Level == LEVELPROJECT { name = fmt.Sprintf("%s+%s", r.ProjectName, r.Name) } - robotID, err := d.robotMgr.Create(ctx, &model.Robot{ + + rCreate := &model.Robot{ Name: name, Description: r.Description, ProjectID: r.ProjectID, @@ -130,7 +133,9 @@ func (d *controller) Create(ctx context.Context, r *Robot) (int64, string, error Duration: r.Duration, Salt: salt, Visible: r.Visible, - }) + Creator: r.Creator, + } + robotID, err := d.robotMgr.Create(ctx, rCreate) if err != nil { return 0, "", err } @@ -138,17 +143,31 @@ func (d *controller) Create(ctx context.Context, r *Robot) (int64, string, error if err := d.createPermission(ctx, r); err != nil { return 0, "", err } + // fire event + notification.AddEvent(ctx, &metadata.CreateRobotEventMetadata{ + Ctx: ctx, + Robot: rCreate, + }) return robotID, pwd, nil } // Delete ... func (d *controller) Delete(ctx context.Context, id int64) error { + rDelete, err := d.robotMgr.Get(ctx, id) + if err != nil { + return err + } if err := d.robotMgr.Delete(ctx, id); err != nil { return err } if err := d.rbacMgr.DeletePermissionsByRole(ctx, ROBOTTYPE, id); err != nil { return err } + // fire event + notification.AddEvent(ctx, &metadata.DeleteRobotEventMetadata{ + Ctx: ctx, + Robot: rDelete, + }) return nil } diff --git a/src/controller/robot/controller_test.go b/src/controller/robot/controller_test.go index 2a97d05928..1475d39e35 100644 --- a/src/controller/robot/controller_test.go +++ b/src/controller/robot/controller_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/utils/test" "github.com/goharbor/harbor/src/lib/config" "github.com/goharbor/harbor/src/lib/q" @@ -18,6 +19,7 @@ import ( rbac_model "github.com/goharbor/harbor/src/pkg/rbac/model" "github.com/goharbor/harbor/src/pkg/robot/model" htesting "github.com/goharbor/harbor/src/testing" + testsec "github.com/goharbor/harbor/src/testing/common/security" "github.com/goharbor/harbor/src/testing/mock" "github.com/goharbor/harbor/src/testing/pkg/project" "github.com/goharbor/harbor/src/testing/pkg/rbac" @@ -102,7 +104,9 @@ func (suite *ControllerTestSuite) TestCreate() { robotMgr := &robot.Manager{} c := controller{robotMgr: robotMgr, rbacMgr: rbacMgr, proMgr: projectMgr} - ctx := context.TODO() + secCtx := &testsec.Context{} + secCtx.On("GetUsername").Return("security-context-user") + ctx := security.NewContext(context.Background(), secCtx) projectMgr.On("Get", mock.Anything, mock.Anything).Return(&proModels.Project{ProjectID: 1, Name: "library"}, nil) robotMgr.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil) rbacMgr.On("CreateRbacPolicy", mock.Anything, mock.Anything, mock.Anything).Return(int64(1), nil) @@ -145,6 +149,12 @@ func (suite *ControllerTestSuite) TestDelete() { c := controller{robotMgr: robotMgr, rbacMgr: rbacMgr, proMgr: projectMgr} ctx := context.TODO() + robotMgr.On("Get", mock.Anything, mock.Anything).Return(&model.Robot{ + Name: "library+test", + Description: "test get method", + ProjectID: 1, + Secret: utils.RandStringBytes(10), + }, nil) robotMgr.On("Delete", mock.Anything, mock.Anything).Return(nil) rbacMgr.On("DeletePermissionsByRole", mock.Anything, mock.Anything, mock.Anything).Return(nil) diff --git a/src/controller/scan/base_controller.go b/src/controller/scan/base_controller.go index f6a0427b4d..fe4a15faf8 100644 --- a/src/controller/scan/base_controller.go +++ b/src/controller/scan/base_controller.go @@ -864,6 +864,7 @@ func (bc *basicController) makeRobotAccount(ctx context.Context, projectID int64 Description: "for scan", ProjectID: projectID, Duration: -1, + Creator: "harbor-core-for-scan-all", }, Level: robot.LEVELPROJECT, Permissions: []*robot.Permission{ diff --git a/src/controller/scan/base_controller_test.go b/src/controller/scan/base_controller_test.go index 28811ce68b..028c860d7a 100644 --- a/src/controller/scan/base_controller_test.go +++ b/src/controller/scan/base_controller_test.go @@ -82,7 +82,7 @@ type ControllerTestSuite struct { reportMgr *reporttesting.Manager ar artifact.Controller c *basicController - reportConverter *postprocessorstesting.ScanReportV1ToV2Converter + reportConverter *postprocessorstesting.NativeScanReportConverter cache *mockcache.Cache } @@ -235,6 +235,7 @@ func (suite *ControllerTestSuite) SetupSuite() { Description: "for scan", ProjectID: suite.artifact.ProjectID, Duration: -1, + Creator: "harbor-core-for-scan-all", }, Level: robot.LEVELPROJECT, Permissions: []*robot.Permission{ @@ -266,6 +267,7 @@ func (suite *ControllerTestSuite) SetupSuite() { Description: "for scan", ProjectID: suite.artifact.ProjectID, Duration: -1, + Creator: "harbor-core-for-scan-all", }, Level: "project", }, nil) @@ -339,7 +341,7 @@ func (suite *ControllerTestSuite) SetupSuite() { execMgr: suite.execMgr, taskMgr: suite.taskMgr, - reportConverter: &postprocessorstesting.ScanReportV1ToV2Converter{}, + reportConverter: &postprocessorstesting.NativeScanReportConverter{}, cache: func() cache.Cache { return suite.cache }, } mock.OnAnything(suite.scanHandler, "JobVendorType").Return("IMAGE_SCAN") @@ -486,6 +488,7 @@ func (suite *ControllerTestSuite) TestScanControllerGetReport() { {ExtraAttrs: suite.makeExtraAttrs(int64(1), "rp-uuid-001")}, }, nil).Once() mock.OnAnything(suite.accessoryMgr, "List").Return(nil, nil) + mock.OnAnything(suite.c.reportConverter, "FromRelationalSchema").Return("", nil) rep, err := suite.c.GetReport(ctx, suite.artifact, []string{v1.MimeTypeNativeReport}) require.NoError(suite.T(), err) assert.Equal(suite.T(), 1, len(rep)) diff --git a/src/controller/scan/callback_test.go b/src/controller/scan/callback_test.go index e1fe6c4e28..165524e458 100644 --- a/src/controller/scan/callback_test.go +++ b/src/controller/scan/callback_test.go @@ -51,7 +51,7 @@ type CallbackTestSuite struct { scanCtl Controller taskMgr *tasktesting.Manager - reportConverter *postprocessorstesting.ScanReportV1ToV2Converter + reportConverter *postprocessorstesting.NativeScanReportConverter } func (suite *CallbackTestSuite) SetupSuite() { @@ -69,7 +69,7 @@ func (suite *CallbackTestSuite) SetupSuite() { suite.taskMgr = &tasktesting.Manager{} taskMgr = suite.taskMgr - suite.reportConverter = &postprocessorstesting.ScanReportV1ToV2Converter{} + suite.reportConverter = &postprocessorstesting.NativeScanReportConverter{} suite.scanCtl = &basicController{ makeCtx: context.TODO, diff --git a/src/controller/securityhub/controller_test.go b/src/controller/securityhub/controller_test.go index 600c692bf8..68140d2f6b 100644 --- a/src/controller/securityhub/controller_test.go +++ b/src/controller/securityhub/controller_test.go @@ -44,7 +44,7 @@ type ControllerTestSuite struct { c *controller scannerMgr *scannerMock.Manager secHubMgr *securityMock.Manager - tagMgr *tagMock.FakeManager + tagMgr *tagMock.Manager } // TestController is the entry of controller test suite @@ -56,7 +56,7 @@ func TestController(t *testing.T) { func (suite *ControllerTestSuite) SetupTest() { suite.secHubMgr = &securityMock.Manager{} suite.scannerMgr = &scannerMock.Manager{} - suite.tagMgr = &tagMock.FakeManager{} + suite.tagMgr = &tagMock.Manager{} suite.c = &controller{ secHubMgr: suite.secHubMgr, diff --git a/src/controller/tag/controller_test.go b/src/controller/tag/controller_test.go index bd8d1e7ecd..9755bbd607 100644 --- a/src/controller/tag/controller_test.go +++ b/src/controller/tag/controller_test.go @@ -18,7 +18,6 @@ import ( "testing" "time" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "github.com/goharbor/harbor/src/lib/errors" @@ -27,6 +26,7 @@ import ( _ "github.com/goharbor/harbor/src/pkg/config/inmemory" "github.com/goharbor/harbor/src/pkg/tag/model/tag" ormtesting "github.com/goharbor/harbor/src/testing/lib/orm" + "github.com/goharbor/harbor/src/testing/mock" "github.com/goharbor/harbor/src/testing/pkg/artifact" "github.com/goharbor/harbor/src/testing/pkg/immutable" "github.com/goharbor/harbor/src/testing/pkg/repository" @@ -38,14 +38,14 @@ type controllerTestSuite struct { ctl *controller repoMgr *repository.Manager artMgr *artifact.Manager - tagMgr *tagtesting.FakeManager + tagMgr *tagtesting.Manager immutableMtr *immutable.FakeMatcher } func (c *controllerTestSuite) SetupTest() { c.repoMgr = &repository.Manager{} c.artMgr = &artifact.Manager{} - c.tagMgr = &tagtesting.FakeManager{} + c.tagMgr = &tagtesting.Manager{} c.immutableMtr = &immutable.FakeMatcher{} c.ctl = &controller{ tagMgr: c.tagMgr, @@ -56,7 +56,7 @@ func (c *controllerTestSuite) SetupTest() { func (c *controllerTestSuite) TestEnsureTag() { // the tag already exists under the repository and is attached to the artifact - c.tagMgr.On("List").Return([]*tag.Tag{ + c.tagMgr.On("List", mock.Anything, mock.Anything).Return([]*tag.Tag{ { ID: 1, RepositoryID: 1, @@ -67,7 +67,7 @@ func (c *controllerTestSuite) TestEnsureTag() { c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(false, nil) + mock.OnAnything(c.immutableMtr, "Match").Return(false, nil) _, err := c.ctl.Ensure(orm.NewContext(nil, &ormtesting.FakeOrmer{}), 1, 1, "latest") c.Require().Nil(err) c.tagMgr.AssertExpectations(c.T()) @@ -76,7 +76,7 @@ func (c *controllerTestSuite) TestEnsureTag() { c.SetupTest() // the tag exists under the repository, but it is attached to other artifact - c.tagMgr.On("List").Return([]*tag.Tag{ + c.tagMgr.On("List", mock.Anything, mock.Anything).Return([]*tag.Tag{ { ID: 1, RepositoryID: 1, @@ -84,11 +84,11 @@ func (c *controllerTestSuite) TestEnsureTag() { Name: "latest", }, }, nil) - c.tagMgr.On("Update").Return(nil) + c.tagMgr.On("Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(false, nil) + mock.OnAnything(c.immutableMtr, "Match").Return(false, nil) _, err = c.ctl.Ensure(orm.NewContext(nil, &ormtesting.FakeOrmer{}), 1, 1, "latest") c.Require().Nil(err) c.tagMgr.AssertExpectations(c.T()) @@ -97,26 +97,26 @@ func (c *controllerTestSuite) TestEnsureTag() { c.SetupTest() // the tag doesn't exist under the repository, create it - c.tagMgr.On("List").Return([]*tag.Tag{}, nil) - c.tagMgr.On("Create").Return(1, nil) + c.tagMgr.On("List", mock.Anything, mock.Anything).Return([]*tag.Tag{}, nil) + c.tagMgr.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil) c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(false, nil) + mock.OnAnything(c.immutableMtr, "Match").Return(false, nil) _, err = c.ctl.Ensure(orm.NewContext(nil, &ormtesting.FakeOrmer{}), 1, 1, "latest") c.Require().Nil(err) c.tagMgr.AssertExpectations(c.T()) } func (c *controllerTestSuite) TestCount() { - c.tagMgr.On("Count").Return(1, nil) + c.tagMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil) total, err := c.ctl.Count(nil, nil) c.Require().Nil(err) c.Equal(int64(1), total) } func (c *controllerTestSuite) TestList() { - c.tagMgr.On("List").Return([]*tag.Tag{ + c.tagMgr.On("List", mock.Anything, mock.Anything).Return([]*tag.Tag{ { RepositoryID: 1, Name: "testlist", @@ -134,7 +134,7 @@ func (c *controllerTestSuite) TestGet() { getTest.RepositoryID = 1 getTest.Name = "testget" - c.tagMgr.On("Get").Return(getTest, nil) + c.tagMgr.On("Get", mock.Anything, mock.Anything).Return(getTest, nil) tag, err := c.ctl.Get(nil, 1, nil) c.Require().Nil(err) c.tagMgr.AssertExpectations(c.T()) @@ -143,36 +143,36 @@ func (c *controllerTestSuite) TestGet() { } func (c *controllerTestSuite) TestDelete() { - c.tagMgr.On("Get").Return(&tag.Tag{ + c.tagMgr.On("Get", mock.Anything, mock.Anything).Return(&tag.Tag{ RepositoryID: 1, Name: "test", }, nil) c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(false, nil) - c.tagMgr.On("Delete").Return(nil) + mock.OnAnything(c.immutableMtr, "Match").Return(false, nil) + c.tagMgr.On("Delete", mock.Anything, mock.Anything).Return(nil) err := c.ctl.Delete(nil, 1) c.Require().Nil(err) } func (c *controllerTestSuite) TestDeleteImmutable() { - c.tagMgr.On("Get").Return(&tag.Tag{ + c.tagMgr.On("Get", mock.Anything, mock.Anything).Return(&tag.Tag{ RepositoryID: 1, Name: "test", }, nil) c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(true, nil) - c.tagMgr.On("Delete").Return(nil) + mock.OnAnything(c.immutableMtr, "Match").Return(true, nil) + c.tagMgr.On("Delete", mock.Anything, mock.Anything).Return(nil) err := c.ctl.Delete(nil, 1) c.Require().NotNil(err) c.True(errors.IsErr(err, errors.PreconditionCode)) } func (c *controllerTestSuite) TestUpdate() { - c.tagMgr.On("Update").Return(nil) + mock.OnAnything(c.tagMgr, "Update").Return(nil) err := c.ctl.Update(nil, &Tag{ Tag: tag.Tag{ RepositoryID: 1, @@ -184,14 +184,14 @@ func (c *controllerTestSuite) TestUpdate() { } func (c *controllerTestSuite) TestDeleteTags() { - c.tagMgr.On("Get").Return(&tag.Tag{ + c.tagMgr.On("Get", mock.Anything, mock.Anything).Return(&tag.Tag{ RepositoryID: 1, }, nil) c.artMgr.On("Get", mock.Anything, mock.Anything).Return(&pkg_artifact.Artifact{ ID: 1, }, nil) - c.immutableMtr.On("Match").Return(false, nil) - c.tagMgr.On("Delete").Return(nil) + mock.OnAnything(c.immutableMtr, "Match").Return(false, nil) + c.tagMgr.On("Delete", mock.Anything, mock.Anything).Return(nil) ids := []int64{1, 2, 3, 4} err := c.ctl.DeleteTags(nil, ids) c.Require().Nil(err) @@ -218,7 +218,7 @@ func (c *controllerTestSuite) TestAssembleTag() { } c.artMgr.On("Get", mock.Anything, mock.Anything).Return(art, nil) - c.immutableMtr.On("Match").Return(true, nil) + mock.OnAnything(c.immutableMtr, "Match").Return(true, nil) tag := c.ctl.assembleTag(nil, tg, option) c.Require().NotNil(tag) c.Equal(tag.ID, tg.ID) diff --git a/src/core/auth/ldap/ldap.go b/src/core/auth/ldap/ldap.go index 38fa5a6f93..56533b2875 100644 --- a/src/core/auth/ldap/ldap.go +++ b/src/core/auth/ldap/ldap.go @@ -21,6 +21,7 @@ import ( "strings" goldap "github.com/go-ldap/ldap/v3" + "golang.org/x/sync/errgroup" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/models" @@ -38,6 +39,10 @@ import ( ugModel "github.com/goharbor/harbor/src/pkg/usergroup/model" ) +const ( + workerCount = 5 +) + // Auth implements AuthenticateHelper interface to authenticate against LDAP type Auth struct { auth.DefaultAuthenticateHelper @@ -117,24 +122,94 @@ func (l *Auth) attachLDAPGroup(ctx context.Context, ldapUsers []model.User, u *m return } userGroups := make([]ugModel.UserGroup, 0) + if groupCfg.AttachParallel { + log.Debug("Attach LDAP group in parallel") + l.attachGroupParallel(ctx, ldapUsers, u) + return + } + // Attach LDAP group sequencially for _, dn := range ldapUsers[0].GroupDNList { - lGroups, err := sess.SearchGroupByDN(dn) - if err != nil { - log.Warningf("Can not get the ldap group name with DN %v, error %v", dn, err) - continue + if lgroup, exist := verifyGroupInLDAP(dn, sess); exist { + userGroups = append(userGroups, ugModel.UserGroup{GroupName: lgroup.Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType}) } - if len(lGroups) == 0 { - log.Warningf("Can not get the ldap group name with DN %v", dn) - continue - } - userGroups = append(userGroups, ugModel.UserGroup{GroupName: lGroups[0].Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType}) } u.GroupIDs, err = ugCtl.Ctl.Populate(ctx, userGroups) if err != nil { - log.Warningf("Failed to fetch ldap group configuration:%v", err) + log.Warningf("Failed to populate ldap group, error: %v", err) } } +func (l *Auth) attachGroupParallel(ctx context.Context, ldapUsers []model.User, u *models.User) { + userGroupsList := make([][]ugModel.UserGroup, workerCount) + gdsList := make([][]string, workerCount) + // Divide the groupDNs into workerCount parts + for index, dn := range ldapUsers[0].GroupDNList { + idx := index % workerCount + gdsList[idx] = append(gdsList[idx], dn) + } + g := new(errgroup.Group) + g.SetLimit(workerCount) + + for i := 0; i < workerCount; i++ { + curIndex := i + g.Go(func() error { + userGroups := make([]ugModel.UserGroup, 0) + groups := gdsList[curIndex] + if len(groups) == 0 { + return nil + } + // use different ldap session for each go routine + ldapSession, err := ldapCtl.Ctl.Session(ctx) + if err != nil { + return err + } + if err = ldapSession.Open(); err != nil { + return err + } + defer ldapSession.Close() + log.Debugf("Current worker index is %v", curIndex) + // verify and populate group + for _, dn := range groups { + if lgroup, exist := verifyGroupInLDAP(dn, ldapSession); exist { + userGroups = append(userGroups, ugModel.UserGroup{GroupName: lgroup.Name, LdapGroupDN: dn, GroupType: common.LDAPGroupType}) + } + } + userGroupsList[curIndex] = userGroups + + return nil + }) + } + if err := g.Wait(); err != nil { + log.Warningf("failed to verify and populate ldap group parallel, error %v", err) + } + ugs := make([]ugModel.UserGroup, 0) + for _, userGroups := range userGroupsList { + ugs = append(ugs, userGroups...) + } + + groupIDsList, err := ugCtl.Ctl.Populate(ctx, ugs) + if err != nil { + log.Warningf("Failed to populate user groups :%v", err) + } + u.GroupIDs = groupIDsList +} + +func verifyGroupInLDAP(groupDN string, sess *ldap.Session) (*model.Group, bool) { + if _, err := goldap.ParseDN(groupDN); err != nil { + return nil, false + } + lGroups, err := sess.SearchGroupByDN(groupDN) + if err != nil { + log.Warningf("Can not get the ldap group name with DN %v, error %v", groupDN, err) + return nil, false + } + if len(lGroups) == 0 { + log.Warningf("Can not get the ldap group name with DN %v", groupDN) + return nil, false + } + return &lGroups[0], true +} + func (l *Auth) syncUserInfoFromDB(ctx context.Context, u *models.User) { // Retrieve SysAdminFlag from DB so that it transfer to session dbUser, err := l.userMgr.GetByName(ctx, u.Username) diff --git a/src/go.mod b/src/go.mod index cb3fc840e9..f33eaa3faf 100644 --- a/src/go.mod +++ b/src/go.mod @@ -21,19 +21,19 @@ require ( github.com/go-asn1-ber/asn1-ber v1.5.7 github.com/go-ldap/ldap/v3 v3.4.6 github.com/go-openapi/errors v0.22.0 - github.com/go-openapi/loads v0.21.2 // indirect + github.com/go-openapi/loads v0.21.2 github.com/go-openapi/runtime v0.26.2 - github.com/go-openapi/spec v0.20.11 // indirect + github.com/go-openapi/spec v0.20.11 github.com/go-openapi/strfmt v0.23.0 github.com/go-openapi/swag v0.23.0 - github.com/go-openapi/validate v0.22.3 // indirect + github.com/go-openapi/validate v0.22.3 github.com/go-redis/redis/v8 v8.11.4 github.com/gocarina/gocsv v0.0.0-20210516172204-ca9e8a8ddea8 github.com/gocraft/work v0.5.1 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang-migrate/migrate/v4 v4.17.1 github.com/gomodule/redigo v2.0.0+incompatible - github.com/google/go-containerregistry v0.19.2 + github.com/google/go-containerregistry v0.20.2 github.com/google/uuid v1.6.0 github.com/gorilla/csrf v1.7.2 github.com/gorilla/handlers v1.5.2 @@ -63,7 +63,7 @@ require ( go.opentelemetry.io/otel/sdk v1.27.0 go.opentelemetry.io/otel/trace v1.28.0 go.uber.org/ratelimit v0.3.1 - golang.org/x/crypto v0.24.0 + golang.org/x/crypto v0.25.0 golang.org/x/net v0.26.0 golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 @@ -71,10 +71,10 @@ require ( golang.org/x/time v0.5.0 gopkg.in/h2non/gock.v1 v1.1.2 gopkg.in/yaml.v2 v2.4.0 - helm.sh/helm/v3 v3.15.2 - k8s.io/api v0.30.2 - k8s.io/apimachinery v0.30.2 - k8s.io/client-go v0.30.0 + helm.sh/helm/v3 v3.15.4 + k8s.io/api v0.30.3 + k8s.io/apimachinery v0.30.3 + k8s.io/client-go v0.30.3 sigs.k8s.io/yaml v1.4.0 ) @@ -100,8 +100,7 @@ require ( github.com/denverdino/aliyungo v0.0.0-20191227032621-df38c6fa730c // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dnaeon/go-vcr v1.2.0 // indirect - github.com/docker/cli v25.0.1+incompatible // indirect - github.com/docker/docker v25.0.5+incompatible // indirect + github.com/docker/cli v27.1.1+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -173,8 +172,8 @@ require ( go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.21.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.22.0 // indirect google.golang.org/api v0.171.0 // indirect google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect diff --git a/src/go.sum b/src/go.sum index 3f704380ad..784cde69e6 100644 --- a/src/go.sum +++ b/src/go.sum @@ -117,10 +117,10 @@ github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/cli v25.0.1+incompatible h1:mFpqnrS6Hsm3v1k7Wa/BO23oz0k121MTbTO1lpcGSkU= -github.com/docker/cli v25.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE= -github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE= +github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg= +github.com/docker/docker v25.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= @@ -250,8 +250,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.19.2 h1:TannFKE1QSajsP6hPWb5oJNgKe1IKjHukIKDUmvsV6w= -github.com/google/go-containerregistry v0.19.2/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo= +github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -627,8 +627,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= @@ -668,8 +668,6 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -716,16 +714,16 @@ golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -783,8 +781,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go. google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -834,17 +830,17 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -helm.sh/helm/v3 v3.15.2 h1:/3XINUFinJOBjQplGnjw92eLGpgXXp1L8chWPkCkDuw= -helm.sh/helm/v3 v3.15.2/go.mod h1:FzSIP8jDQaa6WAVg9F+OkKz7J0ZmAga4MABtTbsb9WQ= +helm.sh/helm/v3 v3.15.4 h1:UFHd6oZ1IN3FsUZ7XNhOQDyQ2QYknBNWRHH57e9cbHY= +helm.sh/helm/v3 v3.15.4/go.mod h1:phOwlxqGSgppCY/ysWBNRhG3MtnpsttOzxaTK+Mt40E= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= -k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= -k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= -k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= -k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= +k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= +k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= +k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= +k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= +k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= diff --git a/src/jobservice/job/impl/gc/garbage_collection.go b/src/jobservice/job/impl/gc/garbage_collection.go index 7ae9a0b684..7968a1020f 100644 --- a/src/jobservice/job/impl/gc/garbage_collection.go +++ b/src/jobservice/job/impl/gc/garbage_collection.go @@ -318,18 +318,16 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { return errGcStop } - atomic.AddInt64(&index, 1) - index := atomic.LoadInt64(&index) - + localIndex := atomic.AddInt64(&index, 1) // set the status firstly, if the blob is updated by any HEAD/PUT request, it should be fail and skip. blob.Status = blobModels.StatusDeleting count, err := gc.blobMgr.UpdateBlobStatus(ctx.SystemContext(), blob) if err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to mark gc candidate deleting, skip: %s, %s", uid, index, total, blob.Digest, blob.Status) + gc.logger.Errorf("[%s][%d/%d] failed to mark gc candidate deleting, skip: %s, %s", uid, localIndex, total, blob.Digest, blob.Status) continue } if count == 0 { - gc.logger.Warningf("[%s][%d/%d] no blob found to mark gc candidate deleting, ID:%d, digest:%s", uid, index, total, blob.ID, blob.Digest) + gc.logger.Warningf("[%s][%d/%d] no blob found to mark gc candidate deleting, ID:%d, digest:%s", uid, localIndex, total, blob.ID, blob.Digest) continue } @@ -339,7 +337,7 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { for _, art := range gc.trashedArts[blob.Digest] { // Harbor cannot know the existing tags in the backend from its database, so let the v2 DELETE manifest to remove all of them. gc.logger.Infof("[%s][%d/%d] delete the manifest with registry v2 API: %s, %s, %s", - uid, index, total, art.RepositoryName, blob.ContentType, blob.Digest) + uid, localIndex, total, art.RepositoryName, blob.ContentType, blob.Digest) if err := retry.Retry(func() error { return ignoreNotFound(func() error { err := v2DeleteManifest(art.RepositoryName, blob.Digest) @@ -350,13 +348,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { return err }) }, retry.Callback(func(err error, sleep time.Duration) { - gc.logger.Infof("[%s][%d/%d] failed to exec v2DeleteManifest, error: %v, will retry again after: %s", uid, index, total, err, sleep) + gc.logger.Infof("[%s][%d/%d] failed to exec v2DeleteManifest, error: %v, will retry again after: %s", uid, localIndex, total, err, sleep) })); err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to delete manifest with v2 API, %s, %s, %v", uid, index, total, art.RepositoryName, blob.Digest, err) + gc.logger.Errorf("[%s][%d/%d] failed to delete manifest with v2 API, %s, %s, %v", uid, localIndex, total, art.RepositoryName, blob.Digest, err) if err := ignoreNotFound(func() error { return gc.markDeleteFailed(ctx, blob) }); err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to call gc.markDeleteFailed() after v2DeleteManifest() error out: %s, %v", uid, index, total, blob.Digest, err) + gc.logger.Errorf("[%s][%d/%d] failed to call gc.markDeleteFailed() after v2DeleteManifest() error out: %s, %v", uid, localIndex, total, blob.Digest, err) return err } // if the system is set to read-only mode, return directly @@ -367,7 +365,7 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { continue } // for manifest, it has to delete the revisions folder of each repository - gc.logger.Infof("[%s][%d/%d] delete manifest from storage: %s", uid, index, total, blob.Digest) + gc.logger.Infof("[%s][%d/%d] delete manifest from storage: %s", uid, localIndex, total, blob.Digest) if err := retry.Retry(func() error { return ignoreNotFound(func() error { err := gc.registryCtlClient.DeleteManifest(art.RepositoryName, blob.Digest) @@ -378,13 +376,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { return err }) }, retry.Callback(func(err error, sleep time.Duration) { - gc.logger.Infof("[%s][%d/%d] failed to exec DeleteManifest, error: %v, will retry again after: %s", uid, index, total, err, sleep) + gc.logger.Infof("[%s][%d/%d] failed to exec DeleteManifest, error: %v, will retry again after: %s", uid, localIndex, total, err, sleep) })); err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to remove manifest from storage: %s, %s, errMsg=%v", uid, index, total, art.RepositoryName, blob.Digest, err) + gc.logger.Errorf("[%s][%d/%d] failed to remove manifest from storage: %s, %s, errMsg=%v", uid, localIndex, total, art.RepositoryName, blob.Digest, err) if err := ignoreNotFound(func() error { return gc.markDeleteFailed(ctx, blob) }); err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to call gc.markDeleteFailed() after gc.registryCtlClient.DeleteManifest() error out: %s, %s, %v", uid, index, total, art.RepositoryName, blob.Digest, err) + gc.logger.Errorf("[%s][%d/%d] failed to call gc.markDeleteFailed() after gc.registryCtlClient.DeleteManifest() error out: %s, %s, %v", uid, localIndex, total, art.RepositoryName, blob.Digest, err) return err } // if the system is set to read-only mode, return directly @@ -395,19 +393,19 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { continue } - gc.logger.Infof("[%s][%d/%d] delete artifact blob record from database: %d, %s, %s", uid, index, total, art.ID, art.RepositoryName, art.Digest) + gc.logger.Infof("[%s][%d/%d] delete artifact blob record from database: %d, %s, %s", uid, localIndex, total, art.ID, art.RepositoryName, art.Digest) if err := ignoreNotFound(func() error { return gc.blobMgr.CleanupAssociationsForArtifact(ctx.SystemContext(), art.Digest) }); err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to call gc.blobMgr.CleanupAssociationsForArtifact(): %v, errMsg=%v", uid, index, total, art.Digest, err) + gc.logger.Errorf("[%s][%d/%d] failed to call gc.blobMgr.CleanupAssociationsForArtifact(): %v, errMsg=%v", uid, localIndex, total, art.Digest, err) return err } - gc.logger.Infof("[%s][%d/%d] delete artifact trash record from database: %d, %s, %s", uid, index, total, art.ID, art.RepositoryName, art.Digest) + gc.logger.Infof("[%s][%d/%d] delete artifact trash record from database: %d, %s, %s", uid, localIndex, total, art.ID, art.RepositoryName, art.Digest) if err := ignoreNotFound(func() error { return gc.artrashMgr.Delete(ctx.SystemContext(), art.ID) }); err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to call gc.artrashMgr.Delete(): %v, errMsg=%v", uid, index, total, art.ID, err) + gc.logger.Errorf("[%s][%d/%d] failed to call gc.artrashMgr.Delete(): %v, errMsg=%v", uid, localIndex, total, art.ID, err) return err } } @@ -421,7 +419,7 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { // delete all the blobs, which include config, layer and manifest // for the foreign layer, as it's not stored in the storage, no need to call the delete api and count size, but still have to delete the DB record. if !blob.IsForeignLayer() { - gc.logger.Infof("[%s][%d/%d] delete blob from storage: %s", uid, index, total, blob.Digest) + gc.logger.Infof("[%s][%d/%d] delete blob from storage: %s", uid, localIndex, total, blob.Digest) if err := retry.Retry(func() error { return ignoreNotFound(func() error { err := gc.registryCtlClient.DeleteBlob(blob.Digest) @@ -432,13 +430,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { return err }) }, retry.Callback(func(err error, sleep time.Duration) { - gc.logger.Infof("[%s][%d/%d] failed to exec DeleteBlob, error: %v, will retry again after: %s", uid, index, total, err, sleep) + gc.logger.Infof("[%s][%d/%d] failed to exec DeleteBlob, error: %v, will retry again after: %s", uid, localIndex, total, err, sleep) })); err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to delete blob from storage: %s, %s, errMsg=%v", uid, index, total, blob.Digest, blob.Status, err) + gc.logger.Errorf("[%s][%d/%d] failed to delete blob from storage: %s, %s, errMsg=%v", uid, localIndex, total, blob.Digest, blob.Status, err) if err := ignoreNotFound(func() error { return gc.markDeleteFailed(ctx, blob) }); err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to call gc.markDeleteFailed() after gc.registryCtlClient.DeleteBlob() error out: %s, %v", uid, index, total, blob.Digest, err) + gc.logger.Errorf("[%s][%d/%d] failed to call gc.markDeleteFailed() after gc.registryCtlClient.DeleteBlob() error out: %s, %v", uid, localIndex, total, blob.Digest, err) return err } // if the system is set to read-only mode, return directly @@ -450,15 +448,15 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { atomic.AddInt64(&sweepSize, blob.Size) } - gc.logger.Infof("[%s][%d/%d] delete blob record from database: %d, %s", uid, index, total, blob.ID, blob.Digest) + gc.logger.Infof("[%s][%d/%d] delete blob record from database: %d, %s", uid, localIndex, total, blob.ID, blob.Digest) if err := ignoreNotFound(func() error { return gc.blobMgr.Delete(ctx.SystemContext(), blob.ID) }); err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to delete blob from database: %s, %s, errMsg=%v", uid, index, total, blob.Digest, blob.Status, err) + gc.logger.Errorf("[%s][%d/%d] failed to delete blob from database: %s, %s, errMsg=%v", uid, localIndex, total, blob.Digest, blob.Status, err) if err := ignoreNotFound(func() error { return gc.markDeleteFailed(ctx, blob) }); err != nil { - gc.logger.Errorf("[%s][%d/%d] failed to call gc.markDeleteFailed() after gc.blobMgr.Delete() error out, %d, %s %v", uid, index, total, blob.ID, blob.Digest, err) + gc.logger.Errorf("[%s][%d/%d] failed to call gc.markDeleteFailed() after gc.blobMgr.Delete() error out, %d, %s %v", uid, localIndex, total, blob.ID, blob.Digest, err) return err } return err diff --git a/src/jobservice/job/impl/gc/garbage_collection_test.go b/src/jobservice/job/impl/gc/garbage_collection_test.go index 124cff7f98..5aacd737e7 100644 --- a/src/jobservice/job/impl/gc/garbage_collection_test.go +++ b/src/jobservice/job/impl/gc/garbage_collection_test.go @@ -42,8 +42,8 @@ import ( type gcTestSuite struct { htesting.Suite artifactCtl *artifacttesting.Controller - artrashMgr *trashtesting.FakeManager - registryCtlClient *registryctl.Mockclient + artrashMgr *trashtesting.Manager + registryCtlClient *registryctl.Client projectCtl *projecttesting.Controller blobMgr *blob.Manager @@ -54,8 +54,8 @@ type gcTestSuite struct { func (suite *gcTestSuite) SetupTest() { suite.artifactCtl = &artifacttesting.Controller{} - suite.artrashMgr = &trashtesting.FakeManager{} - suite.registryCtlClient = ®istryctl.Mockclient{} + suite.artrashMgr = &trashtesting.Manager{} + suite.registryCtlClient = ®istryctl.Client{} suite.blobMgr = &blob.Manager{} suite.projectCtl = &projecttesting.Controller{} @@ -98,7 +98,7 @@ func (suite *gcTestSuite) TestDeletedArt() { }, }, nil) suite.artifactCtl.On("Delete").Return(nil) - suite.artrashMgr.On("Filter").Return([]model.ArtifactTrash{ + mock.OnAnything(suite.artrashMgr, "Filter").Return([]model.ArtifactTrash{ { ID: 1, Digest: suite.DigestString(), @@ -163,6 +163,8 @@ func (suite *gcTestSuite) TestInit() { "time_window": 1, "workers": float64(3), } + + mock.OnAnything(gc.registryCtlClient, "Health").Return(nil) suite.Nil(gc.init(ctx, params)) suite.True(gc.deleteUntagged) suite.Equal(3, gc.workers) @@ -230,7 +232,7 @@ func (suite *gcTestSuite) TestRun() { }, }, nil) suite.artifactCtl.On("Delete").Return(nil) - suite.artrashMgr.On("Filter").Return([]model.ArtifactTrash{}, nil) + mock.OnAnything(suite.artrashMgr, "Filter").Return([]model.ArtifactTrash{}, nil) mock.OnAnything(suite.projectCtl, "List").Return([]*proModels.Project{ { @@ -271,6 +273,8 @@ func (suite *gcTestSuite) TestRun() { mock.OnAnything(suite.blobMgr, "Delete").Return(nil) + mock.OnAnything(suite.registryCtlClient, "Health").Return(nil) + gc := &GarbageCollector{ artCtl: suite.artifactCtl, artrashMgr: suite.artrashMgr, @@ -284,6 +288,7 @@ func (suite *gcTestSuite) TestRun() { "workers": 3, } + mock.OnAnything(gc.registryCtlClient, "DeleteBlob").Return(nil) suite.Nil(gc.Run(ctx, params)) } @@ -302,7 +307,7 @@ func (suite *gcTestSuite) TestMark() { }, }, nil) suite.artifactCtl.On("Delete").Return(nil) - suite.artrashMgr.On("Filter").Return([]model.ArtifactTrash{ + mock.OnAnything(suite.artrashMgr, "Filter").Return([]model.ArtifactTrash{ { ID: 1, Digest: suite.DigestString(), @@ -381,6 +386,7 @@ func (suite *gcTestSuite) TestSweep() { workers: 3, } + mock.OnAnything(gc.registryCtlClient, "DeleteBlob").Return(nil) suite.Nil(gc.sweep(ctx)) } diff --git a/src/lib/config/metadata/metadatalist.go b/src/lib/config/metadata/metadatalist.go index dd2a7f67cd..aab4919fd8 100644 --- a/src/lib/config/metadata/metadatalist.go +++ b/src/lib/config/metadata/metadatalist.go @@ -96,6 +96,7 @@ var ( {Name: common.LDAPURL, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_URL", DefaultValue: "", ItemType: &NonEmptyStringType{}, Editable: false, Description: `The URL of LDAP server`}, {Name: common.LDAPVerifyCert, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_VERIFY_CERT", DefaultValue: "true", ItemType: &BoolType{}, Editable: false, Description: `Whether verify your OIDC server certificate, disable it if your OIDC server is hosted via self-hosted certificate.`}, {Name: common.LDAPGroupMembershipAttribute, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_MEMBERSHIP_ATTRIBUTE", DefaultValue: "memberof", ItemType: &StringType{}, Editable: true, Description: `The user attribute to identify the group membership`}, + {Name: common.LDAPGroupAttachParallel, Scope: UserScope, Group: LdapBasicGroup, EnvKey: "LDAP_GROUP_ATTACH_PARALLEL", DefaultValue: "false", ItemType: &BoolType{}, Editable: true, Description: `Attach LDAP group information to Harbor in parallel`}, {Name: common.MaxJobWorkers, Scope: SystemScope, Group: BasicGroup, EnvKey: "MAX_JOB_WORKERS", DefaultValue: "10", ItemType: &IntType{}, Editable: false}, {Name: common.ScanAllPolicy, Scope: UserScope, Group: BasicGroup, EnvKey: "", DefaultValue: "", ItemType: &MapType{}, Editable: false, Description: `The policy to scan images`}, diff --git a/src/lib/config/models/model.go b/src/lib/config/models/model.go index 51a9c0c2cb..25b41388ac 100644 --- a/src/lib/config/models/model.go +++ b/src/lib/config/models/model.go @@ -94,6 +94,7 @@ type GroupConf struct { SearchScope int `json:"ldap_group_search_scope"` AdminDN string `json:"ldap_group_admin_dn,omitempty"` MembershipAttribute string `json:"ldap_group_membership_attribute,omitempty"` + AttachParallel bool `json:"ldap_group_attach_parallel,omitempty"` } type GDPRSetting struct { diff --git a/src/lib/config/userconfig.go b/src/lib/config/userconfig.go index a0fd5f90ae..4012097c9e 100644 --- a/src/lib/config/userconfig.go +++ b/src/lib/config/userconfig.go @@ -81,6 +81,7 @@ func LDAPGroupConf(ctx context.Context) (*cfgModels.GroupConf, error) { SearchScope: mgr.Get(ctx, common.LDAPGroupSearchScope).GetInt(), AdminDN: mgr.Get(ctx, common.LDAPGroupAdminDn).GetString(), MembershipAttribute: mgr.Get(ctx, common.LDAPGroupMembershipAttribute).GetString(), + AttachParallel: mgr.Get(ctx, common.LDAPGroupAttachParallel).GetBool(), }, nil } diff --git a/src/controller/replication/transfer/iothrottler.go b/src/lib/iothrottler.go similarity index 98% rename from src/controller/replication/transfer/iothrottler.go rename to src/lib/iothrottler.go index 828c440357..b0853e0e58 100644 --- a/src/controller/replication/transfer/iothrottler.go +++ b/src/lib/iothrottler.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package transfer +package lib import ( "fmt" diff --git a/src/pkg/artifact/dao/dao.go b/src/pkg/artifact/dao/dao.go index 0e2e79a41c..c59baeb7f5 100644 --- a/src/pkg/artifact/dao/dao.go +++ b/src/pkg/artifact/dao/dao.go @@ -54,6 +54,8 @@ type DAO interface { DeleteReference(ctx context.Context, id int64) (err error) // DeleteReferences deletes the references referenced by the artifact specified by parent ID DeleteReferences(ctx context.Context, parentID int64) (err error) + // ListWithLatest ... + ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error) } const ( @@ -282,6 +284,53 @@ func (d *dao) DeleteReferences(ctx context.Context, parentID int64) error { return err } +func (d *dao) ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error) { + ormer, err := orm.FromContext(ctx) + if err != nil { + return nil, err + } + + sql := `SELECT a.* + FROM artifact a + JOIN ( + SELECT repository_name, MAX(push_time) AS latest_push_time + FROM artifact + WHERE project_id = ? and %s = ? + GROUP BY repository_name + ) latest ON a.repository_name = latest.repository_name AND a.push_time = latest.latest_push_time` + + queryParam := make([]interface{}, 0) + var ok bool + var pid interface{} + if pid, ok = query.Keywords["ProjectID"]; !ok { + return nil, errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage(`the value of "ProjectID" must be set`) + } + queryParam = append(queryParam, pid) + + var attributionValue interface{} + if attributionValue, ok = query.Keywords["media_type"]; ok { + sql = fmt.Sprintf(sql, "media_type") + } else if attributionValue, ok = query.Keywords["artifact_type"]; ok { + sql = fmt.Sprintf(sql, "artifact_type") + } + + if attributionValue == "" { + return nil, errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage(`the value of "media_type" or "artifact_type" must be set`) + } + queryParam = append(queryParam, attributionValue) + + sql, queryParam = orm.PaginationOnRawSQL(query, sql, queryParam) + arts := []*Artifact{} + _, err = ormer.Raw(sql, queryParam...).QueryRows(&arts) + if err != nil { + return nil, err + } + + return arts, nil +} + func querySetter(ctx context.Context, query *q.Query, options ...orm.Option) (beegoorm.QuerySeter, error) { qs, err := orm.QuerySetter(ctx, &Artifact{}, query, options...) if err != nil { diff --git a/src/pkg/artifact/dao/dao_test.go b/src/pkg/artifact/dao/dao_test.go index adefcfff72..08b448fba8 100644 --- a/src/pkg/artifact/dao/dao_test.go +++ b/src/pkg/artifact/dao/dao_test.go @@ -472,6 +472,75 @@ func (d *daoTestSuite) TestDeleteReferences() { d.True(errors.IsErr(err, errors.NotFoundCode)) } +func (d *daoTestSuite) TestListWithLatest() { + now := time.Now() + art := &Artifact{ + Type: "IMAGE", + MediaType: v1.MediaTypeImageConfig, + ManifestMediaType: v1.MediaTypeImageIndex, + ProjectID: 1234, + RepositoryID: 1234, + RepositoryName: "library2/hello-world1", + Digest: "digest", + PushTime: now, + PullTime: now, + Annotations: `{"anno1":"value1"}`, + } + id, err := d.dao.Create(d.ctx, art) + d.Require().Nil(err) + + time.Sleep(1 * time.Second) + now = time.Now() + + art2 := &Artifact{ + Type: "IMAGE", + MediaType: v1.MediaTypeImageConfig, + ManifestMediaType: v1.MediaTypeImageIndex, + ProjectID: 1234, + RepositoryID: 1235, + RepositoryName: "library2/hello-world2", + Digest: "digest", + PushTime: now, + PullTime: now, + Annotations: `{"anno1":"value1"}`, + } + id1, err := d.dao.Create(d.ctx, art2) + d.Require().Nil(err) + + time.Sleep(1 * time.Second) + now = time.Now() + + art3 := &Artifact{ + Type: "IMAGE", + MediaType: v1.MediaTypeImageConfig, + ManifestMediaType: v1.MediaTypeImageIndex, + ProjectID: 1234, + RepositoryID: 1235, + RepositoryName: "library2/hello-world2", + Digest: "digest2", + PushTime: now, + PullTime: now, + Annotations: `{"anno1":"value1"}`, + } + id2, err := d.dao.Create(d.ctx, art3) + d.Require().Nil(err) + + latest, err := d.dao.ListWithLatest(d.ctx, &q.Query{ + Keywords: map[string]interface{}{ + "ProjectID": 1234, + "media_type": v1.MediaTypeImageConfig, + }, + }) + + d.Require().Nil(err) + d.Require().Equal(2, len(latest)) + d.Equal("library2/hello-world1", latest[0].RepositoryName) + + defer d.dao.Delete(d.ctx, id) + defer d.dao.Delete(d.ctx, id1) + defer d.dao.Delete(d.ctx, id2) +} + func TestDaoTestSuite(t *testing.T) { suite.Run(t, &daoTestSuite{}) } diff --git a/src/pkg/artifact/manager.go b/src/pkg/artifact/manager.go index b28fd8b253..ec133c341b 100644 --- a/src/pkg/artifact/manager.go +++ b/src/pkg/artifact/manager.go @@ -48,6 +48,8 @@ type Manager interface { ListReferences(ctx context.Context, query *q.Query) (references []*Reference, err error) // DeleteReference specified by ID DeleteReference(ctx context.Context, id int64) (err error) + // ListWithLatest list the artifacts when the latest_in_repository in the query was set + ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error) } // NewManager returns an instance of the default manager @@ -147,6 +149,22 @@ func (m *manager) DeleteReference(ctx context.Context, id int64) error { return m.dao.DeleteReference(ctx, id) } +func (m *manager) ListWithLatest(ctx context.Context, query *q.Query) ([]*Artifact, error) { + arts, err := m.dao.ListWithLatest(ctx, query) + if err != nil { + return nil, err + } + var artifacts []*Artifact + for _, art := range arts { + artifact, err := m.assemble(ctx, art) + if err != nil { + return nil, err + } + artifacts = append(artifacts, artifact) + } + return artifacts, nil +} + // assemble the artifact with references populated func (m *manager) assemble(ctx context.Context, art *dao.Artifact) (*Artifact, error) { artifact := &Artifact{} diff --git a/src/pkg/artifact/manager_test.go b/src/pkg/artifact/manager_test.go index 9418705cfd..af434d7c22 100644 --- a/src/pkg/artifact/manager_test.go +++ b/src/pkg/artifact/manager_test.go @@ -80,6 +80,11 @@ func (f *fakeDao) DeleteReferences(ctx context.Context, parentID int64) error { return args.Error(0) } +func (f *fakeDao) ListWithLatest(ctx context.Context, query *q.Query) ([]*dao.Artifact, error) { + args := f.Called() + return args.Get(0).([]*dao.Artifact), args.Error(1) +} + type managerTestSuite struct { suite.Suite mgr *manager @@ -135,6 +140,28 @@ func (m *managerTestSuite) TestAssemble() { m.Equal(2, len(artifact.References)) } +func (m *managerTestSuite) TestListWithLatest() { + art := &dao.Artifact{ + ID: 1, + Type: "IMAGE", + MediaType: "application/vnd.oci.image.config.v1+json", + ManifestMediaType: "application/vnd.oci.image.manifest.v1+json", + ProjectID: 1, + RepositoryID: 1, + Digest: "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180", + Size: 1024, + PushTime: time.Now(), + PullTime: time.Now(), + ExtraAttrs: `{"attr1":"value1"}`, + Annotations: `{"anno1":"value1"}`, + } + m.dao.On("ListWithLatest", mock.Anything).Return([]*dao.Artifact{art}, nil) + artifacts, err := m.mgr.ListWithLatest(nil, nil) + m.Require().Nil(err) + m.Equal(1, len(artifacts)) + m.Equal(art.ID, artifacts[0].ID) +} + func (m *managerTestSuite) TestList() { art := &dao.Artifact{ ID: 1, diff --git a/src/pkg/cached/artifact/redis/manager.go b/src/pkg/cached/artifact/redis/manager.go index c5371a3a74..11879affde 100644 --- a/src/pkg/cached/artifact/redis/manager.go +++ b/src/pkg/cached/artifact/redis/manager.go @@ -65,6 +65,10 @@ func (m *Manager) List(ctx context.Context, query *q.Query) ([]*artifact.Artifac return m.delegator.List(ctx, query) } +func (m *Manager) ListWithLatest(ctx context.Context, query *q.Query) ([]*artifact.Artifact, error) { + return m.delegator.ListWithLatest(ctx, query) +} + func (m *Manager) Create(ctx context.Context, artifact *artifact.Artifact) (int64, error) { return m.delegator.Create(ctx, artifact) } diff --git a/src/pkg/project/models/pro_meta.go b/src/pkg/project/models/pro_meta.go index a8a0659fe8..25f7e41bee 100644 --- a/src/pkg/project/models/pro_meta.go +++ b/src/pkg/project/models/pro_meta.go @@ -24,4 +24,5 @@ const ( ProMetaAutoScan = "auto_scan" ProMetaReuseSysCVEAllowlist = "reuse_sys_cve_allowlist" ProMetaAutoSBOMGen = "auto_sbom_generation" + ProMetaProxySpeed = "proxy_speed_kb" ) diff --git a/src/pkg/project/models/project.go b/src/pkg/project/models/project.go index e533bd9112..80318aa21a 100644 --- a/src/pkg/project/models/project.go +++ b/src/pkg/project/models/project.go @@ -156,6 +156,19 @@ func (p *Project) AutoSBOMGen() bool { return isTrue(auto) } +// ProxyCacheSpeed ... +func (p *Project) ProxyCacheSpeed() int32 { + speed, exist := p.GetMetadata(ProMetaProxySpeed) + if !exist { + return 0 + } + speedInt, err := strconv.ParseInt(speed, 10, 32) + if err != nil { + return 0 + } + return int32(speedInt) +} + // FilterByPublic returns orm.QuerySeter with public filter func (p *Project) FilterByPublic(_ context.Context, qs orm.QuerySeter, _ string, value interface{}) orm.QuerySeter { subQuery := `SELECT project_id FROM project_metadata WHERE name = 'public' AND value = '%s'` diff --git a/src/pkg/reg/adapter/awsecr/adapter.go b/src/pkg/reg/adapter/awsecr/adapter.go index ca8de422e0..257d40e4a8 100644 --- a/src/pkg/reg/adapter/awsecr/adapter.go +++ b/src/pkg/reg/adapter/awsecr/adapter.go @@ -29,7 +29,7 @@ import ( ) const ( - ecrPattern = "https://(?:api|(\\d+)\\.dkr)\\.ecr\\.([\\w\\-]+)\\.amazonaws\\.com" + ecrPattern = "https://(?:api|(\\d+)\\.dkr)\\.ecr(\\-fips)?\\.([\\w\\-]+)\\.(amazonaws\\.com(\\.cn)?|sc2s\\.sgov\\.gov|c2s\\.ic\\.gov)" ) var ( @@ -64,10 +64,10 @@ func newAdapter(registry *model.Registry) (*adapter, error) { func parseAccountRegion(url string) (string, string, error) { rs := ecrRegexp.FindStringSubmatch(url) - if rs == nil { + if rs == nil || len(rs) < 4 { return "", "", errors.New("bad aws url") } - return rs[1], rs[2], nil + return rs[1], rs[3], nil } type factory struct { diff --git a/src/pkg/reg/adapter/awsecr/adapter_test.go b/src/pkg/reg/adapter/awsecr/adapter_test.go index e55b250a5b..8d910b840c 100644 --- a/src/pkg/reg/adapter/awsecr/adapter_test.go +++ b/src/pkg/reg/adapter/awsecr/adapter_test.go @@ -83,6 +83,38 @@ func TestAdapter_NewAdapter(t *testing.T) { assert.Nil(t, adapter) assert.NotNil(t, err) + adapter, err = newAdapter(&model.Registry{ + Type: model.RegistryTypeAwsEcr, + Credential: &model.Credential{ + AccessKey: "xxx", + AccessSecret: "ppp", + }, + URL: "https://123456.dkr.ecr-fips.test-region.amazonaws.com", + }) + assert.Nil(t, err) + assert.NotNil(t, adapter) + + adapter, err = newAdapter(&model.Registry{ + Type: model.RegistryTypeAwsEcr, + Credential: &model.Credential{ + AccessKey: "xxx", + AccessSecret: "ppp", + }, + URL: "https://123456.dkr.ecr.us-isob-east-1.sc2s.sgov.gov", + }) + assert.Nil(t, err) + assert.NotNil(t, adapter) + + adapter, err = newAdapter(&model.Registry{ + Type: model.RegistryTypeAwsEcr, + Credential: &model.Credential{ + AccessKey: "xxx", + AccessSecret: "ppp", + }, + URL: "https://123456.dkr.ecr.us-iso-east-1.c2s.ic.gov", + }) + assert.Nil(t, err) + assert.NotNil(t, adapter) } func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Server) { diff --git a/src/pkg/robot/dao/dao_test.go b/src/pkg/robot/dao/dao_test.go index 4723b64019..972c75514d 100644 --- a/src/pkg/robot/dao/dao_test.go +++ b/src/pkg/robot/dao/dao_test.go @@ -52,6 +52,7 @@ func (suite *DaoTestSuite) robots() { Description: "test3 description", ProjectID: 1, Secret: suite.RandString(10), + Creator: "tester", }) suite.Nil(err) @@ -120,6 +121,7 @@ func (suite *DaoTestSuite) TestGet() { r, err := suite.dao.Get(orm.Context(), suite.robotID3) suite.Nil(err) suite.Equal("test3", r.Name) + suite.Equal("tester", r.Creator) } func (suite *DaoTestSuite) TestCount() { diff --git a/src/pkg/robot/model/model.go b/src/pkg/robot/model/model.go index f35e309c62..a31cb0eeda 100644 --- a/src/pkg/robot/model/model.go +++ b/src/pkg/robot/model/model.go @@ -39,6 +39,7 @@ type Robot struct { ExpiresAt int64 `orm:"column(expiresat)" json:"expires_at"` Disabled bool `orm:"column(disabled)" json:"disabled"` Visible bool `orm:"column(visible)" json:"-"` + Creator string `orm:"column(creator)" json:"creator"` CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` } diff --git a/src/pkg/scan/vulnerability/vul_test.go b/src/pkg/scan/vulnerability/vul_test.go index 2d0dcbe983..be0eb50c8f 100644 --- a/src/pkg/scan/vulnerability/vul_test.go +++ b/src/pkg/scan/vulnerability/vul_test.go @@ -62,12 +62,12 @@ func TestPostScan(t *testing.T) { origRp := &scan.Report{} rawReport := "" - mocker := &postprocessorstesting.ScanReportV1ToV2Converter{} - mocker.On("ToRelationalSchema", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, "original report", nil) + mocker := &postprocessorstesting.NativeScanReportConverter{} + mocker.On("ToRelationalSchema", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("", "original report", nil) postprocessors.Converter = mocker sr := &v1.ScanRequest{Artifact: artifact} refreshedReport, err := v.PostScan(ctx, sr, origRp, rawReport, time.Now(), &model.Robot{}) - assert.Equal(t, "", refreshedReport, "PostScan should return the refreshed report") + assert.Equal(t, "original report", refreshedReport, "PostScan should return the refreshed report") assert.Nil(t, err, "PostScan should not return an error") } @@ -209,6 +209,7 @@ func (suite *VulHandlerTestSuite) TestMakeReportPlaceHolder() { mock.OnAnything(suite.reportMgr, "Create").Return("uuid", nil).Once() mock.OnAnything(suite.reportMgr, "Delete").Return(nil).Once() mock.OnAnything(suite.taskMgr, "ListScanTasksByReportUUID").Return([]*task.Task{{Status: "Success"}}, nil) + mock.OnAnything(suite.handler.reportConverter, "FromRelationalSchema").Return("", nil) rps, err := suite.handler.MakePlaceHolder(ctx, art, r) require.NoError(suite.T(), err) assert.Equal(suite.T(), 1, len(rps)) diff --git a/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html b/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html index b64ba7cf51..2b1401d1b5 100644 --- a/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html +++ b/src/portal/src/app/base/left-side-nav/config/auth/config-auth.component.html @@ -498,6 +498,37 @@ + + + + + +