diff --git a/VERSION b/VERSION index 1defe531b..a4b6ac3de 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.1.0 +v2.2.0 diff --git a/src/common/models/project.go b/src/common/models/project.go index c89b0aba5..3bf34a7a5 100644 --- a/src/common/models/project.go +++ b/src/common/models/project.go @@ -80,6 +80,11 @@ func (p *Project) IsPublic() bool { return isTrue(public) } +// IsProxy returns true when the project type is proxy cache +func (p *Project) IsProxy() bool { + return p.RegistryID > 0 +} + // ContentTrustEnabled ... func (p *Project) ContentTrustEnabled() bool { enabled, exist := p.GetMetadata(ProMetaEnableContentTrust) diff --git a/src/controller/p2p/preheat/controller.go b/src/controller/p2p/preheat/controller.go index acd752c63..bad90f5dd 100644 --- a/src/controller/p2p/preheat/controller.go +++ b/src/controller/p2p/preheat/controller.go @@ -117,6 +117,8 @@ type Controller interface { ListPoliciesByProject(ctx context.Context, project int64, query *q.Query) ([]*policyModels.Schema, error) // CheckHealth checks the instance health, for test connection CheckHealth(ctx context.Context, instance *providerModels.Instance) error + // DeletePoliciesOfProject delete all policies under one project + DeletePoliciesOfProject(ctx context.Context, project int64) error } var _ Controller = (*controller)(nil) @@ -419,6 +421,21 @@ func (c *controller) DeletePolicy(ctx context.Context, id int64) error { return c.pManager.Delete(ctx, id) } +// DeletePoliciesOfProject deletes all the policy under project. +func (c *controller) DeletePoliciesOfProject(ctx context.Context, project int64) error { + policies, err := c.ListPoliciesByProject(ctx, project, nil) + if err != nil { + return err + } + + for _, p := range policies { + if err = c.DeletePolicy(ctx, p.ID); err != nil { + return err + } + } + return nil +} + // deleteExecs delete executions func (c *controller) deleteExecs(ctx context.Context, vendorID int64) error { executions, err := c.executionMgr.List(ctx, &q.Query{ diff --git a/src/controller/p2p/preheat/controllor_test.go b/src/controller/p2p/preheat/controllor_test.go index e1c290822..a9649c909 100644 --- a/src/controller/p2p/preheat/controllor_test.go +++ b/src/controller/p2p/preheat/controllor_test.go @@ -335,6 +335,22 @@ func (s *preheatSuite) TestListPoliciesByProject() { s.NotNil(p) } +func (s *preheatSuite) TestDeletePoliciesOfProject() { + fakePolicies := []*policy.Schema{ + {ID: 1000, Name: "1-should-delete", ProjectID: 10}, + {ID: 1001, Name: "2-should-delete", ProjectID: 10}, + } + s.fakePolicyMgr.On("ListPoliciesByProject", s.ctx, int64(10), mock.Anything).Return(fakePolicies, nil) + for _, p := range fakePolicies { + s.fakePolicyMgr.On("Get", s.ctx, p.ID).Return(p, nil) + s.fakePolicyMgr.On("Delete", s.ctx, p.ID).Return(nil) + s.fakeExecutionMgr.On("List", s.ctx, &q.Query{Keywords: map[string]interface{}{"VendorID": p.ID, "VendorType": "P2P_PREHEAT"}}).Return([]*taskModel.Execution{}, nil) + } + + err := s.controller.DeletePoliciesOfProject(s.ctx, 10) + s.NoError(err) +} + func (s *preheatSuite) TestCheckHealth() { // if instance is nil var instance *providerModel.Instance diff --git a/src/core/api/base.go b/src/core/api/base.go index 3c22aaa38..f29d3d322 100644 --- a/src/core/api/base.go +++ b/src/core/api/base.go @@ -35,11 +35,11 @@ import ( "github.com/goharbor/harbor/src/pkg/repository" "github.com/goharbor/harbor/src/pkg/retention" "github.com/goharbor/harbor/src/pkg/scheduler" + sec "github.com/goharbor/harbor/src/server/middleware/security" ) const ( yamlFileContentType = "application/x-yaml" - userSessionKey = "user" ) // the managers/controllers used globally @@ -176,7 +176,7 @@ func (b *BaseController) WriteYamlData(object interface{}) { // PopulateUserSession generates a new session ID and fill the user model in parm to the session func (b *BaseController) PopulateUserSession(u models.User) { b.SessionRegenerateID() - b.SetSession(userSessionKey, u) + b.SetSession(sec.UserIDSessionKey, u.UserID) } // Init related objects/configurations for the API controllers diff --git a/src/server/middleware/contenttrust/contenttrust.go b/src/server/middleware/contenttrust/contenttrust.go index 70792e844..152a89728 100644 --- a/src/server/middleware/contenttrust/contenttrust.go +++ b/src/server/middleware/contenttrust/contenttrust.go @@ -21,6 +21,9 @@ var ( if err != nil { return false, err } + if len(art.Tag) > 0 { + return checker.IsTagSigned(art.Tag, art.Digest), nil + } return checker.IsArtifactSigned(art.Digest), nil } ) diff --git a/src/server/middleware/repoproxy/proxy.go b/src/server/middleware/repoproxy/proxy.go index f147d58d5..dc7266d7d 100644 --- a/src/server/middleware/repoproxy/proxy.go +++ b/src/server/middleware/repoproxy/proxy.go @@ -17,20 +17,20 @@ package repoproxy import ( "context" "fmt" - "github.com/goharbor/harbor/src/common/security" - "github.com/goharbor/harbor/src/common/security/proxycachesecret" - "github.com/goharbor/harbor/src/lib/errors" - httpLib "github.com/goharbor/harbor/src/lib/http" - "github.com/goharbor/harbor/src/replication/model" - "github.com/goharbor/harbor/src/replication/registry" "io" "net/http" "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/security" + "github.com/goharbor/harbor/src/common/security/proxycachesecret" "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/controller/proxy" "github.com/goharbor/harbor/src/lib" + "github.com/goharbor/harbor/src/lib/errors" + httpLib "github.com/goharbor/harbor/src/lib/http" "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/replication/model" + "github.com/goharbor/harbor/src/replication/registry" "github.com/goharbor/harbor/src/server/middleware" ) @@ -163,14 +163,6 @@ func setHeaders(w http.ResponseWriter, size int64, mediaType string, dig string) h.Set("Etag", dig) } -// isProxyProject check the project is a proxy project -func isProxyProject(p *models.Project) bool { - if p == nil { - return false - } - return p.RegistryID > 0 -} - // isProxySession check if current security context is proxy session func isProxySession(ctx context.Context) bool { sc, ok := security.FromContext(ctx) @@ -194,7 +186,7 @@ func DisableBlobAndManifestUploadMiddleware() func(http.Handler) http.Handler { httpLib.SendError(w, err) return } - if isProxyProject(p) && !isProxySession(ctx) { + if p.IsProxy() && !isProxySession(ctx) { httpLib.SendError(w, errors.DeniedError( errors.Errorf("can not push artifact to a proxy project: %v", p.Name))) diff --git a/src/server/middleware/repoproxy/proxy_test.go b/src/server/middleware/repoproxy/proxy_test.go index e75b79012..7c83d7669 100644 --- a/src/server/middleware/repoproxy/proxy_test.go +++ b/src/server/middleware/repoproxy/proxy_test.go @@ -18,44 +18,12 @@ import ( "context" "testing" - "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/security/proxycachesecret" securitySecret "github.com/goharbor/harbor/src/common/security/secret" "github.com/goharbor/harbor/src/core/config" ) -func TestIsProxyProject(t *testing.T) { - cases := []struct { - name string - in *models.Project - want bool - }{ - { - name: `no proxy`, - in: &models.Project{RegistryID: 0}, - want: false, - }, - { - name: `normal proxy`, - in: &models.Project{RegistryID: 1}, - want: true, - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - - got := isProxyProject(tt.in) - - if got != tt.want { - t.Errorf(`(%v) = %v; want "%v"`, tt.in, got, tt.want) - } - - }) - } -} - func TestIsProxySession(t *testing.T) { config.Init() sc1 := securitySecret.NewSecurityContext("123456789", config.SecretStore) diff --git a/src/server/middleware/security/auth_proxy_test.go b/src/server/middleware/security/auth_proxy_test.go index 32b4ed068..863fd5260 100644 --- a/src/server/middleware/security/auth_proxy_test.go +++ b/src/server/middleware/security/auth_proxy_test.go @@ -17,25 +17,23 @@ package security import ( "encoding/json" "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/models" - "github.com/goharbor/harbor/src/common/utils/test" _ "github.com/goharbor/harbor/src/core/auth/authproxy" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/lib" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "io/ioutil" "k8s.io/api/authentication/v1beta1" - "net/http" - "net/http/httptest" - "net/url" - "testing" ) func TestAuthProxy(t *testing.T) { - config.Init() - test.InitDatabaseFromEnv() authProxy := &authProxy{} server, err := newAuthProxyTestServer() @@ -49,7 +47,7 @@ func TestAuthProxy(t *testing.T) { common.HTTPAuthProxyTokenReviewEndpoint: server.URL, common.AUTHMode: common.HTTPAuth, } - config.Upload(c) + config.InitWithSettings(c) v, e := config.HTTPAuthProxySetting() require.Nil(t, e) assert.Equal(t, *v, models.HTTPAuthProxy{ diff --git a/src/server/middleware/security/basic_auth_test.go b/src/server/middleware/security/basic_auth_test.go index dd3e23e97..8c50ace7f 100644 --- a/src/server/middleware/security/basic_auth_test.go +++ b/src/server/middleware/security/basic_auth_test.go @@ -15,18 +15,39 @@ package security import ( + "fmt" + + "github.com/goharbor/harbor/src/common" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" _ "github.com/goharbor/harbor/src/core/auth/db" + "github.com/goharbor/harbor/src/core/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "net/http" "testing" ) func TestBasicAuth(t *testing.T) { + c := map[string]interface{}{ + common.AUTHMode: common.DBAuth, + } + config.InitWithSettings(c) + + user := models.User{ + Username: "tester", + Password: "Harbor12345", + } + uid, err := dao.Register(user) + defer func(id int64) { + sql := fmt.Sprintf("DELETE FROM harbor_user WHERE user_id=%d", id) + dao.ExecuteBatchSQL([]string{sql}) + }(uid) basicAuth := &basicAuth{} req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1/api/projects/", nil) require.Nil(t, err) - req.SetBasicAuth("admin", "Harbor12345") + req.SetBasicAuth("tester", "Harbor12345") ctx := basicAuth.Generate(req) assert.NotNil(t, ctx) } diff --git a/src/server/middleware/security/security_test.go b/src/server/middleware/security/security_test.go index 5f2a1f5ed..64a892891 100644 --- a/src/server/middleware/security/security_test.go +++ b/src/server/middleware/security/security_test.go @@ -15,13 +15,21 @@ package security import ( + "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/security" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "net/http" + "os" "testing" ) +func TestMain(m *testing.M) { + dao.PrepareTestForPostgresSQL() + os.Exit(m.Run()) +} + func TestSecurity(t *testing.T) { var ctx security.Context var exist bool diff --git a/src/server/middleware/security/session.go b/src/server/middleware/security/session.go index 5f6f8467d..637cf4b64 100644 --- a/src/server/middleware/security/session.go +++ b/src/server/middleware/security/session.go @@ -19,6 +19,7 @@ import ( "net/http/httptest" "github.com/astaxie/beego" + "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/security/local" @@ -26,6 +27,9 @@ import ( "github.com/goharbor/harbor/src/lib/log" ) +// UserIDSessionKey is the key for storing user id in session +const UserIDSessionKey = "user_id" + type session struct{} func (s *session) Generate(req *http.Request) security.Context { @@ -35,15 +39,24 @@ func (s *session) Generate(req *http.Request) security.Context { log.Errorf("failed to get the session store for request: %v", err) return nil } - userInterface := store.Get("user") + userInterface := store.Get(UserIDSessionKey) if userInterface == nil { return nil } - user, ok := userInterface.(models.User) + uid, ok := userInterface.(int) if !ok { - log.Warning("can not convert the user in session to user model") + log.Warning("can not convert the data in session to user id") + return nil + } + user, err := dao.GetUser(models.User{UserID: uid}) + if err != nil { + log.Errorf("failed to get user info, error: %v, skip generating security context via session", err) + return nil + } + if user == nil { + log.Errorf("the user id from session: %d, not exist in DB", uid) return nil } log.Debugf("a session security context generated for request %s %s", req.Method, req.URL.Path) - return local.NewSecurityContext(&user, config.GlobalProjectMgr) + return local.NewSecurityContext(user, config.GlobalProjectMgr) } diff --git a/src/server/middleware/security/session_test.go b/src/server/middleware/security/session_test.go index 2ca3c3918..1a5dead2b 100644 --- a/src/server/middleware/security/session_test.go +++ b/src/server/middleware/security/session_test.go @@ -15,11 +15,15 @@ package security import ( + "fmt" + "github.com/astaxie/beego" beegosession "github.com/astaxie/beego/session" + "github.com/goharbor/harbor/src/common/dao" "github.com/goharbor/harbor/src/common/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "net/http" "net/http/httptest" "path/filepath" @@ -41,20 +45,31 @@ func TestSession(t *testing.T) { beego.GlobalSessions, err = beegosession.NewManager("memory", conf) require.Nil(t, err) - user := models.User{ - Username: "admin", - UserID: 1, - Email: "admin@example.com", + user := &models.User{ + Username: "session_test", + Email: "session_test@example.com", SysAdminFlag: true, } + err = dao.OnBoardUser(user) + require.Nil(t, err) + defer func(id int) { + sql := fmt.Sprintf("DELETE FROM harbor_user WHERE user_id=%d", id) + dao.ExecuteBatchSQL([]string{sql}) + }(user.UserID) + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1/api/projects/", nil) require.Nil(t, err) store, err := beego.GlobalSessions.SessionStart(httptest.NewRecorder(), req) require.Nil(t, err) - err = store.Set("user", user) + err = store.Set(UserIDSessionKey, 999) require.Nil(t, err) session := &session{} ctx := session.Generate(req) + assert.Nil(t, ctx) + err = store.Set(UserIDSessionKey, user.UserID) + require.Nil(t, err) + ctx = session.Generate(req) assert.NotNil(t, ctx) + } diff --git a/src/server/v2.0/handler/artifact.go b/src/server/v2.0/handler/artifact.go index 53a56f279..265b5d528 100644 --- a/src/server/v2.0/handler/artifact.go +++ b/src/server/v2.0/handler/artifact.go @@ -17,9 +17,6 @@ package handler import ( "context" "fmt" - "github.com/goharbor/harbor/src/controller/event/metadata" - "github.com/goharbor/harbor/src/controller/project" - "github.com/goharbor/harbor/src/pkg/notification" "net/http" "strings" "time" @@ -31,10 +28,13 @@ import ( "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/controller/artifact" "github.com/goharbor/harbor/src/controller/artifact/processor" + "github.com/goharbor/harbor/src/controller/event/metadata" + "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/controller/repository" "github.com/goharbor/harbor/src/controller/scan" "github.com/goharbor/harbor/src/controller/tag" "github.com/goharbor/harbor/src/lib/errors" + "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/server/v2.0/handler/assembler" "github.com/goharbor/harbor/src/server/v2.0/handler/model" "github.com/goharbor/harbor/src/server/v2.0/models" @@ -244,7 +244,7 @@ func (a *artifactAPI) requireNonProxyCacheProject(ctx context.Context, name stri if err != nil { return err } - if pro.RegistryID > 0 { + if pro.IsProxy() { return errors.New(nil).WithCode(errors.MethodNotAllowedCode). WithMessage("the operation isn't supported for a proxy cache project") } diff --git a/src/server/v2.0/handler/project.go b/src/server/v2.0/handler/project.go index c8c1dc258..225a0f2b6 100644 --- a/src/server/v2.0/handler/project.go +++ b/src/server/v2.0/handler/project.go @@ -3,7 +3,6 @@ package handler import ( "context" "fmt" - "github.com/goharbor/harbor/src/pkg/robot" "strconv" "strings" "sync" @@ -15,6 +14,7 @@ import ( "github.com/goharbor/harbor/src/common/rbac" "github.com/goharbor/harbor/src/common/security" "github.com/goharbor/harbor/src/common/security/local" + "github.com/goharbor/harbor/src/controller/p2p/preheat" "github.com/goharbor/harbor/src/controller/project" "github.com/goharbor/harbor/src/controller/quota" "github.com/goharbor/harbor/src/controller/repository" @@ -29,6 +29,7 @@ import ( "github.com/goharbor/harbor/src/pkg/project/metadata" "github.com/goharbor/harbor/src/pkg/quota/types" "github.com/goharbor/harbor/src/pkg/retention/policy" + "github.com/goharbor/harbor/src/pkg/robot" "github.com/goharbor/harbor/src/pkg/user" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/server/v2.0/handler/model" @@ -48,6 +49,7 @@ func newProjectAPI() *projectAPI { projectCtl: project.Ctl, quotaCtl: quota.Ctl, robotMgr: robot.Mgr, + preheatCtl: preheat.Ctl, } } @@ -60,6 +62,7 @@ type projectAPI struct { projectCtl project.Controller quotaCtl quota.Controller robotMgr robot.Manager + preheatCtl preheat.Controller } func (a *projectAPI) CreateProject(ctx context.Context, params operation.CreateProjectParams) middleware.Responder { @@ -116,6 +119,12 @@ func (a *projectAPI) CreateProject(ctx context.Context, params operation.CreateP req.Metadata.Public = strconv.FormatBool(false) } + // ignore enable_content_trust metadata for proxy cache project + // see https://github.com/goharbor/harbor/issues/12940 to get more info + if req.RegistryID != nil { + req.Metadata.EnableContentTrust = nil + } + // validate the RegistryID and StorageLimit in the body of the request if err := a.validateProjectReq(ctx, req); err != nil { return a.SendError(ctx, err) @@ -212,6 +221,11 @@ func (a *projectAPI) DeleteProject(ctx context.Context, params operation.DeleteP } } + // preheat policies under the project should be deleted after deleting the project + if err = a.preheatCtl.DeletePoliciesOfProject(ctx, params.ProjectID); err != nil { + return a.SendError(ctx, err) + } + return operation.NewDeleteProjectOK() } @@ -306,7 +320,7 @@ func (a *projectAPI) GetProjectSummary(ctx context.Context, params operation.Get fetchSummaries = append(fetchSummaries, getProjectMemberSummary) } - if p.RegistryID > 0 { + if p.IsProxy() { fetchSummaries = append(fetchSummaries, getProjectRegistrySummary) } @@ -450,6 +464,11 @@ func (a *projectAPI) UpdateProject(ctx context.Context, params operation.UpdateP } } + // ignore enable_content_trust metadata for proxy cache project + // see https://github.com/goharbor/harbor/issues/12940 to get more info + if params.Project.Metadata != nil && p.IsProxy() { + params.Project.Metadata.EnableContentTrust = nil + } lib.JSONCopy(&p.Metadata, params.Project.Metadata) if err := a.projectCtl.Update(ctx, p); err != nil { diff --git a/tests/ci/api_common_install.sh b/tests/ci/api_common_install.sh index 02c4851c5..a54c8c156 100755 --- a/tests/ci/api_common_install.sh +++ b/tests/ci/api_common_install.sh @@ -25,7 +25,10 @@ sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && su sudo make swagger_client #TODO: Swagger python package used to installed into dist-packages, but it's changed into site-packages all in a sudden, we havn't found the root cause. # so current workround is to copy swagger packages from site-packages to dist-packages. -sudo cp -r /usr/lib/python3.7/site-packages/* /usr/local/lib/python3.7/dist-packages +package_dir=/usr/lib/python3.7/site-packages +if [ -d $package_dir ] && [ $(find $package_dir -type f -name "*client*.egg" | wc -l) -gt 0 ];then + sudo cp -rf ${package_dir}/* /usr/local/lib/python3.7/dist-packages +fi if [ $GITHUB_TOKEN ]; then