Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2025-01-11 10:27:58 +01:00
Code Issues Projects Releases Wiki Activity

update codes per review comments

Browse Source 
Signed-off-by: wang yan <wangyan@vmware.com>

fix middlewares per review comments
1, add scheme1 and scheme2 check
2, change MustCompile to Compile

Signed-off-by: wang yan <wangyan@vmware.com>
This commit is contained in:
wang yan 2019-07-05 13:23:17 +08:00
parent b3c5137a2f
commit 7b38389898
17 changed files with 1120 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
make/migrations/postgresql/0010_1.9.0_schema.up.sql
View File

@ -7,4 +7,26 @@ CREATE TABLE cve_whitelist (
expires_at bigint,
items text NOT NULL,
UNIQUE (project_id)
);
);
/* add quota table */
CREATE TABLE quota (
id SERIAL PRIMARY KEY NOT NULL,
reference VARCHAR(255) NOT NULL,
reference_id VARCHAR(255) NOT NULL,
hard JSONB NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
UNIQUE(reference, reference_id)
);
/* add quota usage table */
CREATE TABLE quota_usage (
id SERIAL PRIMARY KEY NOT NULL,
reference VARCHAR(255) NOT NULL,
reference_id VARCHAR(255) NOT NULL,
used JSONB NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
UNIQUE(reference, reference_id)
);

144
src/common/dao/quota.go Normal file
View File

@ -0,0 +1,144 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"fmt"
"strings"
"time"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/models"
)
var (
quotaOrderMap = map[string]string{
"id": "id asc",
"+id": "id asc",
"-id": "id desc",
"creation_time": "creation_time asc",
"+creation_time": "creation_time asc",
"-creation_time": "creation_time desc",
"update_time": "update_time asc",
"+update_time": "update_time asc",
"-update_time": "update_time desc",
}
)
// AddQuota add quota to the database.
func AddQuota(quota models.Quota) (int64, error) {
now := time.Now()
quota.CreationTime = now
quota.UpdateTime = now
return GetOrmer().Insert(&quota)
}
// GetQuota returns quota by id.
func GetQuota(id int64) (*models.Quota, error) {
q := models.Quota{ID: id}
err := GetOrmer().Read(&q, "ID")
if err == orm.ErrNoRows {
return nil, nil
}
return &q, err
}
// UpdateQuota update the quota.
func UpdateQuota(quota models.Quota) error {
quota.UpdateTime = time.Now()
_, err := GetOrmer().Update(&quota)
return err
}
// ListQuotas returns quotas by query.
func ListQuotas(query ...*models.QuotaQuery) ([]*models.Quota, error) {
condition, params := quotaQueryConditions(query...)
sql := fmt.Sprintf(`select * %s`, condition)
orderBy := quotaOrderBy(query...)
if orderBy != "" {
sql += ` order by ` + orderBy
}
if len(query) > 0 && query[0] != nil {
page, size := query[0].Page, query[0].Size
if size > 0 {
sql += ` limit ?`
params = append(params, size)
if page > 0 {
sql += ` offset ?`
params = append(params, size*(page-1))
}
}
}
var quotas []*models.Quota
if _, err := GetOrmer().Raw(sql, params).QueryRows(&quotas); err != nil {
return nil, err
}
return quotas, nil
}
func quotaQueryConditions(query ...*models.QuotaQuery) (string, []interface{}) {
params := []interface{}{}
sql := `from quota `
if len(query) == 0 || query[0] == nil {
return sql, params
}
sql += `where 1=1 `
q := query[0]
if q.Reference != "" {
sql += `and reference = ? `
params = append(params, q.Reference)
}
if q.ReferenceID != "" {
sql += `and reference_id = ? `
params = append(params, q.ReferenceID)
}
if len(q.ReferenceIDs) != 0 {
sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs)))
params = append(params, q.ReferenceIDs)
}
return sql, params
}
func quotaOrderBy(query ...*models.QuotaQuery) string {
orderBy := ""
if len(query) > 0 && query[0] != nil && query[0].Sort != "" {
if val, ok := quotaOrderMap[query[0].Sort]; ok {
orderBy = val
} else {
sort := query[0].Sort
order := "asc"
if sort[0] == '-' {
order = "desc"
sort = sort[1:]
}
prefix := "hard."
if strings.HasPrefix(sort, prefix) {
orderBy = fmt.Sprintf("hard->>'%s' %s", strings.TrimPrefix(sort, prefix), order)
}
}
}
return orderBy
}

135
src/common/dao/quota_test.go Normal file
View File

@ -0,0 +1,135 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"testing"
"time"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/suite"
)
var (
quotaReference = "project"
quotaUserReference = "user"
quotaHard = models.QuotaHard{"storage": 1024}
quotaHardLarger = models.QuotaHard{"storage": 2048}
)
type QuotaDaoSuite struct {
suite.Suite
}
func (suite *QuotaDaoSuite) equalHard(quota1 *models.Quota, quota2 *models.Quota) {
hard1, err := quota1.GetHard()
suite.Nil(err, "hard1 invalid")
hard2, err := quota2.GetHard()
suite.Nil(err, "hard2 invalid")
suite.Equal(hard1, hard2)
}
func (suite *QuotaDaoSuite) TearDownTest() {
ClearTable("quota")
}
func (suite *QuotaDaoSuite) TestAddQuota() {
_, err1 := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()})
suite.Nil(err1)
// Will failed for reference and reference_id should unique in db
_, err2 := AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()})
suite.Error(err2)
_, err3 := AddQuota(models.Quota{Reference: quotaUserReference, ReferenceID: "1", Hard: quotaHard.String()})
suite.Nil(err3)
}
func (suite *QuotaDaoSuite) TestGetQuota() {
quota1 := models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}
id, err := AddQuota(quota1)
suite.Nil(err)
// Get the new added quota
quota2, err := GetQuota(id)
suite.Nil(err)
suite.NotNil(quota2)
// Get the quota which id is 10000 not found
quota3, err := GetQuota(10000)
suite.Nil(err)
suite.Nil(quota3)
}
func (suite *QuotaDaoSuite) TestUpdateQuota() {
quota1 := models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()}
id, err := AddQuota(quota1)
suite.Nil(err)
// Get the new added quota
quota2, err := GetQuota(id)
suite.Nil(err)
suite.equalHard(&quota1, quota2)
// Update the quota
quota2.SetHard(quotaHardLarger)
time.Sleep(time.Millisecond * 10) // Ensure that UpdateTime changed
suite.Nil(UpdateQuota(*quota2))
// Get the updated quota
quota3, err := GetQuota(id)
suite.Nil(err)
suite.equalHard(quota2, quota3)
suite.NotEqual(quota2.UpdateTime, quota3.UpdateTime)
}
func (suite *QuotaDaoSuite) TestListQuotas() {
AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "1", Hard: quotaHard.String()})
AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "2", Hard: quotaHard.String()})
AddQuota(models.Quota{Reference: quotaReference, ReferenceID: "3", Hard: quotaHard.String()})
AddQuota(models.Quota{Reference: quotaUserReference, ReferenceID: "1", Hard: quotaHardLarger.String()})
// List all the quotas
quotas, err := ListQuotas()
suite.Nil(err)
suite.Equal(4, len(quotas))
suite.Equal(quotaReference, quotas[0].Reference)
// List quotas filter by reference
quotas, err = ListQuotas(&models.QuotaQuery{Reference: quotaReference})
suite.Nil(err)
suite.Equal(3, len(quotas))
// List quotas filter by reference ids
quotas, err = ListQuotas(&models.QuotaQuery{Reference: quotaReference, ReferenceIDs: []string{"1", "2"}})
suite.Nil(err)
suite.Equal(2, len(quotas))
// List quotas by pagination
quotas, err = ListQuotas(&models.QuotaQuery{Pagination: models.Pagination{Size: 2}})
suite.Nil(err)
suite.Equal(2, len(quotas))
// List quotas by sorting
quotas, err = ListQuotas(&models.QuotaQuery{Sorting: models.Sorting{Sort: "-hard.storage"}})
suite.Nil(err)
suite.Equal(quotaUserReference, quotas[0].Reference)
}
func TestRunQuotaDaoSuite(t *testing.T) {
suite.Run(t, new(QuotaDaoSuite))
}

144
src/common/dao/quota_usage.go Normal file
View File

@ -0,0 +1,144 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"fmt"
"strings"
"time"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/models"
)
var (
quotaUsageOrderMap = map[string]string{
"id": "id asc",
"+id": "id asc",
"-id": "id desc",
"creation_time": "creation_time asc",
"+creation_time": "creation_time asc",
"-creation_time": "creation_time desc",
"update_time": "update_time asc",
"+update_time": "update_time asc",
"-update_time": "update_time desc",
}
)
// AddQuotaUsage add quota usage to the database.
func AddQuotaUsage(quotaUsage models.QuotaUsage) (int64, error) {
now := time.Now()
quotaUsage.CreationTime = now
quotaUsage.UpdateTime = now
return GetOrmer().Insert(&quotaUsage)
}
// GetQuotaUsage returns quota usage by id.
func GetQuotaUsage(id int64) (*models.QuotaUsage, error) {
q := models.QuotaUsage{ID: id}
err := GetOrmer().Read(&q, "ID")
if err == orm.ErrNoRows {
return nil, nil
}
return &q, err
}
// UpdateQuotaUsage update the quota usage.
func UpdateQuotaUsage(quotaUsage models.QuotaUsage) error {
quotaUsage.UpdateTime = time.Now()
_, err := GetOrmer().Update(&quotaUsage)
return err
}
// ListQuotaUsages returns quota usages by query.
func ListQuotaUsages(query ...*models.QuotaUsageQuery) ([]*models.QuotaUsage, error) {
condition, params := quotaUsageQueryConditions(query...)
sql := fmt.Sprintf(`select * %s`, condition)
orderBy := quotaUsageOrderBy(query...)
if orderBy != "" {
sql += ` order by ` + orderBy
}
if len(query) > 0 && query[0] != nil {
page, size := query[0].Page, query[0].Size
if size > 0 {
sql += ` limit ?`
params = append(params, size)
if page > 0 {
sql += ` offset ?`
params = append(params, size*(page-1))
}
}
}
var quotaUsages []*models.QuotaUsage
if _, err := GetOrmer().Raw(sql, params).QueryRows(&quotaUsages); err != nil {
return nil, err
}
return quotaUsages, nil
}
func quotaUsageQueryConditions(query ...*models.QuotaUsageQuery) (string, []interface{}) {
params := []interface{}{}
sql := `from quota_usage `
if len(query) == 0 || query[0] == nil {
return sql, params
}
sql += `where 1=1 `
q := query[0]
if q.Reference != "" {
sql += `and reference = ? `
params = append(params, q.Reference)
}
if q.ReferenceID != "" {
sql += `and reference_id = ? `
params = append(params, q.ReferenceID)
}
if len(q.ReferenceIDs) != 0 {
sql += fmt.Sprintf(`and reference_id in (%s) `, paramPlaceholder(len(q.ReferenceIDs)))
params = append(params, q.ReferenceIDs)
}
return sql, params
}
func quotaUsageOrderBy(query ...*models.QuotaUsageQuery) string {
orderBy := ""
if len(query) > 0 && query[0] != nil && query[0].Sort != "" {
if val, ok := quotaUsageOrderMap[query[0].Sort]; ok {
orderBy = val
} else {
sort := query[0].Sort
order := "asc"
if sort[0] == '-' {
order = "desc"
sort = sort[1:]
}
prefix := "used."
if strings.HasPrefix(sort, prefix) {
orderBy = fmt.Sprintf("used->>'%s' %s", strings.TrimPrefix(sort, prefix), order)
}
}
}
return orderBy
}

135
src/common/dao/quota_usage_test.go Normal file
View File

@ -0,0 +1,135 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"testing"
"time"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/suite"
)
var (
quotaUsageReference = "project"
quotaUsageUserReference = "user"
quotaUsageUsed = models.QuotaUsed{"storage": 1024}
quotaUsageUsedLarger = models.QuotaUsed{"storage": 2048}
)
type QuotaUsageDaoSuite struct {
suite.Suite
}
func (suite *QuotaUsageDaoSuite) equalUsed(usage1 *models.QuotaUsage, usage2 *models.QuotaUsage) {
used1, err := usage1.GetUsed()
suite.Nil(err, "used1 invalid")
used2, err := usage2.GetUsed()
suite.Nil(err, "used2 invalid")
suite.Equal(used1, used2)
}
func (suite *QuotaUsageDaoSuite) TearDownTest() {
ClearTable("quota_usage")
}
func (suite *QuotaUsageDaoSuite) TestAddQuotaUsage() {
_, err1 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
suite.Nil(err1)
// Will failed for reference and reference_id should unique in db
_, err2 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
suite.Error(err2)
_, err3 := AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageUserReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
suite.Nil(err3)
}
func (suite *QuotaUsageDaoSuite) TestGetQuotaUsage() {
quotaUsage1 := models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}
id, err := AddQuotaUsage(quotaUsage1)
suite.Nil(err)
// Get the new added quotaUsage
quotaUsage2, err := GetQuotaUsage(id)
suite.Nil(err)
suite.NotNil(quotaUsage2)
// Get the quotaUsage which id is 10000 not found
quotaUsage3, err := GetQuotaUsage(10000)
suite.Nil(err)
suite.Nil(quotaUsage3)
}
func (suite *QuotaUsageDaoSuite) TestUpdateQuotaUsage() {
quotaUsage1 := models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()}
id, err := AddQuotaUsage(quotaUsage1)
suite.Nil(err)
// Get the new added quotaUsage
quotaUsage2, err := GetQuotaUsage(id)
suite.Nil(err)
suite.equalUsed(&quotaUsage1, quotaUsage2)
// Update the quotaUsage
quotaUsage2.SetUsed(quotaUsageUsedLarger)
time.Sleep(time.Millisecond * 10) // Ensure that UpdateTime changed
suite.Nil(UpdateQuotaUsage(*quotaUsage2))
// Get the updated quotaUsage
quotaUsage3, err := GetQuotaUsage(id)
suite.Nil(err)
suite.equalUsed(quotaUsage2, quotaUsage3)
suite.NotEqual(quotaUsage2.UpdateTime, quotaUsage3.UpdateTime)
}
func (suite *QuotaUsageDaoSuite) TestListQuotaUsages() {
AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "1", Used: quotaUsageUsed.String()})
AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "2", Used: quotaUsageUsed.String()})
AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageReference, ReferenceID: "3", Used: quotaUsageUsed.String()})
AddQuotaUsage(models.QuotaUsage{Reference: quotaUsageUserReference, ReferenceID: "1", Used: quotaUsageUsedLarger.String()})
// List all the quotaUsages
quotaUsages, err := ListQuotaUsages()
suite.Nil(err)
suite.Equal(4, len(quotaUsages))
suite.Equal(quotaUsageReference, quotaUsages[0].Reference)
// List quotaUsages filter by reference
quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Reference: quotaUsageReference})
suite.Nil(err)
suite.Equal(3, len(quotaUsages))
// List quotaUsages filter by reference ids
quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Reference: quotaUsageReference, ReferenceIDs: []string{"1", "2"}})
suite.Nil(err)
suite.Equal(2, len(quotaUsages))
// List quotaUsages by pagination
quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Pagination: models.Pagination{Size: 2}})
suite.Nil(err)
suite.Equal(2, len(quotaUsages))
// List quotaUsages by sorting
quotaUsages, err = ListQuotaUsages(&models.QuotaUsageQuery{Sorting: models.Sorting{Sort: "-used.storage"}})
suite.Nil(err)
suite.Equal(quotaUsageUserReference, quotaUsages[0].Reference)
}
func TestRunQuotaUsageDaoSuite(t *testing.T) {
suite.Run(t, new(QuotaUsageDaoSuite))
}

5
src/common/models/base.go
View File

@ -37,5 +37,8 @@ func init() {
new(JobLog),
new(Robot),
new(OIDCUser),
new(CVEWhitelist))
new(CVEWhitelist),
new(Quota),
new(QuotaUsage),
)
}

77
src/common/models/quota.go Normal file
View File

@ -0,0 +1,77 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"encoding/json"
"time"
)
// QuotaHard a map for the quota hard
type QuotaHard map[string]int64
func (h QuotaHard) String() string {
bytes, _ := json.Marshal(h)
return string(bytes)
}
// Copy returns copied quota hard
func (h QuotaHard) Copy() QuotaHard {
hard := QuotaHard{}
for key, value := range h {
hard[key] = value
}
return hard
}
// Quota model for quota
type Quota struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Reference string `orm:"column(reference)" json:"reference"`
ReferenceID string `orm:"column(reference_id)" json:"reference_id"`
Hard string `orm:"column(hard);type(jsonb)" json:"-"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// TableName returns table name for orm
func (q *Quota) TableName() string {
return "quota"
}
// GetHard returns quota hard
func (q *Quota) GetHard() (QuotaHard, error) {
var hard QuotaHard
if err := json.Unmarshal([]byte(q.Hard), &hard); err != nil {
return nil, err
}
return hard, nil
}
// SetHard set new quota hard
func (q *Quota) SetHard(hard QuotaHard) {
q.Hard = hard.String()
}
// QuotaQuery query parameters for quota
type QuotaQuery struct {
Reference string
ReferenceID string
ReferenceIDs []string
Pagination
Sorting
}

77
src/common/models/quota_usage.go Normal file
View File

@ -0,0 +1,77 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"encoding/json"
"time"
)
// QuotaUsed a map for the quota used
type QuotaUsed map[string]int64
func (u QuotaUsed) String() string {
bytes, _ := json.Marshal(u)
return string(bytes)
}
// Copy returns copied quota used
func (u QuotaUsed) Copy() QuotaUsed {
used := QuotaUsed{}
for key, value := range u {
used[key] = value
}
return used
}
// QuotaUsage model for quota usage
type QuotaUsage struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Reference string `orm:"column(reference)" json:"reference"`
ReferenceID string `orm:"column(reference_id)" json:"reference_id"`
Used string `orm:"column(used);type(jsonb)" json:"-"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// TableName returns table name for orm
func (qu *QuotaUsage) TableName() string {
return "quota_usage"
}
// GetUsed returns quota used
func (qu *QuotaUsage) GetUsed() (QuotaUsed, error) {
var used QuotaUsed
if err := json.Unmarshal([]byte(qu.Used), &used); err != nil {
return nil, err
}
return used, nil
}
// SetUsed set quota used
func (qu *QuotaUsage) SetUsed(used QuotaUsed) {
qu.Used = used.String()
}
// QuotaUsageQuery query parameters for quota
type QuotaUsageQuery struct {
Reference string
ReferenceID string
ReferenceIDs []string
Pagination
Sorting
}

5
src/core/controllers/controllers_test.go
View File

@ -32,7 +32,7 @@ import (
"github.com/goharbor/harbor/src/common/models"
utilstest "github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/review/harbor/src/core/proxy"
"github.com/goharbor/harbor/src/core/middlewares"
"github.com/stretchr/testify/assert"
)
@ -102,8 +102,9 @@ func TestRedirectForOIDC(t *testing.T) {
// TestMain is a sample to run an endpoint test
func TestAll(t *testing.T) {
config.InitWithSettings(utilstest.GetUnitTestConfig())
proxy.Init()
assert := assert.New(t)
err := middlewares.Init()
assert.Nil(err)
r, _ := http.NewRequest("POST", "/c/login", nil)
w := httptest.NewRecorder()

1
src/core/controllers/proxy.go
View File

@ -21,4 +21,3 @@ func (p *RegistryProxy) Handle() {
func (p *RegistryProxy) Render() error {
return nil
}

78
src/core/middlewares/chain.go
View File

@ -46,7 +46,7 @@ func (b *DefaultCreator) Create() *alice.Chain {
for _, mName := range b.middlewares {
middlewareName := mName
chain = chain.Append(func(next http.Handler) http.Handler {
constructor := b.getMiddleware(middlewareName)
constructor := b.geMiddleware(middlewareName)
if constructor == nil {
log.Errorf("cannot init middle %s", middlewareName)
return nil
@ -57,70 +57,16 @@ func (b *DefaultCreator) Create() *alice.Chain {
return &chain
}
func (b *DefaultCreator) getMiddleware(mName string) alice.Constructor {
var middleware alice.Constructor
if mName == READONLY {
middleware = func(next http.Handler) http.Handler {
return readonly.New(next)
}
func (b *DefaultCreator) geMiddleware(mName string) alice.Constructor {
middlewares := map[string]alice.Constructor{
READONLY: func(next http.Handler) http.Handler { return readonly.New(next) },
URL: func(next http.Handler) http.Handler { return url.New(next) },
MUITIPLEMANIFEST: func(next http.Handler) http.Handler { return multiplmanifest.New(next) },
LISTREPO: func(next http.Handler) http.Handler { return listrepo.New(next) },
CONTENTTRUST: func(next http.Handler) http.Handler { return contenttrust.New(next) },
VULNERABLE: func(next http.Handler) http.Handler { return vulnerable.New(next) },
REGQUOTA: func(next http.Handler) http.Handler { return regquota.New(next) },
BLOBQUOTA: func(next http.Handler) http.Handler { return blobquota.New(next) },
}
if mName == URL {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return url.New(next)
}
}
if mName == MUITIPLEMANIFEST {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return multiplmanifest.New(next)
}
}
if mName == LISTREPO {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return listrepo.New(next)
}
}
if mName == CONTENTTRUST {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return contenttrust.New(next)
}
}
if mName == VULNERABLE {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return vulnerable.New(next)
}
}
if mName == REGQUOTA {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return regquota.New(next)
}
}
if mName == BLOBQUOTA {
if middleware != nil {
return nil
}
middleware = func(next http.Handler) http.Handler {
return blobquota.New(next)
}
}
return middleware
return middlewares[mName]
}

69
src/core/middlewares/contenttrust/handler_test.go Normal file
View File

@ -0,0 +1,69 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package contenttrust
import (
"github.com/goharbor/harbor/src/common"
notarytest "github.com/goharbor/harbor/src/common/utils/notary/test"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/middlewares/util"
"github.com/stretchr/testify/assert"
"net/http/httptest"
"os"
"testing"
)
var endpoint = "10.117.4.142"
var notaryServer *httptest.Server
var admiralEndpoint = "http://127.0.0.1:8282"
var token = ""
func TestMain(m *testing.M) {
notaryServer = notarytest.NewNotaryServer(endpoint)
defer notaryServer.Close()
NotaryEndpoint = notaryServer.URL
var defaultConfig = map[string]interface{}{
common.ExtEndpoint: "https://" + endpoint,
common.WithNotary: true,
common.TokenExpiration: 30,
}
config.InitWithSettings(defaultConfig)
result := m.Run()
if result != 0 {
os.Exit(result)
}
}
func TestMatchNotaryDigest(t *testing.T) {
assert := assert.New(t)
// The data from common/utils/notary/helper_test.go
img1 := util.ImageInfo{Repository: "notary-demo/busybox", Reference: "1.0", ProjectName: "notary-demo", Digest: "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"}
img2 := util.ImageInfo{Repository: "notary-demo/busybox", Reference: "2.0", ProjectName: "notary-demo", Digest: "sha256:12345678"}
res1, err := matchNotaryDigest(img1)
assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1)
assert.True(res1)
res2, err := matchNotaryDigest(img2)
assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2)
assert.False(res2)
}
func TestIsDigest(t *testing.T) {
assert := assert.New(t)
assert.False(isDigest("latest"))
assert.True(isDigest("sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"))
}

37
src/core/middlewares/listrepo/handler_test.go Normal file
View File

@ -0,0 +1,37 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package listrepo
import (
"github.com/stretchr/testify/assert"
"net/http"
"testing"
)
func TestMatchListRepos(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/_catalog", nil)
res1 := matchListRepos(req1)
assert.False(res1, "%s %v is not a request to list repos", req1.Method, req1.URL)
req2, _ := http.NewRequest("GET", "http://127.0.0.1:5000/v2/_catalog", nil)
res2 := matchListRepos(req2)
assert.True(res2, "%s %v is a request to list repos", req2.Method, req2.URL)
req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/_catalog", nil)
res3 := matchListRepos(req3)
assert.False(res3, "%s %v is not a request to pull manifest", req3.Method, req3.URL)
}

2
src/core/middlewares/multiplmanifest/handler.go
View File

@ -34,7 +34,7 @@ func New(next http.Handler) http.Handler {
// ServeHTTP The handler is responsible for blocking request to upload manifest list by docker client, which is not supported so far by Harbor.
func (mh multipleManifestHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
match, _, _ := util.MatchManifestURL(req)
match, _, _ := util.MatchPushManifest(req)
if match {
contentType := req.Header.Get("Content-type")
// application/vnd.docker.distribution.manifest.list.v2+json

40
src/core/middlewares/regquota/handler.go
View File

@ -18,6 +18,8 @@ import (
"bytes"
"fmt"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/middlewares/util"
"io/ioutil"
@ -42,25 +44,29 @@ func (rqh regQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request)
var mfSize int64
var mfDigest string
mediaType := req.Header.Get("Content-Type")
if req.Method == http.MethodPut && mediaType == "application/vnd.docker.distribution.manifest.v2+json" {
data, err := ioutil.ReadAll(req.Body)
if err != nil {
log.Warningf("Error occurred when to copy manifest body %v", err)
http.Error(rw, util.MarshalError("InternalServerError", fmt.Sprintf("Error occurred when to decode manifest body %v", err)), http.StatusInternalServerError)
return
}
req.Body = ioutil.NopCloser(bytes.NewBuffer(data))
if req.Method == http.MethodPut {
if mediaType == schema1.MediaTypeManifest ||
mediaType == schema1.MediaTypeSignedManifest ||
mediaType == schema2.MediaTypeManifest {
data, err := ioutil.ReadAll(req.Body)
if err != nil {
log.Warningf("Error occurred when to copy manifest body %v", err)
http.Error(rw, util.MarshalError("InternalServerError", fmt.Sprintf("Error occurred when to decode manifest body %v", err)), http.StatusInternalServerError)
return
}
req.Body = ioutil.NopCloser(bytes.NewBuffer(data))
_, desc, err := distribution.UnmarshalManifest(mediaType, data)
if err != nil {
log.Warningf("Error occurred when to Unmarshal Manifest %v", err)
http.Error(rw, util.MarshalError("InternalServerError", fmt.Sprintf("Error occurred when to Unmarshal Manifest %v", err)), http.StatusInternalServerError)
return
_, desc, err := distribution.UnmarshalManifest(mediaType, data)
if err != nil {
log.Warningf("Error occurred when to Unmarshal Manifest %v", err)
http.Error(rw, util.MarshalError("InternalServerError", fmt.Sprintf("Error occurred when to Unmarshal Manifest %v", err)), http.StatusInternalServerError)
return
}
mfDigest = desc.Digest.String()
mfSize = desc.Size
log.Infof("manifest digest... %s", mfDigest)
log.Infof("manifest size... %v", mfSize)
}
mfDigest = desc.Digest.String()
mfSize = desc.Size
log.Infof("manifest digest... %s", mfDigest)
log.Infof("manifest size... %v", mfSize)
}
}

20
src/core/middlewares/util/util.go
View File

@ -74,7 +74,11 @@ func MarshalError(code, msg string) string {
// MatchManifestURL ...
func MatchManifestURL(req *http.Request) (bool, string, string) {
re := regexp.MustCompile(manifestURLPattern)
re, err := regexp.Compile(manifestURLPattern)
if err != nil {
log.Errorf("error to match manifest url, %v", err)
return false, "", ""
}
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 3 {
s[1] = strings.TrimSuffix(s[1], "/")
@ -88,7 +92,11 @@ func MatchPutBlobURL(req *http.Request) (bool, string) {
if req.Method != http.MethodPut {
return false, ""
}
re := regexp.MustCompile(blobURLPattern)
re, err := regexp.Compile(blobURLPattern)
if err != nil {
log.Errorf("error to match put blob url, %v", err)
return false, ""
}
s := re.FindStringSubmatch(req.URL.Path)
if len(s) == 2 {
s[1] = strings.TrimSuffix(s[1], "/")
@ -105,6 +113,14 @@ func MatchPullManifest(req *http.Request) (bool, string, string) {
return MatchManifestURL(req)
}
// MatchPushManifest checks if the request looks like a request to push manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
func MatchPushManifest(req *http.Request) (bool, string, string) {
if req.Method != http.MethodPut {
return false, "", ""
}
return MatchManifestURL(req)
}
// CopyResp ...
func CopyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) {
for k, v := range rec.Header() {

218
src/core/middlewares/util/util_test.go Normal file
View File

@ -0,0 +1,218 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/models"
notarytest "github.com/goharbor/harbor/src/common/utils/notary/test"
testutils "github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/core/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"net/http/httptest"
"os"
"testing"
)
var endpoint = "10.117.4.142"
var notaryServer *httptest.Server
var admiralEndpoint = "http://127.0.0.1:8282"
var token = ""
func TestMain(m *testing.M) {
notaryServer = notarytest.NewNotaryServer(endpoint)
defer notaryServer.Close()
var defaultConfig = map[string]interface{}{
common.ExtEndpoint: "https://" + endpoint,
common.WithNotary: true,
common.TokenExpiration: 30,
}
config.InitWithSettings(defaultConfig)
result := m.Run()
if result != 0 {
os.Exit(result)
}
}
func TestMatchPullManifest(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil)
res1, _, _ := MatchPullManifest(req1)
assert.False(res1, "%s %v is not a request to pull manifest", req1.Method, req1.URL)
req2, _ := http.NewRequest("GET", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil)
res2, repo2, tag2 := MatchPullManifest(req2)
assert.True(res2, "%s %v is a request to pull manifest", req2.Method, req2.URL)
assert.Equal("library/ubuntu", repo2)
assert.Equal("14.04", tag2)
req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/library/ubuntu/manifests/14.04", nil)
res3, _, _ := MatchPullManifest(req3)
assert.False(res3, "%s %v is not a request to pull manifest", req3.Method, req3.URL)
req4, _ := http.NewRequest("GET", "https://192.168.0.5/v2/library/ubuntu/manifests/14.04", nil)
res4, repo4, tag4 := MatchPullManifest(req4)
assert.True(res4, "%s %v is a request to pull manifest", req4.Method, req4.URL)
assert.Equal("library/ubuntu", repo4)
assert.Equal("14.04", tag4)
req5, _ := http.NewRequest("GET", "https://myregistry.com/v2/path1/path2/golang/manifests/1.6.2", nil)
res5, repo5, tag5 := MatchPullManifest(req5)
assert.True(res5, "%s %v is a request to pull manifest", req5.Method, req5.URL)
assert.Equal("path1/path2/golang", repo5)
assert.Equal("1.6.2", tag5)
req6, _ := http.NewRequest("GET", "https://myregistry.com/v2/myproject/registry/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil)
res6, repo6, tag6 := MatchPullManifest(req6)
assert.True(res6, "%s %v is a request to pull manifest", req6.Method, req6.URL)
assert.Equal("myproject/registry", repo6)
assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag6)
req7, _ := http.NewRequest("GET", "https://myregistry.com/v2/myproject/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil)
res7, repo7, tag7 := MatchPullManifest(req7)
assert.True(res7, "%s %v is a request to pull manifest", req7.Method, req7.URL)
assert.Equal("myproject", repo7)
assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7)
}
func TestMatchPutBlob(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/ubuntu/blobs/uploads/67bb4d9b-4dab-4bbe-b726-2e39322b8303?_state=7W3kWkgdr3fTW", nil)
res1, repo1 := MatchPutBlobURL(req1)
assert.True(res1, "%s %v is not a request to put blob", req1.Method, req1.URL)
assert.Equal("library/ubuntu", repo1)
req2, _ := http.NewRequest("PATCH", "http://127.0.0.1:5000/v2/library/blobs/uploads/67bb4d9b-4dab-4bbe-b726-2e39322b8303?_state=7W3kWkgdr3fTW", nil)
res2, _ := MatchPutBlobURL(req2)
assert.False(res2, "%s %v is a request to put blob", req2.Method, req2.URL)
req3, _ := http.NewRequest("PUT", "http://127.0.0.1:5000/v2/library/manifest/67bb4d9b-4dab-4bbe-b726-2e39322b8303?_state=7W3kWkgdr3fTW", nil)
res3, _ := MatchPutBlobURL(req3)
assert.False(res3, "%s %v is not a request to put blob", req3.Method, req3.URL)
}
func TestMatchPushManifest(t *testing.T) {
assert := assert.New(t)
req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil)
res1, _, _ := MatchPushManifest(req1)
assert.False(res1, "%s %v is not a request to push manifest", req1.Method, req1.URL)
req2, _ := http.NewRequest("PUT", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil)
res2, repo2, tag2 := MatchPushManifest(req2)
assert.True(res2, "%s %v is a request to push manifest", req2.Method, req2.URL)
assert.Equal("library/ubuntu", repo2)
assert.Equal("14.04", tag2)
req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/library/ubuntu/manifests/14.04", nil)
res3, _, _ := MatchPushManifest(req3)
assert.False(res3, "%s %v is not a request to push manifest", req3.Method, req3.URL)
req4, _ := http.NewRequest("PUT", "https://192.168.0.5/v2/library/ubuntu/manifests/14.04", nil)
res4, repo4, tag4 := MatchPushManifest(req4)
assert.True(res4, "%s %v is a request to push manifest", req4.Method, req4.URL)
assert.Equal("library/ubuntu", repo4)
assert.Equal("14.04", tag4)
req5, _ := http.NewRequest("PUT", "https://myregistry.com/v2/path1/path2/golang/manifests/1.6.2", nil)
res5, repo5, tag5 := MatchPushManifest(req5)
assert.True(res5, "%s %v is a request to push manifest", req5.Method, req5.URL)
assert.Equal("path1/path2/golang", repo5)
assert.Equal("1.6.2", tag5)
req6, _ := http.NewRequest("PUT", "https://myregistry.com/v2/myproject/registry/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil)
res6, repo6, tag6 := MatchPushManifest(req6)
assert.True(res6, "%s %v is a request to push manifest", req6.Method, req6.URL)
assert.Equal("myproject/registry", repo6)
assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag6)
req7, _ := http.NewRequest("PUT", "https://myregistry.com/v2/myproject/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil)
res7, repo7, tag7 := MatchPushManifest(req7)
assert.True(res7, "%s %v is a request to push manifest", req7.Method, req7.URL)
assert.Equal("myproject", repo7)
assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7)
req8, _ := http.NewRequest("PUT", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil)
res8, repo8, tag8 := MatchPushManifest(req8)
assert.True(res8, "%s %v is a request to push manifest", req8.Method, req8.URL)
assert.Equal("library/ubuntu", repo8)
assert.Equal("14.04", tag8)
}
func TestPMSPolicyChecker(t *testing.T) {
var defaultConfigAdmiral = map[string]interface{}{
common.ExtEndpoint: "https://" + endpoint,
common.WithNotary: true,
common.TokenExpiration: 30,
common.DatabaseType: "postgresql",
common.PostGreSQLHOST: "127.0.0.1",
common.PostGreSQLPort: 5432,
common.PostGreSQLUsername: "postgres",
common.PostGreSQLPassword: "root123",
common.PostGreSQLDatabase: "registry",
}
if err := config.Init(); err != nil {
panic(err)
}
testutils.InitDatabaseFromEnv()
config.Upload(defaultConfigAdmiral)
name := "project_for_test_get_sev_low"
id, err := config.GlobalProjectMgr.Create(&models.Project{
Name: name,
OwnerID: 1,
Metadata: map[string]string{
models.ProMetaEnableContentTrust: "true",
models.ProMetaPreventVul: "true",
models.ProMetaSeverity: "low",
},
})
require.Nil(t, err)
defer func(id int64) {
if err := config.GlobalProjectMgr.Delete(id); err != nil {
t.Logf("failed to delete project %d: %v", id, err)
}
}(id)
contentTrustFlag := GetPolicyChecker().ContentTrustEnabled("project_for_test_get_sev_low")
assert.True(t, contentTrustFlag)
projectVulnerableEnabled, projectVulnerableSeverity := GetPolicyChecker().VulnerablePolicy("project_for_test_get_sev_low")
assert.True(t, projectVulnerableEnabled)
assert.Equal(t, projectVulnerableSeverity, models.SevLow)
}
func TestCopyResp(t *testing.T) {
assert := assert.New(t)
rec1 := httptest.NewRecorder()
rec2 := httptest.NewRecorder()
rec1.Header().Set("X-Test", "mytest")
rec1.WriteHeader(418)
CopyResp(rec1, rec2)
assert.Equal(418, rec2.Result().StatusCode)
assert.Equal("mytest", rec2.Header().Get("X-Test"))
}
func TestMarshalError(t *testing.T) {
assert := assert.New(t)
js1 := MarshalError("PROJECT_POLICY_VIOLATION", "Not Found")
assert.Equal("{\"errors\":[{\"code\":\"PROJECT_POLICY_VIOLATION\",\"message\":\"Not Found\",\"detail\":\"Not Found\"}]}", js1)
js2 := MarshalError("DENIED", "The action is denied")
assert.Equal("{\"errors\":[{\"code\":\"DENIED\",\"message\":\"The action is denied\",\"detail\":\"The action is denied\"}]}", js2)
}