mirror of
https://github.com/goharbor/harbor.git
synced 2024-09-27 21:12:42 +02:00
feat(helm-chart,quota): count quota support for helm chart (#8439)
* feat(helm-chart,quota): count quota support for helm chart Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
parent
270f9ea213
commit
8cc9314984
@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
|
||||
hlog "github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/justinas/alice"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -42,7 +43,7 @@ type Controller struct {
|
||||
}
|
||||
|
||||
// NewController is constructor of the chartserver.Controller
|
||||
func NewController(backendServer *url.URL) (*Controller, error) {
|
||||
func NewController(backendServer *url.URL, chains ...*alice.Chain) (*Controller, error) {
|
||||
if backendServer == nil {
|
||||
return nil, errors.New("failed to create chartserver.Controller: backend sever address is required")
|
||||
}
|
||||
@ -68,7 +69,7 @@ func NewController(backendServer *url.URL) (*Controller, error) {
|
||||
return &Controller{
|
||||
backendServerAddress: backendServer,
|
||||
// Use customized reverse proxy
|
||||
trafficProxy: NewProxyEngine(backendServer, cred),
|
||||
trafficProxy: NewProxyEngine(backendServer, cred, chains...),
|
||||
// Initialize chart operator for use
|
||||
chartOperator: &ChartOperator{},
|
||||
// Create http client with customized timeouts
|
||||
|
@ -3,10 +3,13 @@ package chartserver
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/ghodss/yaml"
|
||||
commonhttp "github.com/goharbor/harbor/src/common/http"
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/replication"
|
||||
rep_event "github.com/goharbor/harbor/src/replication/event"
|
||||
@ -66,12 +69,22 @@ func (c *Controller) DeleteChartVersion(namespace, chartName, version string) er
|
||||
return errors.New("invalid chart for deleting")
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/%s/%s", c.APIPrefix(namespace), chartName, version)
|
||||
url := fmt.Sprintf("/api/chartrepo/%s/charts/%s/%s", namespace, chartName, version)
|
||||
req, _ := http.NewRequest(http.MethodDelete, url, nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
err := c.apiClient.DeleteContent(url)
|
||||
c.trafficProxy.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
text, err := extractError(w.Body.Bytes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return &commonhttp.Error{
|
||||
Code: w.Code,
|
||||
Message: text,
|
||||
}
|
||||
}
|
||||
|
||||
// send notification to replication handler
|
||||
// Todo: it used as the replacement of webhook, will be removed when webhook to be introduced.
|
||||
|
@ -17,6 +17,7 @@ import (
|
||||
hlog "github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/replication"
|
||||
rep_event "github.com/goharbor/harbor/src/replication/event"
|
||||
"github.com/justinas/alice"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -36,20 +37,29 @@ type ProxyEngine struct {
|
||||
backend *url.URL
|
||||
|
||||
// Use go reverse proxy as engine
|
||||
engine *httputil.ReverseProxy
|
||||
engine http.Handler
|
||||
}
|
||||
|
||||
// NewProxyEngine is constructor of NewProxyEngine
|
||||
func NewProxyEngine(target *url.URL, cred *Credential) *ProxyEngine {
|
||||
return &ProxyEngine{
|
||||
backend: target,
|
||||
engine: &httputil.ReverseProxy{
|
||||
func NewProxyEngine(target *url.URL, cred *Credential, chains ...*alice.Chain) *ProxyEngine {
|
||||
var engine http.Handler
|
||||
|
||||
engine = &httputil.ReverseProxy{
|
||||
ErrorLog: log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile),
|
||||
Director: func(req *http.Request) {
|
||||
director(target, cred, req)
|
||||
},
|
||||
ModifyResponse: modifyResponse,
|
||||
},
|
||||
}
|
||||
|
||||
if len(chains) > 0 {
|
||||
hlog.Info("New chart server traffic proxy with middlewares")
|
||||
engine = chains[0].Then(engine)
|
||||
}
|
||||
|
||||
return &ProxyEngine{
|
||||
backend: target,
|
||||
engine: engine,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -16,9 +16,16 @@ package redis
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/garyburd/redigo/redis"
|
||||
"github.com/goharbor/harbor/src/common/utils"
|
||||
"time"
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -34,9 +41,6 @@ else
|
||||
return 0
|
||||
end
|
||||
`
|
||||
defaultDelay = 5 * time.Second
|
||||
defaultMaxRetry = 5
|
||||
defaultExpiry = 600 * time.Second
|
||||
)
|
||||
|
||||
// Mutex ...
|
||||
@ -103,12 +107,126 @@ type Options struct {
|
||||
maxRetry int
|
||||
}
|
||||
|
||||
var (
|
||||
opt *Options
|
||||
optOnce sync.Once
|
||||
|
||||
defaultDelay = int64(1) // 1 second
|
||||
defaultMaxRetry = 600
|
||||
defaultExpire = int64(2 * time.Hour / time.Second) // 2 hours
|
||||
)
|
||||
|
||||
// DefaultOptions ...
|
||||
func DefaultOptions() *Options {
|
||||
opt := &Options{
|
||||
retryDelay: defaultDelay,
|
||||
expiry: defaultExpiry,
|
||||
maxRetry: defaultMaxRetry,
|
||||
optOnce.Do(func() {
|
||||
retryDelay, err := strconv.ParseInt(os.Getenv("REDIS_LOCK_RETRY_DELAY"), 10, 64)
|
||||
if err != nil || retryDelay < 0 {
|
||||
retryDelay = defaultDelay
|
||||
}
|
||||
|
||||
maxRetry, err := strconv.Atoi(os.Getenv("REDIS_LOCK_MAX_RETRY"))
|
||||
if err != nil || maxRetry < 0 {
|
||||
maxRetry = defaultMaxRetry
|
||||
}
|
||||
|
||||
expire, err := strconv.ParseInt(os.Getenv("REDIS_LOCK_EXPIRE"), 10, 64)
|
||||
if err != nil || expire < 0 {
|
||||
expire = defaultExpire
|
||||
}
|
||||
|
||||
opt = &Options{
|
||||
retryDelay: time.Duration(retryDelay) * time.Second,
|
||||
expiry: time.Duration(expire) * time.Second,
|
||||
maxRetry: maxRetry,
|
||||
}
|
||||
})
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
var (
|
||||
pool *redis.Pool
|
||||
poolOnce sync.Once
|
||||
|
||||
poolMaxIdle = 200
|
||||
poolMaxActive = 1000
|
||||
poolIdleTimeout int64 = 180
|
||||
)
|
||||
|
||||
// DefaultPool return default redis pool
|
||||
func DefaultPool() *redis.Pool {
|
||||
poolOnce.Do(func() {
|
||||
maxIdle, err := strconv.Atoi(os.Getenv("REDIS_POOL_MAX_IDLE"))
|
||||
if err != nil || maxIdle < 0 {
|
||||
maxIdle = poolMaxIdle
|
||||
}
|
||||
|
||||
maxActive, err := strconv.Atoi(os.Getenv("REDIS_POOL_MAX_ACTIVE"))
|
||||
if err != nil || maxActive < 0 {
|
||||
maxActive = poolMaxActive
|
||||
}
|
||||
|
||||
idleTimeout, err := strconv.ParseInt(os.Getenv("REDIS_POOL_IDLE_TIMEOUT"), 10, 64)
|
||||
if err != nil || idleTimeout < 0 {
|
||||
idleTimeout = poolIdleTimeout
|
||||
}
|
||||
|
||||
pool = &redis.Pool{
|
||||
Dial: func() (redis.Conn, error) {
|
||||
url := config.GetRedisOfRegURL()
|
||||
if url == "" {
|
||||
url = "redis://localhost:6379/1"
|
||||
}
|
||||
|
||||
return redis.DialURL(url)
|
||||
},
|
||||
TestOnBorrow: func(c redis.Conn, t time.Time) error {
|
||||
_, err := c.Do("PING")
|
||||
return err
|
||||
},
|
||||
MaxIdle: maxIdle,
|
||||
MaxActive: maxActive,
|
||||
IdleTimeout: time.Duration(idleTimeout) * time.Second,
|
||||
Wait: true,
|
||||
}
|
||||
})
|
||||
|
||||
return pool
|
||||
}
|
||||
|
||||
// RequireLock returns lock by key
|
||||
func RequireLock(key string, conns ...redis.Conn) (*Mutex, error) {
|
||||
var conn redis.Conn
|
||||
if len(conns) > 0 {
|
||||
conn = conns[0]
|
||||
} else {
|
||||
conn = DefaultPool().Get()
|
||||
}
|
||||
|
||||
m := New(conn, key, utils.GenerateRandomString())
|
||||
ok, err := m.Require()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("require redis lock failed: %v", err)
|
||||
}
|
||||
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unable to require lock for %s", key)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// FreeLock free lock
|
||||
func FreeLock(m *Mutex) error {
|
||||
if _, err := m.Free(); err != nil {
|
||||
log.Warningf("failed to free lock %s, error: %v", m.key, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := m.Conn.Close(); err != nil {
|
||||
log.Warningf("failed to close the redis con for lock %s, error: %v", m.key, err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -16,23 +16,23 @@ package redis
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/garyburd/redigo/redis"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/garyburd/redigo/redis"
|
||||
"github.com/goharbor/harbor/src/common/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const testingRedisHost = "REDIS_HOST"
|
||||
|
||||
func init() {
|
||||
os.Setenv("REDIS_LOCK_MAX_RETRY", "5")
|
||||
}
|
||||
|
||||
func TestRedisLock(t *testing.T) {
|
||||
con, err := redis.Dial(
|
||||
"tcp",
|
||||
fmt.Sprintf("%s:%d", getRedisHost(), 6379),
|
||||
redis.DialConnectTimeout(30*time.Second),
|
||||
redis.DialReadTimeout(time.Minute+10*time.Second),
|
||||
redis.DialWriteTimeout(10*time.Second),
|
||||
)
|
||||
con, err := redis.Dial("tcp", fmt.Sprintf("%s:%d", getRedisHost(), 6379))
|
||||
assert.Nil(t, err)
|
||||
defer con.Close()
|
||||
|
||||
@ -52,6 +52,46 @@ func TestRedisLock(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestRequireLock(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
conn, err := redis.Dial("tcp", fmt.Sprintf("%s:%d", getRedisHost(), 6379))
|
||||
assert.Nil(err)
|
||||
defer conn.Close()
|
||||
|
||||
if l, err := RequireLock(utils.GenerateRandomString(), conn); assert.Nil(err) {
|
||||
l.Free()
|
||||
}
|
||||
|
||||
if l, err := RequireLock(utils.GenerateRandomString()); assert.Nil(err) {
|
||||
FreeLock(l)
|
||||
}
|
||||
|
||||
key := utils.GenerateRandomString()
|
||||
if l, err := RequireLock(key); assert.Nil(err) {
|
||||
defer FreeLock(l)
|
||||
|
||||
_, err = RequireLock(key)
|
||||
assert.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreeLock(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
if l, err := RequireLock(utils.GenerateRandomString()); assert.Nil(err) {
|
||||
assert.Nil(FreeLock(l))
|
||||
}
|
||||
|
||||
conn, err := redis.Dial("tcp", fmt.Sprintf("%s:%d", getRedisHost(), 6379))
|
||||
assert.Nil(err)
|
||||
|
||||
if l, err := RequireLock(utils.GenerateRandomString(), conn); assert.Nil(err) {
|
||||
conn.Close()
|
||||
assert.Error(FreeLock(l))
|
||||
}
|
||||
}
|
||||
|
||||
func getRedisHost() string {
|
||||
redisHost := os.Getenv(testingRedisHost)
|
||||
if redisHost == "" {
|
||||
|
@ -12,13 +12,13 @@ import (
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/goharbor/harbor/src/common"
|
||||
"github.com/goharbor/harbor/src/core/label"
|
||||
|
||||
"github.com/goharbor/harbor/src/chartserver"
|
||||
"github.com/goharbor/harbor/src/common"
|
||||
"github.com/goharbor/harbor/src/common/rbac"
|
||||
hlog "github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
"github.com/goharbor/harbor/src/core/label"
|
||||
"github.com/goharbor/harbor/src/core/middlewares"
|
||||
rep_event "github.com/goharbor/harbor/src/replication/event"
|
||||
"github.com/goharbor/harbor/src/replication/model"
|
||||
)
|
||||
@ -531,7 +531,7 @@ func initializeChartController() (*chartserver.Controller, error) {
|
||||
return nil, errors.New("Endpoint URL of chart storage server is malformed")
|
||||
}
|
||||
|
||||
controller, err := chartserver.NewController(url)
|
||||
controller, err := chartserver.NewController(url, middlewares.New(middlewares.ChartMiddlewares).Create())
|
||||
if err != nil {
|
||||
return nil, errors.New("Failed to initialize chart API controller")
|
||||
}
|
||||
|
@ -15,7 +15,10 @@
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/chart"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/contenttrust"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/countquota"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/listrepo"
|
||||
@ -25,7 +28,6 @@ import (
|
||||
"github.com/goharbor/harbor/src/core/middlewares/url"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/vulnerable"
|
||||
"github.com/justinas/alice"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// DefaultCreator ...
|
||||
@ -59,6 +61,7 @@ func (b *DefaultCreator) Create() *alice.Chain {
|
||||
|
||||
func (b *DefaultCreator) geMiddleware(mName string) alice.Constructor {
|
||||
middlewares := map[string]alice.Constructor{
|
||||
CHART: func(next http.Handler) http.Handler { return chart.New(next) },
|
||||
READONLY: func(next http.Handler) http.Handler { return readonly.New(next) },
|
||||
URL: func(next http.Handler) http.Handler { return url.New(next) },
|
||||
MUITIPLEMANIFEST: func(next http.Handler) http.Handler { return multiplmanifest.New(next) },
|
||||
|
128
src/core/middlewares/chart/builder.go
Normal file
128
src/core/middlewares/chart/builder.go
Normal file
@ -0,0 +1,128 @@
|
||||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chart
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/dao"
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/interceptor"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/interceptor/quota"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/util"
|
||||
"github.com/goharbor/harbor/src/pkg/types"
|
||||
)
|
||||
|
||||
var (
|
||||
deleteChartVersionRe = regexp.MustCompile(`^/api/chartrepo/(?P<namespace>\w+)/charts/(?P<name>\w+)/(?P<version>[\w\d\.]+)/?$`)
|
||||
uploadChartVersionRe = regexp.MustCompile(`^/api/chartrepo/(?P<namespace>\w+)/charts/?$`)
|
||||
)
|
||||
|
||||
var (
|
||||
defaultBuilders = []interceptor.Builder{
|
||||
&deleteChartVersionBuilder{},
|
||||
&uploadChartVersionBuilder{},
|
||||
}
|
||||
)
|
||||
|
||||
type deleteChartVersionBuilder struct {
|
||||
}
|
||||
|
||||
func (m *deleteChartVersionBuilder) Build(req *http.Request) interceptor.Interceptor {
|
||||
if req.Method != http.MethodDelete {
|
||||
return nil
|
||||
}
|
||||
|
||||
matches := deleteChartVersionRe.FindStringSubmatch(req.URL.String())
|
||||
if len(matches) <= 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
namespace, chartName, version := matches[1], matches[2], matches[3]
|
||||
|
||||
project, err := dao.GetProjectByName(namespace)
|
||||
if err != nil {
|
||||
log.Errorf("Failed to get project %s, error: %v", namespace, err)
|
||||
return nil
|
||||
}
|
||||
if project == nil {
|
||||
log.Warningf("Project %s not found", namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
opts := []quota.Option{
|
||||
quota.WithManager("project", strconv.FormatInt(project.ProjectID, 10)),
|
||||
quota.WithAction(quota.SubtractAction),
|
||||
quota.StatusCode(http.StatusOK),
|
||||
quota.MutexKeys(mutexKey(namespace, chartName, version)),
|
||||
quota.Resources(types.ResourceList{types.ResourceCount: 1}),
|
||||
}
|
||||
|
||||
return quota.New(opts...)
|
||||
}
|
||||
|
||||
type uploadChartVersionBuilder struct {
|
||||
}
|
||||
|
||||
func (m *uploadChartVersionBuilder) Build(req *http.Request) interceptor.Interceptor {
|
||||
if req.Method != http.MethodPost {
|
||||
return nil
|
||||
}
|
||||
|
||||
matches := uploadChartVersionRe.FindStringSubmatch(req.URL.String())
|
||||
if len(matches) <= 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
namespace := matches[1]
|
||||
|
||||
project, err := dao.GetProjectByName(namespace)
|
||||
if err != nil {
|
||||
log.Errorf("Failed to get project %s, error: %v", namespace, err)
|
||||
return nil
|
||||
}
|
||||
if project == nil {
|
||||
log.Warningf("Project %s not found", namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
chart, err := parseChart(req)
|
||||
if err != nil {
|
||||
log.Errorf("Failed to parse chart from body, error: %v", err)
|
||||
return nil
|
||||
}
|
||||
chartName, version := chart.Metadata.Name, chart.Metadata.Version
|
||||
|
||||
info := &util.ChartVersionInfo{
|
||||
ProjectID: project.ProjectID,
|
||||
Namespace: namespace,
|
||||
ChartName: chartName,
|
||||
Version: version,
|
||||
}
|
||||
// Chart version info will be used by computeQuotaForUpload
|
||||
*req = *req.WithContext(util.NewChartVersionInfoContext(req.Context(), info))
|
||||
|
||||
opts := []quota.Option{
|
||||
quota.WithManager("project", strconv.FormatInt(project.ProjectID, 10)),
|
||||
quota.WithAction(quota.AddAction),
|
||||
quota.StatusCode(http.StatusCreated),
|
||||
quota.MutexKeys(mutexKey(namespace, chartName, version)),
|
||||
quota.OnResources(computeQuotaForUpload),
|
||||
}
|
||||
|
||||
return quota.New(opts...)
|
||||
}
|
73
src/core/middlewares/chart/handler.go
Normal file
73
src/core/middlewares/chart/handler.go
Normal file
@ -0,0 +1,73 @@
|
||||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chart
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/interceptor"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/util"
|
||||
)
|
||||
|
||||
type chartHandler struct {
|
||||
builders []interceptor.Builder
|
||||
next http.Handler
|
||||
}
|
||||
|
||||
// New ...
|
||||
func New(next http.Handler, builders ...interceptor.Builder) http.Handler {
|
||||
if len(builders) == 0 {
|
||||
builders = defaultBuilders
|
||||
}
|
||||
|
||||
return &chartHandler{
|
||||
builders: builders,
|
||||
next: next,
|
||||
}
|
||||
}
|
||||
|
||||
// ServeHTTP manifest ...
|
||||
func (h *chartHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
|
||||
interceptor := h.getInterceptor(req)
|
||||
if interceptor == nil {
|
||||
h.next.ServeHTTP(rw, req)
|
||||
return
|
||||
}
|
||||
|
||||
if err := interceptor.HandleRequest(req); err != nil {
|
||||
log.Warningf("Error occurred when to handle request in count quota handler: %v", err)
|
||||
http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in chart count quota handler: %v", err)),
|
||||
http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w := util.NewCustomResponseWriter(rw)
|
||||
h.next.ServeHTTP(w, req)
|
||||
|
||||
interceptor.HandleResponse(w, req)
|
||||
}
|
||||
|
||||
func (h *chartHandler) getInterceptor(req *http.Request) interceptor.Interceptor {
|
||||
for _, builder := range h.builders {
|
||||
interceptor := builder.Build(req)
|
||||
if interceptor != nil {
|
||||
return interceptor
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
118
src/core/middlewares/chart/util.go
Normal file
118
src/core/middlewares/chart/util.go
Normal file
@ -0,0 +1,118 @@
|
||||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chart
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/goharbor/harbor/src/chartserver"
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/util"
|
||||
"github.com/goharbor/harbor/src/pkg/types"
|
||||
"k8s.io/helm/pkg/chartutil"
|
||||
"k8s.io/helm/pkg/proto/hapi/chart"
|
||||
)
|
||||
|
||||
const (
|
||||
formFieldNameForChart = "chart"
|
||||
)
|
||||
|
||||
var (
|
||||
controller *chartserver.Controller
|
||||
controllerErr error
|
||||
controllerOnce sync.Once
|
||||
)
|
||||
|
||||
func chartController() (*chartserver.Controller, error) {
|
||||
controllerOnce.Do(func() {
|
||||
addr, err := config.GetChartMuseumEndpoint()
|
||||
if err != nil {
|
||||
controllerErr = fmt.Errorf("failed to get the endpoint URL of chart storage server: %s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
addr = strings.TrimSuffix(addr, "/")
|
||||
url, err := url.Parse(addr)
|
||||
if err != nil {
|
||||
controllerErr = errors.New("endpoint URL of chart storage server is malformed")
|
||||
return
|
||||
}
|
||||
|
||||
ctr, err := chartserver.NewController(url)
|
||||
if err != nil {
|
||||
controllerErr = errors.New("failed to initialize chart API controller")
|
||||
}
|
||||
|
||||
controller = ctr
|
||||
|
||||
log.Debugf("Chart storage server is set to %s", url.String())
|
||||
log.Info("API controller for chart repository server is successfully initialized")
|
||||
})
|
||||
|
||||
return controller, controllerErr
|
||||
}
|
||||
|
||||
func chartVersionExists(namespace, chartName, version string) bool {
|
||||
ctr, err := chartController()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
chartVersion, err := ctr.GetChartVersion(namespace, chartName, version)
|
||||
if err != nil {
|
||||
log.Debugf("Get chart %s of version %s in namespace %s failed, error: %v", chartName, version, namespace, err)
|
||||
return false
|
||||
}
|
||||
|
||||
return !chartVersion.Removed
|
||||
}
|
||||
|
||||
func computeQuotaForUpload(req *http.Request) types.ResourceList {
|
||||
info, ok := util.ChartVersionInfoFromContext(req.Context())
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
if chartVersionExists(info.Namespace, info.ChartName, info.Version) {
|
||||
log.Debugf("Chart %s with version %s in namespace %s exists", info.ChartName, info.Version, info.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
return types.ResourceList{types.ResourceCount: 1}
|
||||
}
|
||||
|
||||
func mutexKey(str ...string) string {
|
||||
return "chart:" + strings.Join(str, ":")
|
||||
}
|
||||
|
||||
func parseChart(req *http.Request) (*chart.Chart, error) {
|
||||
chartFile, _, err := req.FormFile(formFieldNameForChart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
chart, err := chartutil.LoadArchive(chartFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load chart from archive failed: %s", err.Error())
|
||||
}
|
||||
|
||||
return chart, nil
|
||||
}
|
@ -16,6 +16,7 @@ package middlewares
|
||||
|
||||
// const variables
|
||||
const (
|
||||
CHART = "chart"
|
||||
READONLY = "readonly"
|
||||
URL = "url"
|
||||
MUITIPLEMANIFEST = "manifest"
|
||||
@ -26,6 +27,9 @@ const (
|
||||
COUNTQUOTA = "countquota"
|
||||
)
|
||||
|
||||
// ChartMiddlewares middlewares for chart server
|
||||
var ChartMiddlewares = []string{CHART}
|
||||
|
||||
// Middlewares with sequential organization
|
||||
var Middlewares = []string{READONLY, URL, MUITIPLEMANIFEST, LISTREPO, CONTENTTRUST, VULNERABLE, SIZEQUOTA, COUNTQUOTA}
|
||||
|
||||
|
34
src/core/middlewares/interceptor/interceptor.go
Normal file
34
src/core/middlewares/interceptor/interceptor.go
Normal file
@ -0,0 +1,34 @@
|
||||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package interceptor
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Builder interceptor builder
|
||||
type Builder interface {
|
||||
// Build build interceptor from http.Request returns nil if interceptor not match the request
|
||||
Build(*http.Request) Interceptor
|
||||
}
|
||||
|
||||
// Interceptor interceptor for middleware
|
||||
type Interceptor interface {
|
||||
// HandleRequest ...
|
||||
HandleRequest(*http.Request) error
|
||||
|
||||
// HandleResponse won't return any error
|
||||
HandleResponse(http.ResponseWriter, *http.Request)
|
||||
}
|
121
src/core/middlewares/interceptor/quota/options.go
Normal file
121
src/core/middlewares/interceptor/quota/options.go
Normal file
@ -0,0 +1,121 @@
|
||||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package quota
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/quota"
|
||||
"github.com/goharbor/harbor/src/pkg/types"
|
||||
)
|
||||
|
||||
// Option ...
|
||||
type Option func(*Options)
|
||||
|
||||
// Action ...
|
||||
type Action string
|
||||
|
||||
const (
|
||||
// AddAction action to add resources
|
||||
AddAction Action = "add"
|
||||
// SubtractAction action to subtract resources
|
||||
SubtractAction Action = "subtract"
|
||||
)
|
||||
|
||||
// Options ...
|
||||
type Options struct {
|
||||
Action Action
|
||||
Manager *quota.Manager
|
||||
MutexKeys []string
|
||||
Resources types.ResourceList
|
||||
StatusCode int
|
||||
|
||||
OnResources func(*http.Request) types.ResourceList
|
||||
OnFulfilled func(http.ResponseWriter, *http.Request) error
|
||||
OnRejected func(http.ResponseWriter, *http.Request) error
|
||||
OnFinally func(http.ResponseWriter, *http.Request) error
|
||||
}
|
||||
|
||||
func newOptions(opt ...Option) Options {
|
||||
opts := Options{}
|
||||
|
||||
for _, o := range opt {
|
||||
o(&opts)
|
||||
}
|
||||
|
||||
if opts.Action == "" {
|
||||
opts.Action = AddAction
|
||||
}
|
||||
|
||||
if opts.StatusCode == 0 {
|
||||
opts.StatusCode = http.StatusOK
|
||||
}
|
||||
|
||||
return opts
|
||||
}
|
||||
|
||||
// WithAction sets the interceptor action
|
||||
func WithAction(a Action) Option {
|
||||
return func(o *Options) {
|
||||
o.Action = a
|
||||
}
|
||||
}
|
||||
|
||||
// Manager sets the interceptor manager
|
||||
func Manager(m *quota.Manager) Option {
|
||||
return func(o *Options) {
|
||||
o.Manager = m
|
||||
}
|
||||
}
|
||||
|
||||
// WithManager sets the interceptor manager by reference and referenceID
|
||||
func WithManager(reference, referenceID string) Option {
|
||||
return func(o *Options) {
|
||||
m, err := quota.NewManager(reference, referenceID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
o.Manager = m
|
||||
}
|
||||
}
|
||||
|
||||
// MutexKeys set the interceptor mutex keys
|
||||
func MutexKeys(keys ...string) Option {
|
||||
return func(o *Options) {
|
||||
o.MutexKeys = keys
|
||||
}
|
||||
}
|
||||
|
||||
// Resources set the interceptor resources
|
||||
func Resources(r types.ResourceList) Option {
|
||||
return func(o *Options) {
|
||||
o.Resources = r
|
||||
}
|
||||
}
|
||||
|
||||
// StatusCode set the interceptor status code
|
||||
func StatusCode(c int) Option {
|
||||
return func(o *Options) {
|
||||
o.StatusCode = c
|
||||
}
|
||||
}
|
||||
|
||||
// OnResources sets the interceptor on resources function
|
||||
func OnResources(f func(*http.Request) types.ResourceList) Option {
|
||||
return func(o *Options) {
|
||||
o.OnResources = f
|
||||
}
|
||||
}
|
147
src/core/middlewares/interceptor/quota/quota.go
Normal file
147
src/core/middlewares/interceptor/quota/quota.go
Normal file
@ -0,0 +1,147 @@
|
||||
// Copyright Project Harbor Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package quota
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/goharbor/harbor/src/common/utils/log"
|
||||
"github.com/goharbor/harbor/src/common/utils/redis"
|
||||
"github.com/goharbor/harbor/src/core/middlewares/interceptor"
|
||||
"github.com/goharbor/harbor/src/pkg/types"
|
||||
)
|
||||
|
||||
// New ....
|
||||
func New(opts ...Option) interceptor.Interceptor {
|
||||
options := newOptions(opts...)
|
||||
|
||||
return "aInterceptor{opts: &options}
|
||||
}
|
||||
|
||||
type statusRecorder interface {
|
||||
Status() int
|
||||
}
|
||||
|
||||
type quotaInterceptor struct {
|
||||
opts *Options
|
||||
resources types.ResourceList
|
||||
mutexes []*redis.Mutex
|
||||
}
|
||||
|
||||
// HandleRequest ...
|
||||
func (qi *quotaInterceptor) HandleRequest(req *http.Request) (err error) {
|
||||
defer func() {
|
||||
if err != nil {
|
||||
qi.freeMutexes()
|
||||
}
|
||||
}()
|
||||
|
||||
opts := qi.opts
|
||||
|
||||
for _, key := range opts.MutexKeys {
|
||||
m, err := redis.RequireLock(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
qi.mutexes = append(qi.mutexes, m)
|
||||
}
|
||||
|
||||
resources := opts.Resources
|
||||
if len(resources) == 0 && opts.OnResources != nil {
|
||||
resources = opts.OnResources(req)
|
||||
log.Debugf("Compute the resources for quota, got: %v", resources)
|
||||
}
|
||||
qi.resources = resources
|
||||
|
||||
err = qi.reserve()
|
||||
if err != nil {
|
||||
log.Errorf("Failed to %s resources, error: %v", opts.Action, err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// HandleResponse ...
|
||||
func (qi *quotaInterceptor) HandleResponse(w http.ResponseWriter, req *http.Request) {
|
||||
defer qi.freeMutexes()
|
||||
|
||||
sr, ok := w.(statusRecorder)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
opts := qi.opts
|
||||
|
||||
switch sr.Status() {
|
||||
case opts.StatusCode:
|
||||
if opts.OnFulfilled != nil {
|
||||
opts.OnFulfilled(w, req)
|
||||
}
|
||||
default:
|
||||
if err := qi.unreserve(); err != nil {
|
||||
log.Errorf("Failed to %s resources, error: %v", opts.Action, err)
|
||||
}
|
||||
|
||||
if opts.OnRejected != nil {
|
||||
opts.OnRejected(w, req)
|
||||
}
|
||||
}
|
||||
|
||||
if opts.OnFinally != nil {
|
||||
opts.OnFinally(w, req)
|
||||
}
|
||||
}
|
||||
|
||||
func (qi *quotaInterceptor) freeMutexes() {
|
||||
for i := len(qi.mutexes) - 1; i >= 0; i-- {
|
||||
if err := redis.FreeLock(qi.mutexes[i]); err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (qi *quotaInterceptor) reserve() error {
|
||||
log.Debugf("Reserve %s resources, %v", qi.opts.Action, qi.resources)
|
||||
|
||||
if len(qi.resources) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch qi.opts.Action {
|
||||
case AddAction:
|
||||
return qi.opts.Manager.AddResources(qi.resources)
|
||||
case SubtractAction:
|
||||
return qi.opts.Manager.SubtractResources(qi.resources)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (qi *quotaInterceptor) unreserve() error {
|
||||
log.Debugf("Unreserve %s resources, %v", qi.opts.Action, qi.resources)
|
||||
|
||||
if len(qi.resources) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch qi.opts.Action {
|
||||
case AddAction:
|
||||
return qi.opts.Manager.SubtractResources(qi.resources)
|
||||
case SubtractAction:
|
||||
return qi.opts.Manager.AddResources(qi.resources)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
@ -15,9 +15,18 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/docker/distribution"
|
||||
"github.com/garyburd/redigo/redis"
|
||||
"github.com/goharbor/harbor/src/common/dao"
|
||||
@ -29,13 +38,6 @@ import (
|
||||
"github.com/goharbor/harbor/src/core/config"
|
||||
"github.com/goharbor/harbor/src/core/promgr"
|
||||
"github.com/goharbor/harbor/src/pkg/scan/whitelist"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type contextKey string
|
||||
@ -46,6 +48,9 @@ var ErrRequireQuota = errors.New("cannot get quota on project for request")
|
||||
const (
|
||||
manifestURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})`
|
||||
blobURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)blobs/uploads/`
|
||||
|
||||
chartVersionInfoKey = contextKey("ChartVersionInfo")
|
||||
|
||||
// ImageInfoCtxKey the context key for image information
|
||||
ImageInfoCtxKey = contextKey("ImageInfo")
|
||||
// TokenUsername ...
|
||||
@ -64,6 +69,14 @@ const (
|
||||
DialWriteTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// ChartVersionInfo ...
|
||||
type ChartVersionInfo struct {
|
||||
ProjectID int64
|
||||
Namespace string
|
||||
ChartName string
|
||||
Version string
|
||||
}
|
||||
|
||||
// ImageInfo ...
|
||||
type ImageInfo struct {
|
||||
Repository string
|
||||
@ -386,3 +399,14 @@ func GetRegRedisCon() (redis.Conn, error) {
|
||||
redis.DialWriteTimeout(DialWriteTimeout),
|
||||
)
|
||||
}
|
||||
|
||||
// ChartVersionInfoFromContext returns chart info from context
|
||||
func ChartVersionInfoFromContext(ctx context.Context) (*ChartVersionInfo, bool) {
|
||||
info, ok := ctx.Value(chartVersionInfoKey).(*ChartVersionInfo)
|
||||
return info, ok
|
||||
}
|
||||
|
||||
// NewChartVersionInfoContext returns context with blob info
|
||||
func NewChartVersionInfoContext(ctx context.Context, info *ChartVersionInfo) context.Context {
|
||||
return context.WithValue(ctx, chartVersionInfoKey, info)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user