Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-24 09:38:09 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #14056 from ninjadq/reduce_the_number_of_metrics_in_core

Browse Source 
Aggregate  metrics
This commit is contained in:
Qian Deng 2021-01-26 10:42:53 +08:00 committed by GitHub
commit 9574f8c3c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 87 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/lib/metric/collector.go
View File

@ -17,14 +17,13 @@ func RegisterCollectors() {
var (
// TotalInFlightGauge used to collect total in flight number
TotalInFlightGauge = prometheus.NewGaugeVec(
TotalInFlightGauge = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: os.Getenv(NamespaceEnvKey),
Subsystem: os.Getenv(SubsystemEnvKey),
Name: "http_request_inflight",
Name: "http_inflight_requests",
Help: "The total number of requests",
},
[]string{"url"},
)
// TotalReqCnt used to collect total request counter
@ -32,10 +31,10 @@ var (
prometheus.CounterOpts{
Namespace: os.Getenv(NamespaceEnvKey),
Subsystem: os.Getenv(SubsystemEnvKey),
Name: "http_request",
Name: "http_request_total",
Help: "The total number of requests",
},
[]string{"method", "code", "url"},
[]string{"method", "code", "operation"},
)
// TotalReqDurSummary used to collect total request duration summaries
@ -47,5 +46,5 @@ var (
Help: "The time duration of the requests",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
[]string{"method", "url"})
[]string{"method", "operation"})
)

67
src/server/middleware/metric/metric.go
View File

@ -1,8 +1,10 @@
package metric
import (
"context"
"net/http"
"strconv"
"strings"
"time"
"github.com/goharbor/harbor/src/core/config"
@ -10,16 +12,57 @@ import (
"github.com/goharbor/harbor/src/lib/metric"
)
// ContextOpIDKey ...
type contextOpIDKey struct{}
const (
// CatalogOperationID ...
CatalogOperationID = "v2_catalog"
// ListTagOperationID ...
ListTagOperationID = "v2_tags"
// ManifestOperationID ...
ManifestOperationID = "v2_manifest"
// BlobsOperationID ...
BlobsOperationID = "v2_blob"
// BlobsUploadOperationID ...
BlobsUploadOperationID = "v2_blob_upload"
// OthersOperationID ...
OthersOperationID = "v2_others"
)
// SetMetricOpID used to set operation ID for metrics
func SetMetricOpID(ctx context.Context, value string) {
if config.Metric().Enabled {
v := ctx.Value(contextOpIDKey{}).(*string)
*v = value
}
}
func isChartMuseumURL(url string) bool {
return strings.HasPrefix(url, "/chartrepo/") || strings.HasPrefix(url, "/api/chartrepo/")
}
func instrumentHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
now, url := time.Now(), r.URL.EscapedPath()
metric.TotalInFlightGauge.Inc()
defer metric.TotalInFlightGauge.Dec()
now, rc, op := time.Now(), lib.NewResponseRecorder(w), ""
ctx := context.WithValue(r.Context(), contextOpIDKey{}, &op)
next.ServeHTTP(rc, r.WithContext(ctx))
if len(op) == 0 && isChartMuseumURL(r.URL.Path) {
op = "chartmuseum"
} else {
// From swagger's perspective the operation of this legacy URL is unknown
op = "unknown"
}
metric.TotalReqDurSummary.WithLabelValues(r.Method, op).Observe(time.Since(now).Seconds())
metric.TotalReqCnt.WithLabelValues(r.Method, strconv.Itoa(rc.StatusCode), op).Inc()
})
}
metric.TotalInFlightGauge.WithLabelValues(url).Inc()
defer metric.TotalInFlightGauge.WithLabelValues(url).Dec()
rc := lib.NewResponseRecorder(w)
next.ServeHTTP(rc, r)
metric.TotalReqDurSummary.WithLabelValues(r.Method, url).Observe(time.Since(now).Seconds())
metric.TotalReqCnt.WithLabelValues(r.Method, strconv.Itoa(rc.StatusCode), url).Inc()
func transparentHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r)
})
}
@ -28,9 +71,15 @@ func Middleware() func(http.Handler) http.Handler {
if config.Metric().Enabled {
return instrumentHandler
}
return transparentHandler
}
// InjectOpIDMiddleware returns a middleware used for injecting operations ID
func InjectOpIDMiddleware(opID string) func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
next.ServeHTTP(rw, req)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
SetMetricOpID(r.Context(), opID)
next.ServeHTTP(w, r)
})
}
}

25
src/server/registry/route.go
View File

@ -20,6 +20,7 @@ import (
"github.com/goharbor/harbor/src/server/middleware/blob"
"github.com/goharbor/harbor/src/server/middleware/contenttrust"
"github.com/goharbor/harbor/src/server/middleware/immutable"
"github.com/goharbor/harbor/src/server/middleware/metric"
"github.com/goharbor/harbor/src/server/middleware/quota"
"github.com/goharbor/harbor/src/server/middleware/repoproxy"
"github.com/goharbor/harbor/src/server/middleware/v2auth"
@ -36,16 +37,19 @@ func RegisterRoutes() {
root.NewRoute().
Method(http.MethodGet).
Path("/_catalog").
Middleware(metric.InjectOpIDMiddleware(metric.CatalogOperationID)).
Handler(newRepositoryHandler())
// list tags
root.NewRoute().
Method(http.MethodGet).
Path("/*/tags/list").
Middleware(metric.InjectOpIDMiddleware(metric.ListTagOperationID)).
Handler(newTagHandler())
// manifest
root.NewRoute().
Method(http.MethodGet).
Path("/*/manifests/:reference").
Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)).
Middleware(repoproxy.ManifestMiddleware()).
Middleware(contenttrust.Middleware()).
Middleware(vulnerable.Middleware()).
@ -53,31 +57,43 @@ func RegisterRoutes() {
root.NewRoute().
Method(http.MethodHead).
Path("/*/manifests/:reference").
Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)).
Middleware(repoproxy.ManifestMiddleware()).
HandlerFunc(getManifest)
root.NewRoute().
Method(http.MethodDelete).
Path("/*/manifests/:reference").
Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)).
Middleware(quota.RefreshForProjectMiddleware()).
HandlerFunc(deleteManifest)
root.NewRoute().
Method(http.MethodPut).
Path("/*/manifests/:reference").
Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)).
Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()).
Middleware(immutable.Middleware()).
Middleware(quota.PutManifestMiddleware()).
Middleware(blob.PutManifestMiddleware()).
HandlerFunc(putManifest)
// blob head
root.NewRoute().
Method(http.MethodHead).
Path("/*/blobs/:digest").
Middleware(metric.InjectOpIDMiddleware(metric.BlobsOperationID)).
Middleware(blob.HeadBlobMiddleware()).
Handler(proxy)
// blob get
root.NewRoute().
Method(http.MethodGet).
Path("/*/blobs/:digest").
Middleware(metric.InjectOpIDMiddleware(metric.BlobsOperationID)).
Middleware(repoproxy.BlobGetMiddleware()).
Handler(proxy)
// initiate blob upload
root.NewRoute().
Method(http.MethodPost).
Path("/*/blobs/uploads").
Middleware(metric.InjectOpIDMiddleware(metric.BlobsUploadOperationID)).
Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()).
Middleware(quota.PostInitiateBlobUploadMiddleware()).
Middleware(blob.PostInitiateBlobUploadMiddleware()).
@ -86,19 +102,16 @@ func RegisterRoutes() {
root.NewRoute().
Method(http.MethodPatch).
Path("/*/blobs/uploads/:session_id").
Middleware(metric.InjectOpIDMiddleware(metric.BlobsUploadOperationID)).
Middleware(blob.PatchBlobUploadMiddleware()).
Handler(proxy)
root.NewRoute().
Method(http.MethodPut).
Path("/*/blobs/uploads/:session_id").
Middleware(metric.InjectOpIDMiddleware(metric.BlobsUploadOperationID)).
Middleware(quota.PutBlobUploadMiddleware()).
Middleware(blob.PutBlobUploadMiddleware()).
Handler(proxy)
root.NewRoute().
Method(http.MethodHead).
Path("/*/blobs/:digest").
Middleware(blob.HeadBlobMiddleware()).
Handler(proxy)
// others
root.NewRoute().Path("/*").Handler(proxy)
root.NewRoute().Path("/*").Middleware(metric.InjectOpIDMiddleware(metric.OthersOperationID)).Handler(proxy)
}

2
src/server/v2.0/handler/handler.go
View File

@ -23,6 +23,7 @@ import (
lib_http "github.com/goharbor/harbor/src/lib/http"
"github.com/goharbor/harbor/src/server/middleware"
"github.com/goharbor/harbor/src/server/middleware/blob"
"github.com/goharbor/harbor/src/server/middleware/metric"
"github.com/goharbor/harbor/src/server/middleware/quota"
"github.com/goharbor/harbor/src/server/v2.0/restapi"
)
@ -62,6 +63,7 @@ func New() http.Handler {
// function is called before the Prepare of the operation
func beforePrepare(ctx context.Context, operation string, params interface{}) rmiddleware.Responder {
metric.SetMetricOpID(ctx, operation)
return nil
}

4
tests/apitests/python/test_verify_metrics_enabled.py
View File

@ -10,8 +10,8 @@ class TestMetricsExist(unittest.TestCase):
golang_basic_metrics = ["go_gc_duration_seconds", "go_goroutines", "go_info", "go_memstats_alloc_bytes"]
eigen_metrics = {
'core': golang_basic_metrics + ["harbor_core_http_request", "harbor_core_http_request_duration_seconds",
"harbor_core_http_request_inflight"],
'core': golang_basic_metrics + ["harbor_core_http_request_total", "harbor_core_http_request_duration_seconds",
"harbor_core_http_inflight_requests"],
'registry': golang_basic_metrics + ["registry_http_in_flight_requests"],
'exporter': golang_basic_metrics + ["harbor_image_pulled",
"harbor_project_artifact_total", "harbor_project_member_total", "harbor_project_quota_byte",

2
tests/ci/api_common_install.sh
View File

@ -10,7 +10,7 @@ set -e
if [ -z "$1" ]; then echo no ip specified; exit 1;fi
# prepare cert ...
sudo ./tests/generateCerts.sh $1
sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests --upgrade
sudo apt-get install -y python-pip && sudo pip install --ignore-installed urllib3 chardet requests --upgrade
sudo ./tests/hostcfg.sh
if [ "$2" = 'LDAP' ]; then