Aggregate metrics

1. Add operation id in ctx in baseapi before prepare
2. add operation id for registry proxy request
3. use url for other request

Signed-off-by: DQ <dengq@vmware.com>

src/lib/metric/collector.go

@@ -17,14 +17,13 @@ func RegisterCollectors() {
 
 var (
 	// TotalInFlightGauge used to collect total in flight number
-	TotalInFlightGauge = prometheus.NewGaugeVec(
+	TotalInFlightGauge = prometheus.NewGauge(
 		prometheus.GaugeOpts{
 			Namespace: os.Getenv(NamespaceEnvKey),
 			Subsystem: os.Getenv(SubsystemEnvKey),
-			Name:      "http_request_inflight",
+			Name:      "http_inflight_requests",
 			Help:      "The total number of requests",
 		},
-		[]string{"url"},
 	)
 
 	// TotalReqCnt used to collect total request counter
@@ -32,10 +31,10 @@ var (
 		prometheus.CounterOpts{
 			Namespace: os.Getenv(NamespaceEnvKey),
 			Subsystem: os.Getenv(SubsystemEnvKey),
-			Name:      "http_request",
+			Name:      "http_request_total",
 			Help:      "The total number of requests",
 		},
-		[]string{"method", "code", "url"},
+		[]string{"method", "code", "operation"},
 	)
 
 	// TotalReqDurSummary used to collect total request duration summaries
@@ -47,5 +46,5 @@ var (
 			Help:       "The time duration of the requests",
 			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
 		},
-		[]string{"method", "url"})
+		[]string{"method", "operation"})
 )

src/server/middleware/metric/metric.go

@@ -1,8 +1,10 @@
 package metric
 
 import (
+	"context"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/goharbor/harbor/src/core/config"
@@ -10,16 +12,57 @@ import (
 	"github.com/goharbor/harbor/src/lib/metric"
 )
 
+// ContextOpIDKey ...
+type contextOpIDKey struct{}
+
+const (
+	// CatalogOperationID ...
+	CatalogOperationID = "v2_catalog"
+	// ListTagOperationID ...
+	ListTagOperationID = "v2_tags"
+	// ManifestOperationID ...
+	ManifestOperationID = "v2_manifest"
+	// BlobsOperationID ...
+	BlobsOperationID = "v2_blob"
+	// BlobsUploadOperationID ...
+	BlobsUploadOperationID = "v2_blob_upload"
+	// OthersOperationID ...
+	OthersOperationID = "v2_others"
+)
+
+// SetMetricOpID used to set operation ID for metrics
+func SetMetricOpID(ctx context.Context, value string) {
+	if config.Metric().Enabled {
+		v := ctx.Value(contextOpIDKey{}).(*string)
+		*v = value
+	}
+}
+
+func isChartMuseumURL(url string) bool {
+	return strings.HasPrefix(url, "/chartrepo/") || strings.HasPrefix(url, "/api/chartrepo/")
+}
+
 func instrumentHandler(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		now, url := time.Now(), r.URL.EscapedPath()
+		metric.TotalInFlightGauge.Inc()
+		defer metric.TotalInFlightGauge.Dec()
+		now, rc, op := time.Now(), lib.NewResponseRecorder(w), ""
+		ctx := context.WithValue(r.Context(), contextOpIDKey{}, &op)
+		next.ServeHTTP(rc, r.WithContext(ctx))
+		if len(op) == 0 && isChartMuseumURL(r.URL.Path) {
+			op = "chartmuseum"
+		} else {
+			// From swagger's perspective the operation of this legacy URL is unknown
+			op = "unknown"
+		}
+		metric.TotalReqDurSummary.WithLabelValues(r.Method, op).Observe(time.Since(now).Seconds())
+		metric.TotalReqCnt.WithLabelValues(r.Method, strconv.Itoa(rc.StatusCode), op).Inc()
+	})
+}
 
-		metric.TotalInFlightGauge.WithLabelValues(url).Inc()
-		defer metric.TotalInFlightGauge.WithLabelValues(url).Dec()
-		rc := lib.NewResponseRecorder(w)
-		next.ServeHTTP(rc, r)
-		metric.TotalReqDurSummary.WithLabelValues(r.Method, url).Observe(time.Since(now).Seconds())
-		metric.TotalReqCnt.WithLabelValues(r.Method, strconv.Itoa(rc.StatusCode), url).Inc()
+func transparentHandler(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		next.ServeHTTP(w, r)
 	})
 }
 
@@ -28,9 +71,15 @@ func Middleware() func(http.Handler) http.Handler {
 	if config.Metric().Enabled {
 		return instrumentHandler
 	}
+	return transparentHandler
+}
+
+// InjectOpIDMiddleware returns a middleware used for injecting operations ID
+func InjectOpIDMiddleware(opID string) func(next http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-			next.ServeHTTP(rw, req)
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			SetMetricOpID(r.Context(), opID)
+			next.ServeHTTP(w, r)
 		})
 	}
 }

src/server/registry/route.go

@@ -20,6 +20,7 @@ import (
 	"github.com/goharbor/harbor/src/server/middleware/blob"
 	"github.com/goharbor/harbor/src/server/middleware/contenttrust"
 	"github.com/goharbor/harbor/src/server/middleware/immutable"
+	"github.com/goharbor/harbor/src/server/middleware/metric"
 	"github.com/goharbor/harbor/src/server/middleware/quota"
 	"github.com/goharbor/harbor/src/server/middleware/repoproxy"
 	"github.com/goharbor/harbor/src/server/middleware/v2auth"
@@ -36,16 +37,19 @@ func RegisterRoutes() {
 	root.NewRoute().
 		Method(http.MethodGet).
 		Path("/_catalog").
+		Middleware(metric.InjectOpIDMiddleware(metric.CatalogOperationID)).
 		Handler(newRepositoryHandler())
 	// list tags
 	root.NewRoute().
 		Method(http.MethodGet).
 		Path("/*/tags/list").
+		Middleware(metric.InjectOpIDMiddleware(metric.ListTagOperationID)).
 		Handler(newTagHandler())
 	// manifest
 	root.NewRoute().
 		Method(http.MethodGet).
 		Path("/*/manifests/:reference").
+		Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)).
 		Middleware(repoproxy.ManifestMiddleware()).
 		Middleware(contenttrust.Middleware()).
 		Middleware(vulnerable.Middleware()).
@@ -53,31 +57,43 @@ func RegisterRoutes() {
 	root.NewRoute().
 		Method(http.MethodHead).
 		Path("/*/manifests/:reference").
+		Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)).
 		Middleware(repoproxy.ManifestMiddleware()).
 		HandlerFunc(getManifest)
 	root.NewRoute().
 		Method(http.MethodDelete).
 		Path("/*/manifests/:reference").
+		Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)).
 		Middleware(quota.RefreshForProjectMiddleware()).
 		HandlerFunc(deleteManifest)
 	root.NewRoute().
 		Method(http.MethodPut).
 		Path("/*/manifests/:reference").
+		Middleware(metric.InjectOpIDMiddleware(metric.ManifestOperationID)).
 		Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()).
 		Middleware(immutable.Middleware()).
 		Middleware(quota.PutManifestMiddleware()).
 		Middleware(blob.PutManifestMiddleware()).
 		HandlerFunc(putManifest)
+	// blob head
+	root.NewRoute().
+		Method(http.MethodHead).
+		Path("/*/blobs/:digest").
+		Middleware(metric.InjectOpIDMiddleware(metric.BlobsOperationID)).
+		Middleware(blob.HeadBlobMiddleware()).
+		Handler(proxy)
 	// blob get
 	root.NewRoute().
 		Method(http.MethodGet).
 		Path("/*/blobs/:digest").
+		Middleware(metric.InjectOpIDMiddleware(metric.BlobsOperationID)).
 		Middleware(repoproxy.BlobGetMiddleware()).
 		Handler(proxy)
 	// initiate blob upload
 	root.NewRoute().
 		Method(http.MethodPost).
 		Path("/*/blobs/uploads").
+		Middleware(metric.InjectOpIDMiddleware(metric.BlobsUploadOperationID)).
 		Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()).
 		Middleware(quota.PostInitiateBlobUploadMiddleware()).
 		Middleware(blob.PostInitiateBlobUploadMiddleware()).
@@ -86,19 +102,16 @@ func RegisterRoutes() {
 	root.NewRoute().
 		Method(http.MethodPatch).
 		Path("/*/blobs/uploads/:session_id").
+		Middleware(metric.InjectOpIDMiddleware(metric.BlobsUploadOperationID)).
 		Middleware(blob.PatchBlobUploadMiddleware()).
 		Handler(proxy)
 	root.NewRoute().
 		Method(http.MethodPut).
 		Path("/*/blobs/uploads/:session_id").
+		Middleware(metric.InjectOpIDMiddleware(metric.BlobsUploadOperationID)).
 		Middleware(quota.PutBlobUploadMiddleware()).
 		Middleware(blob.PutBlobUploadMiddleware()).
 		Handler(proxy)
-	root.NewRoute().
-		Method(http.MethodHead).
-		Path("/*/blobs/:digest").
-		Middleware(blob.HeadBlobMiddleware()).
-		Handler(proxy)
 	// others
-	root.NewRoute().Path("/*").Handler(proxy)
+	root.NewRoute().Path("/*").Middleware(metric.InjectOpIDMiddleware(metric.OthersOperationID)).Handler(proxy)
 }

src/server/v2.0/handler/handler.go

@@ -23,6 +23,7 @@ import (
 	lib_http "github.com/goharbor/harbor/src/lib/http"
 	"github.com/goharbor/harbor/src/server/middleware"
 	"github.com/goharbor/harbor/src/server/middleware/blob"
+	"github.com/goharbor/harbor/src/server/middleware/metric"
 	"github.com/goharbor/harbor/src/server/middleware/quota"
 	"github.com/goharbor/harbor/src/server/v2.0/restapi"
 )
@@ -62,6 +63,7 @@ func New() http.Handler {
 
 // function is called before the Prepare of the operation
 func beforePrepare(ctx context.Context, operation string, params interface{}) rmiddleware.Responder {
+	metric.SetMetricOpID(ctx, operation)
 	return nil
 }
 

tests/apitests/python/test_verify_metrics_enabled.py

@@ -10,8 +10,8 @@ class TestMetricsExist(unittest.TestCase):
     golang_basic_metrics = ["go_gc_duration_seconds", "go_goroutines", "go_info", "go_memstats_alloc_bytes"]
 
     eigen_metrics = {
-        'core': golang_basic_metrics + ["harbor_core_http_request", "harbor_core_http_request_duration_seconds",
-    "harbor_core_http_request_inflight"],
+        'core': golang_basic_metrics + ["harbor_core_http_request_total", "harbor_core_http_request_duration_seconds",
+    "harbor_core_http_inflight_requests"],
         'registry': golang_basic_metrics + ["registry_http_in_flight_requests"],
         'exporter': golang_basic_metrics + ["harbor_image_pulled",
     "harbor_project_artifact_total", "harbor_project_member_total", "harbor_project_quota_byte",