Add metrics to Harbor Core

1. Add configs in prepare
 2. Add models and config items in Core
 3. Encapdulate getting metric in commom package
 4. Add a middleware for global request to collect 3 metrics

Signed-off-by: DQ <dengq@vmware.com>

make/harbor.yml.tmpl

@@ -208,3 +208,8 @@ proxy:
     - jobservice
     - clair
     - trivy
+
+# metric:
+#   enabled: false
+#   port: 9090
+#   path: /metrics

make/photon/prepare/models.py

@@ -139,3 +139,8 @@ class InternalTLS:
                 os.chown(file, DEFAULT_UID, DEFAULT_GID)
 
 
+class Metric:
+    def __init__(self, enabled: bool = False, port: int = 8080, path: str = "metrics" ):
+        self.enabled = enabled
+        self.port = port
+        self.path = path

make/photon/prepare/templates/core/env.jinja

@@ -56,3 +56,11 @@ INTERNAL_TLS_TRUST_CA_PATH=/harbor_cust_cert/harbor_internal_ca.crt
 {% else %}
 PORT=8080
 {% endif %}
+
+{% if metric.enabled %}
+METRIC_ENABLE=true
+METRIC_PATH={{ metric.path }}
+METRIC_PORT={{ metric.port }}
+METRIC_NAMESPACE=harbor
+METRIC_SUBSYSTEM=core
+{% endif %}

make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja

@@ -370,6 +370,9 @@ services:
 {% if protocol == 'https' %}
       - {{https_port}}:8443
 {% endif %}
+{% if metric.enabled %}
+      - {{metric.port}}:9090
+{% endif %}
 {% if with_notary %}
       - 4443:4443
 {% endif %}

make/photon/prepare/templates/nginx/nginx.http.conf.jinja

@@ -208,4 +208,21 @@ http {
       return 404;
     }
   }
+  {% if metric.enabled %}
+  upstream core_metrics {
+    server core:9090;
+  }
+
+  upstream registry_metrics {
+    server registry:5001;
+  }
+  server {
+    listen 9090;
+    location = /metrics {
+      if ($arg_comp = core) { proxy_pass http://core_metrics; }
+      if ($arg_comp = registry) { proxy_pass http://registry_metrics; }
+      proxy_pass http://core_metrics;
+    }
+  }
+  {% endif %}
 }

make/photon/prepare/templates/nginx/nginx.https.conf.jinja

@@ -240,4 +240,21 @@ http {
       #server_name harbordomain.com;
       return 308 https://{{https_redirect}}$request_uri;
   }
+  {% if metric.enabled %}
+  upstream core_metrics {
+    server core:{{ metric.port }};
+  }
+
+  upstream registry_metrics {
+    server registry:{{ metric.port }};
+  }
+  server {
+    listen 9090;
+    location = {{ metric.path }} {
+      if ($arg_comp = core) { proxy_pass http://core_metrics; }
+      if ($arg_comp = registry) { proxy_pass http://registry_metrics; }
+      proxy_pass http://core_metrics;
+    }
+  }
+  {% endif %}
 }

make/photon/prepare/templates/registry/config.yml.jinja

@@ -41,7 +41,14 @@ http:
 {% endif %}
   secret: placeholder
   debug:
+{% if metric.enabled %}
+    addr: :{{ metric.port }}
+    prometheus:
+      enabled: true
+      path: {{ metric.path }}
+{% else %}
     addr: localhost:5001
+{% endif %}
 auth:
   htpasswd:
     realm: harbor-registry-basic-realm

make/photon/prepare/utils/configs.py

@@ -3,7 +3,7 @@ import os
 import yaml
 from urllib.parse import urlencode
 from g import versions_file_path, host_root_dir, DEFAULT_UID, INTERNAL_NO_PROXY_DN
-from models import InternalTLS
+from models import InternalTLS, Metric
 from utils.misc import generate_random_string, owner_can_read, other_can_read
 
 default_db_max_idle_conns = 2  # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
@@ -346,6 +346,13 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
     else:
         config_dict['internal_tls'] = InternalTLS()
 
+    # metric configs
+    metric_config = configs.get('metric')
+    if metric_config:
+        config_dict['metric'] = Metric(metric_config['enabled'], metric_config['port'], metric_config['path'])
+    else:
+        config_dict['metric'] = Metric()
+
     if config_dict['internal_tls'].enabled:
         config_dict['portal_url'] = 'https://portal:8443'
         config_dict['registry_url'] = 'https://registry:5443'

make/photon/prepare/utils/docker_compose.py

@@ -61,4 +61,9 @@ def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_ch
     if log_ep_host:
         rendering_variables['external_log_endpoint'] = True
 
+    # for metrics
+    metric = configs.get('metric')
+    if metric:
+        rendering_variables['metric'] = metric
+
     render_jinja(docker_compose_template_path, docker_compose_yml_path,  mode=0o644, **rendering_variables)

make/photon/prepare/utils/nginx.py

@@ -62,7 +62,8 @@ def render_nginx_template(config_dict):
             https_redirect='$host' + ('https_port' in config_dict and (":" + str(config_dict['https_port'])) or ""),
             ssl_cert=SSL_CERT_PATH,
             ssl_cert_key=SSL_CERT_KEY_PATH,
-            internal_tls=config_dict['internal_tls'])
+            internal_tls=config_dict['internal_tls'],
+            metric=config_dict['metric'])
         location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS
 
     else:
@@ -71,7 +72,8 @@ def render_nginx_template(config_dict):
             nginx_conf,
             uid=DEFAULT_UID,
             gid=DEFAULT_GID,
-            internal_tls=config_dict['internal_tls'])
+            internal_tls=config_dict['internal_tls'],
+            metric=config_dict['metric'])
         location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTP
     copy_nginx_location_configs_if_exist(nginx_template_ext_dir, nginx_confd_dir, location_file_pattern)
 

src/common/config/metadata/metadatalist.go

@@ -152,6 +152,9 @@ var (
 		// the unit of expiration is minute, 43200 minutes = 30 days
 		{Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true},
 		{Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true},
+		{Name: common.MetricEnable, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_ENABLE", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
+		{Name: common.MetricPort, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PORT", DefaultValue: "9090", ItemType: &IntType{}, Editable: true},
+		{Name: common.MetricPath, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PATH", DefaultValue: "/metrics", ItemType: &StringType{}, Editable: true},
 
 		{Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true},
 		{Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true},

src/common/const.go

@@ -157,4 +157,9 @@ const (
 
 	// DefaultGCTimeWindowHours is the reserve blob time window used by GC, default is 2 hours
 	DefaultGCTimeWindowHours = int64(2)
+
+	// Metric setting items
+	MetricEnable = "metric_enable"
+	MetricPort   = "metric_port"
+	MetricPath   = "metric_path"
 )

src/common/models/metric.go

@@ -0,0 +1,22 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+// Metric wraps the configurations to access UAA service
+type Metric struct {
+	Enabled bool
+	Port    int
+	Path    string
+}

src/core/config/config.go

@@ -487,3 +487,12 @@ func GetGCTimeWindow() int64 {
 	}
 	return common.DefaultGCTimeWindowHours
 }
+
+// Metric returns the overall metric settings
+func Metric() *models.Metric {
+	return &models.Metric{
+		Enabled: cfgMgr.Get(common.MetricEnable).GetBool(),
+		Port:    cfgMgr.Get(common.MetricPort).GetInt(),
+		Path:    cfgMgr.Get(common.MetricPath).GetString(),
+	}
+}

src/core/main.go

@@ -43,6 +43,7 @@ import (
 	"github.com/goharbor/harbor/src/core/middlewares"
 	"github.com/goharbor/harbor/src/core/service/token"
 	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/lib/metric"
 	"github.com/goharbor/harbor/src/migration"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	_ "github.com/goharbor/harbor/src/pkg/notifier/topic"
@@ -162,6 +163,11 @@ func main() {
 	log.Info("initializing configurations...")
 	config.Init()
 	log.Info("configurations initialization completed")
+	metricCfg := config.Metric()
+	if metricCfg.Enabled {
+		metric.RegisterCollectors()
+		go metric.ServeProm(metricCfg.Path, metricCfg.Port)
+	}
 	token.InitCreators()
 	database, err := config.Database()
 	if err != nil {

src/core/middlewares/middlewares.go

@@ -24,6 +24,7 @@ import (
 	"github.com/goharbor/harbor/src/server/middleware/artifactinfo"
 	"github.com/goharbor/harbor/src/server/middleware/csrf"
 	"github.com/goharbor/harbor/src/server/middleware/log"
+	"github.com/goharbor/harbor/src/server/middleware/metric"
 	"github.com/goharbor/harbor/src/server/middleware/notification"
 	"github.com/goharbor/harbor/src/server/middleware/orm"
 	"github.com/goharbor/harbor/src/server/middleware/readonly"
@@ -68,6 +69,7 @@ var (
 // MiddleWares returns global middlewares
 func MiddleWares() []beego.MiddleWare {
 	return []beego.MiddleWare{
+		metric.Middleware(),
 		requestid.Middleware(),
 		log.Middleware(),
 		session.Middleware(),

src/go.mod

@@ -59,6 +59,7 @@ require (
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
+	github.com/prometheus/client_golang v1.0.0
 	github.com/robfig/cron v1.0.0
 	github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect
 	github.com/spf13/viper v1.4.0 // indirect
@@ -67,7 +68,7 @@ require (
 	golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
 	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 // indirect
+	gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175
 	gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect
 	gopkg.in/fatih/pool.v2 v2.0.0 // indirect
 	gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect

src/lib/metric/collector.go

@@ -0,0 +1,51 @@
+package metric
+
+import (
+	"os"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// RegisterCollectors register all the common static collector
+func RegisterCollectors() {
+	prometheus.MustRegister([]prometheus.Collector{
+		TotalInFlightGauge,
+		TotalReqCnt,
+		TotalReqDurSummary,
+	}...)
+}
+
+var (
+	// TotalInFlightGauge used to collect total in flight number
+	TotalInFlightGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: os.Getenv(NamespaceEnvKey),
+			Subsystem: os.Getenv(SubsystemEnvKey),
+			Name:      "http_request_inflight",
+			Help:      "The total number of requests",
+		},
+		[]string{"url"},
+	)
+
+	// TotalReqCnt used to collect total request counter
+	TotalReqCnt = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: os.Getenv(NamespaceEnvKey),
+			Subsystem: os.Getenv(SubsystemEnvKey),
+			Name:      "http_request",
+			Help:      "The total number of requests",
+		},
+		[]string{"method", "code", "url"},
+	)
+
+	// TotalReqDurSummary used to collect total request duration summaries
+	TotalReqDurSummary = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace:  os.Getenv(NamespaceEnvKey),
+			Subsystem:  os.Getenv(SubsystemEnvKey),
+			Name:       "http_request_duration_seconds",
+			Help:       "The time duration of the requests",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"method", "url"})
+)

src/lib/metric/server.go

@@ -0,0 +1,24 @@
+package metric
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+const (
+	// NamespaceEnvKey is the metric namespace key in environment
+	NamespaceEnvKey = "METRIC_NAMESPACE"
+	// SubsystemEnvKey is the metric subsystem key in environment
+	SubsystemEnvKey = "METRIC_SUBSYSTEM"
+)
+
+// ServeProm return a server to serve prometheus metrics
+func ServeProm(path string, port int) {
+	mux := http.NewServeMux()
+	mux.Handle(path, promhttp.Handler())
+	log.Infof("Prometheus metric server running on port %v", port)
+	log.Errorf("Promethus metrcis server down with %s", http.ListenAndServe(fmt.Sprintf(":%v", port), mux))
+}

src/server/middleware/metric/metric.go

@@ -0,0 +1,44 @@
+package metric
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/goharbor/harbor/src/core/config"
+	"github.com/goharbor/harbor/src/lib/metric"
+)
+
+func instrumentHandler(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		now, url, code := time.Now(), r.URL.EscapedPath(), "0"
+
+		metric.TotalInFlightGauge.WithLabelValues(url).Inc()
+		defer metric.TotalInFlightGauge.WithLabelValues(url).Dec()
+
+		next.ServeHTTP(w, r)
+		if r.Response != nil {
+			code = strconv.Itoa(r.Response.StatusCode)
+		}
+
+		metric.TotalReqDurSummary.WithLabelValues(r.Method, url).Observe(time.Since(now).Seconds())
+		metric.TotalReqCnt.WithLabelValues(r.Method, code, url).Inc()
+	})
+}
+
+// Middleware returns a middleware for handling requests
+func Middleware() func(http.Handler) http.Handler {
+	if config.Metric().Enabled {
+		return func(next http.Handler) http.Handler {
+			return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+				next = instrumentHandler(next)
+				next.ServeHTTP(rw, req)
+			})
+		}
+	}
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+			next.ServeHTTP(rw, req)
+		})
+	}
+}

src/vendor/modules.txt

@@ -427,6 +427,7 @@ github.com/pmezard/go-difflib/difflib
 github.com/pquerna/cachecontrol
 github.com/pquerna/cachecontrol/cacheobject
 # github.com/prometheus/client_golang v1.0.0
+## explicit
 github.com/prometheus/client_golang/prometheus
 github.com/prometheus/client_golang/prometheus/internal
 github.com/prometheus/client_golang/prometheus/promhttp