diff --git a/make/harbor.yml.tmpl b/make/harbor.yml.tmpl index efe174959..d51f410f2 100644 --- a/make/harbor.yml.tmpl +++ b/make/harbor.yml.tmpl @@ -208,3 +208,8 @@ proxy: - jobservice - clair - trivy + +# metric: +# enabled: false +# port: 9090 +# path: /metrics diff --git a/make/photon/prepare/models.py b/make/photon/prepare/models.py index c0317ac12..4efc9b076 100644 --- a/make/photon/prepare/models.py +++ b/make/photon/prepare/models.py @@ -139,3 +139,8 @@ class InternalTLS: os.chown(file, DEFAULT_UID, DEFAULT_GID) +class Metric: + def __init__(self, enabled: bool = False, port: int = 8080, path: str = "metrics" ): + self.enabled = enabled + self.port = port + self.path = path diff --git a/make/photon/prepare/templates/core/env.jinja b/make/photon/prepare/templates/core/env.jinja index 1657ea7db..c1a22cc0e 100644 --- a/make/photon/prepare/templates/core/env.jinja +++ b/make/photon/prepare/templates/core/env.jinja @@ -55,4 +55,12 @@ INTERNAL_TLS_CERT_PATH=/etc/harbor/ssl/core.crt INTERNAL_TLS_TRUST_CA_PATH=/harbor_cust_cert/harbor_internal_ca.crt {% else %} PORT=8080 -{% endif %} \ No newline at end of file +{% endif %} + +{% if metric.enabled %} +METRIC_ENABLE=true +METRIC_PATH={{ metric.path }} +METRIC_PORT={{ metric.port }} +METRIC_NAMESPACE=harbor +METRIC_SUBSYSTEM=core +{% endif %} diff --git a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja index e5f2d1421..8632055de 100644 --- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja +++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja @@ -370,6 +370,9 @@ services: {% if protocol == 'https' %} - {{https_port}}:8443 {% endif %} +{% if metric.enabled %} + - {{metric.port}}:9090 +{% endif %} {% if with_notary %} - 4443:4443 {% endif %} diff --git a/make/photon/prepare/templates/nginx/nginx.http.conf.jinja b/make/photon/prepare/templates/nginx/nginx.http.conf.jinja index 724a6241e..de2ff12c0 100644 --- a/make/photon/prepare/templates/nginx/nginx.http.conf.jinja +++ b/make/photon/prepare/templates/nginx/nginx.http.conf.jinja @@ -208,4 +208,21 @@ http { return 404; } } + {% if metric.enabled %} + upstream core_metrics { + server core:9090; + } + + upstream registry_metrics { + server registry:5001; + } + server { + listen 9090; + location = /metrics { + if ($arg_comp = core) { proxy_pass http://core_metrics; } + if ($arg_comp = registry) { proxy_pass http://registry_metrics; } + proxy_pass http://core_metrics; + } + } + {% endif %} } diff --git a/make/photon/prepare/templates/nginx/nginx.https.conf.jinja b/make/photon/prepare/templates/nginx/nginx.https.conf.jinja index 02b780dbe..dafbbd68b 100644 --- a/make/photon/prepare/templates/nginx/nginx.https.conf.jinja +++ b/make/photon/prepare/templates/nginx/nginx.https.conf.jinja @@ -239,5 +239,22 @@ http { listen 8080; #server_name harbordomain.com; return 308 https://{{https_redirect}}$request_uri; - } + } + {% if metric.enabled %} + upstream core_metrics { + server core:{{ metric.port }}; + } + + upstream registry_metrics { + server registry:{{ metric.port }}; + } + server { + listen 9090; + location = {{ metric.path }} { + if ($arg_comp = core) { proxy_pass http://core_metrics; } + if ($arg_comp = registry) { proxy_pass http://registry_metrics; } + proxy_pass http://core_metrics; + } + } + {% endif %} } diff --git a/make/photon/prepare/templates/registry/config.yml.jinja b/make/photon/prepare/templates/registry/config.yml.jinja index 0617c2e3a..6831daad3 100644 --- a/make/photon/prepare/templates/registry/config.yml.jinja +++ b/make/photon/prepare/templates/registry/config.yml.jinja @@ -41,7 +41,14 @@ http: {% endif %} secret: placeholder debug: +{% if metric.enabled %} + addr: :{{ metric.port }} + prometheus: + enabled: true + path: {{ metric.path }} +{% else %} addr: localhost:5001 +{% endif %} auth: htpasswd: realm: harbor-registry-basic-realm diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index 871793e1d..ef33c243d 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -3,7 +3,7 @@ import os import yaml from urllib.parse import urlencode from g import versions_file_path, host_root_dir, DEFAULT_UID, INTERNAL_NO_PROXY_DN -from models import InternalTLS +from models import InternalTLS, Metric from utils.misc import generate_random_string, owner_can_read, other_can_read default_db_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns @@ -346,6 +346,13 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit else: config_dict['internal_tls'] = InternalTLS() + # metric configs + metric_config = configs.get('metric') + if metric_config: + config_dict['metric'] = Metric(metric_config['enabled'], metric_config['port'], metric_config['path']) + else: + config_dict['metric'] = Metric() + if config_dict['internal_tls'].enabled: config_dict['portal_url'] = 'https://portal:8443' config_dict['registry_url'] = 'https://registry:5443' diff --git a/make/photon/prepare/utils/docker_compose.py b/make/photon/prepare/utils/docker_compose.py index c6c0a8e86..ce6ba31fe 100644 --- a/make/photon/prepare/utils/docker_compose.py +++ b/make/photon/prepare/utils/docker_compose.py @@ -61,4 +61,9 @@ def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_ch if log_ep_host: rendering_variables['external_log_endpoint'] = True + # for metrics + metric = configs.get('metric') + if metric: + rendering_variables['metric'] = metric + render_jinja(docker_compose_template_path, docker_compose_yml_path, mode=0o644, **rendering_variables) diff --git a/make/photon/prepare/utils/nginx.py b/make/photon/prepare/utils/nginx.py index f2b96087e..54d4305d4 100644 --- a/make/photon/prepare/utils/nginx.py +++ b/make/photon/prepare/utils/nginx.py @@ -62,7 +62,8 @@ def render_nginx_template(config_dict): https_redirect='$host' + ('https_port' in config_dict and (":" + str(config_dict['https_port'])) or ""), ssl_cert=SSL_CERT_PATH, ssl_cert_key=SSL_CERT_KEY_PATH, - internal_tls=config_dict['internal_tls']) + internal_tls=config_dict['internal_tls'], + metric=config_dict['metric']) location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS else: @@ -71,7 +72,8 @@ def render_nginx_template(config_dict): nginx_conf, uid=DEFAULT_UID, gid=DEFAULT_GID, - internal_tls=config_dict['internal_tls']) + internal_tls=config_dict['internal_tls'], + metric=config_dict['metric']) location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTP copy_nginx_location_configs_if_exist(nginx_template_ext_dir, nginx_confd_dir, location_file_pattern) diff --git a/src/common/config/metadata/metadatalist.go b/src/common/config/metadata/metadatalist.go index 694cb1ae1..b9760f56d 100644 --- a/src/common/config/metadata/metadatalist.go +++ b/src/common/config/metadata/metadatalist.go @@ -152,6 +152,9 @@ var ( // the unit of expiration is minute, 43200 minutes = 30 days {Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true}, {Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, + {Name: common.MetricEnable, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_ENABLE", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, + {Name: common.MetricPort, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PORT", DefaultValue: "9090", ItemType: &IntType{}, Editable: true}, + {Name: common.MetricPath, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PATH", DefaultValue: "/metrics", ItemType: &StringType{}, Editable: true}, {Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, {Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, diff --git a/src/common/const.go b/src/common/const.go index 47b17ba8d..a40ee470d 100755 --- a/src/common/const.go +++ b/src/common/const.go @@ -157,4 +157,9 @@ const ( // DefaultGCTimeWindowHours is the reserve blob time window used by GC, default is 2 hours DefaultGCTimeWindowHours = int64(2) + + // Metric setting items + MetricEnable = "metric_enable" + MetricPort = "metric_port" + MetricPath = "metric_path" ) diff --git a/src/common/models/metric.go b/src/common/models/metric.go new file mode 100644 index 000000000..c5b184808 --- /dev/null +++ b/src/common/models/metric.go @@ -0,0 +1,22 @@ +// Copyright Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +// Metric wraps the configurations to access UAA service +type Metric struct { + Enabled bool + Port int + Path string +} diff --git a/src/core/config/config.go b/src/core/config/config.go index 6cdc8c53e..1f7eb215c 100755 --- a/src/core/config/config.go +++ b/src/core/config/config.go @@ -487,3 +487,12 @@ func GetGCTimeWindow() int64 { } return common.DefaultGCTimeWindowHours } + +// Metric returns the overall metric settings +func Metric() *models.Metric { + return &models.Metric{ + Enabled: cfgMgr.Get(common.MetricEnable).GetBool(), + Port: cfgMgr.Get(common.MetricPort).GetInt(), + Path: cfgMgr.Get(common.MetricPath).GetString(), + } +} diff --git a/src/core/main.go b/src/core/main.go index e83abe6c5..bccae7221 100755 --- a/src/core/main.go +++ b/src/core/main.go @@ -43,6 +43,7 @@ import ( "github.com/goharbor/harbor/src/core/middlewares" "github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/lib/log" + "github.com/goharbor/harbor/src/lib/metric" "github.com/goharbor/harbor/src/migration" "github.com/goharbor/harbor/src/pkg/notification" _ "github.com/goharbor/harbor/src/pkg/notifier/topic" @@ -162,6 +163,11 @@ func main() { log.Info("initializing configurations...") config.Init() log.Info("configurations initialization completed") + metricCfg := config.Metric() + if metricCfg.Enabled { + metric.RegisterCollectors() + go metric.ServeProm(metricCfg.Path, metricCfg.Port) + } token.InitCreators() database, err := config.Database() if err != nil { diff --git a/src/core/middlewares/middlewares.go b/src/core/middlewares/middlewares.go index e029d8c76..ef04bf24e 100644 --- a/src/core/middlewares/middlewares.go +++ b/src/core/middlewares/middlewares.go @@ -24,6 +24,7 @@ import ( "github.com/goharbor/harbor/src/server/middleware/artifactinfo" "github.com/goharbor/harbor/src/server/middleware/csrf" "github.com/goharbor/harbor/src/server/middleware/log" + "github.com/goharbor/harbor/src/server/middleware/metric" "github.com/goharbor/harbor/src/server/middleware/notification" "github.com/goharbor/harbor/src/server/middleware/orm" "github.com/goharbor/harbor/src/server/middleware/readonly" @@ -68,6 +69,7 @@ var ( // MiddleWares returns global middlewares func MiddleWares() []beego.MiddleWare { return []beego.MiddleWare{ + metric.Middleware(), requestid.Middleware(), log.Middleware(), session.Middleware(), diff --git a/src/go.mod b/src/go.mod index daa626649..9a062ced3 100644 --- a/src/go.mod +++ b/src/go.mod @@ -59,6 +59,7 @@ require ( github.com/opentracing/opentracing-go v1.1.0 // indirect github.com/pkg/errors v0.9.1 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect + github.com/prometheus/client_golang v1.0.0 github.com/robfig/cron v1.0.0 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect github.com/spf13/viper v1.4.0 // indirect @@ -67,7 +68,7 @@ require ( golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 // indirect + gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect diff --git a/src/lib/metric/collector.go b/src/lib/metric/collector.go new file mode 100644 index 000000000..2bf4b165d --- /dev/null +++ b/src/lib/metric/collector.go @@ -0,0 +1,51 @@ +package metric + +import ( + "os" + + "github.com/prometheus/client_golang/prometheus" +) + +// RegisterCollectors register all the common static collector +func RegisterCollectors() { + prometheus.MustRegister([]prometheus.Collector{ + TotalInFlightGauge, + TotalReqCnt, + TotalReqDurSummary, + }...) +} + +var ( + // TotalInFlightGauge used to collect total in flight number + TotalInFlightGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: os.Getenv(NamespaceEnvKey), + Subsystem: os.Getenv(SubsystemEnvKey), + Name: "http_request_inflight", + Help: "The total number of requests", + }, + []string{"url"}, + ) + + // TotalReqCnt used to collect total request counter + TotalReqCnt = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: os.Getenv(NamespaceEnvKey), + Subsystem: os.Getenv(SubsystemEnvKey), + Name: "http_request", + Help: "The total number of requests", + }, + []string{"method", "code", "url"}, + ) + + // TotalReqDurSummary used to collect total request duration summaries + TotalReqDurSummary = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: os.Getenv(NamespaceEnvKey), + Subsystem: os.Getenv(SubsystemEnvKey), + Name: "http_request_duration_seconds", + Help: "The time duration of the requests", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, + []string{"method", "url"}) +) diff --git a/src/lib/metric/server.go b/src/lib/metric/server.go new file mode 100644 index 000000000..05b1b6303 --- /dev/null +++ b/src/lib/metric/server.go @@ -0,0 +1,24 @@ +package metric + +import ( + "fmt" + "net/http" + + "github.com/goharbor/harbor/src/lib/log" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +const ( + // NamespaceEnvKey is the metric namespace key in environment + NamespaceEnvKey = "METRIC_NAMESPACE" + // SubsystemEnvKey is the metric subsystem key in environment + SubsystemEnvKey = "METRIC_SUBSYSTEM" +) + +// ServeProm return a server to serve prometheus metrics +func ServeProm(path string, port int) { + mux := http.NewServeMux() + mux.Handle(path, promhttp.Handler()) + log.Infof("Prometheus metric server running on port %v", port) + log.Errorf("Promethus metrcis server down with %s", http.ListenAndServe(fmt.Sprintf(":%v", port), mux)) +} diff --git a/src/server/middleware/metric/metric.go b/src/server/middleware/metric/metric.go new file mode 100644 index 000000000..1feccc855 --- /dev/null +++ b/src/server/middleware/metric/metric.go @@ -0,0 +1,44 @@ +package metric + +import ( + "net/http" + "strconv" + "time" + + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/lib/metric" +) + +func instrumentHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + now, url, code := time.Now(), r.URL.EscapedPath(), "0" + + metric.TotalInFlightGauge.WithLabelValues(url).Inc() + defer metric.TotalInFlightGauge.WithLabelValues(url).Dec() + + next.ServeHTTP(w, r) + if r.Response != nil { + code = strconv.Itoa(r.Response.StatusCode) + } + + metric.TotalReqDurSummary.WithLabelValues(r.Method, url).Observe(time.Since(now).Seconds()) + metric.TotalReqCnt.WithLabelValues(r.Method, code, url).Inc() + }) +} + +// Middleware returns a middleware for handling requests +func Middleware() func(http.Handler) http.Handler { + if config.Metric().Enabled { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + next = instrumentHandler(next) + next.ServeHTTP(rw, req) + }) + } + } + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + next.ServeHTTP(rw, req) + }) + } +} diff --git a/src/vendor/modules.txt b/src/vendor/modules.txt index 03bdee75c..8e102a8c5 100644 --- a/src/vendor/modules.txt +++ b/src/vendor/modules.txt @@ -427,6 +427,7 @@ github.com/pmezard/go-difflib/difflib github.com/pquerna/cachecontrol github.com/pquerna/cachecontrol/cacheobject # github.com/prometheus/client_golang v1.0.0 +## explicit github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/internal github.com/prometheus/client_golang/prometheus/promhttp