Add metrics to Harbor Core

1. Add configs in prepare
 2. Add models and config items in Core
 3. Encapdulate getting metric in commom package
 4. Add a middleware for global request to collect 3 metrics

Signed-off-by: DQ <dengq@vmware.com>
This commit is contained in:
DQ 2020-10-19 00:16:02 +08:00
parent 99d818f4db
commit eb470501be
21 changed files with 250 additions and 6 deletions

View File

@ -208,3 +208,8 @@ proxy:
- jobservice - jobservice
- clair - clair
- trivy - trivy
# metric:
# enabled: false
# port: 9090
# path: /metrics

View File

@ -139,3 +139,8 @@ class InternalTLS:
os.chown(file, DEFAULT_UID, DEFAULT_GID) os.chown(file, DEFAULT_UID, DEFAULT_GID)
class Metric:
def __init__(self, enabled: bool = False, port: int = 8080, path: str = "metrics" ):
self.enabled = enabled
self.port = port
self.path = path

View File

@ -56,3 +56,11 @@ INTERNAL_TLS_TRUST_CA_PATH=/harbor_cust_cert/harbor_internal_ca.crt
{% else %} {% else %}
PORT=8080 PORT=8080
{% endif %} {% endif %}
{% if metric.enabled %}
METRIC_ENABLE=true
METRIC_PATH={{ metric.path }}
METRIC_PORT={{ metric.port }}
METRIC_NAMESPACE=harbor
METRIC_SUBSYSTEM=core
{% endif %}

View File

@ -370,6 +370,9 @@ services:
{% if protocol == 'https' %} {% if protocol == 'https' %}
- {{https_port}}:8443 - {{https_port}}:8443
{% endif %} {% endif %}
{% if metric.enabled %}
- {{metric.port}}:9090
{% endif %}
{% if with_notary %} {% if with_notary %}
- 4443:4443 - 4443:4443
{% endif %} {% endif %}

View File

@ -208,4 +208,21 @@ http {
return 404; return 404;
} }
} }
{% if metric.enabled %}
upstream core_metrics {
server core:9090;
}
upstream registry_metrics {
server registry:5001;
}
server {
listen 9090;
location = /metrics {
if ($arg_comp = core) { proxy_pass http://core_metrics; }
if ($arg_comp = registry) { proxy_pass http://registry_metrics; }
proxy_pass http://core_metrics;
}
}
{% endif %}
} }

View File

@ -240,4 +240,21 @@ http {
#server_name harbordomain.com; #server_name harbordomain.com;
return 308 https://{{https_redirect}}$request_uri; return 308 https://{{https_redirect}}$request_uri;
} }
{% if metric.enabled %}
upstream core_metrics {
server core:{{ metric.port }};
}
upstream registry_metrics {
server registry:{{ metric.port }};
}
server {
listen 9090;
location = {{ metric.path }} {
if ($arg_comp = core) { proxy_pass http://core_metrics; }
if ($arg_comp = registry) { proxy_pass http://registry_metrics; }
proxy_pass http://core_metrics;
}
}
{% endif %}
} }

View File

@ -41,7 +41,14 @@ http:
{% endif %} {% endif %}
secret: placeholder secret: placeholder
debug: debug:
{% if metric.enabled %}
addr: :{{ metric.port }}
prometheus:
enabled: true
path: {{ metric.path }}
{% else %}
addr: localhost:5001 addr: localhost:5001
{% endif %}
auth: auth:
htpasswd: htpasswd:
realm: harbor-registry-basic-realm realm: harbor-registry-basic-realm

View File

@ -3,7 +3,7 @@ import os
import yaml import yaml
from urllib.parse import urlencode from urllib.parse import urlencode
from g import versions_file_path, host_root_dir, DEFAULT_UID, INTERNAL_NO_PROXY_DN from g import versions_file_path, host_root_dir, DEFAULT_UID, INTERNAL_NO_PROXY_DN
from models import InternalTLS from models import InternalTLS, Metric
from utils.misc import generate_random_string, owner_can_read, other_can_read from utils.misc import generate_random_string, owner_can_read, other_can_read
default_db_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns default_db_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
@ -346,6 +346,13 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
else: else:
config_dict['internal_tls'] = InternalTLS() config_dict['internal_tls'] = InternalTLS()
# metric configs
metric_config = configs.get('metric')
if metric_config:
config_dict['metric'] = Metric(metric_config['enabled'], metric_config['port'], metric_config['path'])
else:
config_dict['metric'] = Metric()
if config_dict['internal_tls'].enabled: if config_dict['internal_tls'].enabled:
config_dict['portal_url'] = 'https://portal:8443' config_dict['portal_url'] = 'https://portal:8443'
config_dict['registry_url'] = 'https://registry:5443' config_dict['registry_url'] = 'https://registry:5443'

View File

@ -61,4 +61,9 @@ def prepare_docker_compose(configs, with_clair, with_trivy, with_notary, with_ch
if log_ep_host: if log_ep_host:
rendering_variables['external_log_endpoint'] = True rendering_variables['external_log_endpoint'] = True
# for metrics
metric = configs.get('metric')
if metric:
rendering_variables['metric'] = metric
render_jinja(docker_compose_template_path, docker_compose_yml_path, mode=0o644, **rendering_variables) render_jinja(docker_compose_template_path, docker_compose_yml_path, mode=0o644, **rendering_variables)

View File

@ -62,7 +62,8 @@ def render_nginx_template(config_dict):
https_redirect='$host' + ('https_port' in config_dict and (":" + str(config_dict['https_port'])) or ""), https_redirect='$host' + ('https_port' in config_dict and (":" + str(config_dict['https_port'])) or ""),
ssl_cert=SSL_CERT_PATH, ssl_cert=SSL_CERT_PATH,
ssl_cert_key=SSL_CERT_KEY_PATH, ssl_cert_key=SSL_CERT_KEY_PATH,
internal_tls=config_dict['internal_tls']) internal_tls=config_dict['internal_tls'],
metric=config_dict['metric'])
location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTPS
else: else:
@ -71,7 +72,8 @@ def render_nginx_template(config_dict):
nginx_conf, nginx_conf,
uid=DEFAULT_UID, uid=DEFAULT_UID,
gid=DEFAULT_GID, gid=DEFAULT_GID,
internal_tls=config_dict['internal_tls']) internal_tls=config_dict['internal_tls'],
metric=config_dict['metric'])
location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTP location_file_pattern = CUSTOM_NGINX_LOCATION_FILE_PATTERN_HTTP
copy_nginx_location_configs_if_exist(nginx_template_ext_dir, nginx_confd_dir, location_file_pattern) copy_nginx_location_configs_if_exist(nginx_template_ext_dir, nginx_confd_dir, location_file_pattern)

View File

@ -152,6 +152,9 @@ var (
// the unit of expiration is minute, 43200 minutes = 30 days // the unit of expiration is minute, 43200 minutes = 30 days
{Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true}, {Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true},
{Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, {Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true},
{Name: common.MetricEnable, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_ENABLE", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
{Name: common.MetricPort, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PORT", DefaultValue: "9090", ItemType: &IntType{}, Editable: true},
{Name: common.MetricPath, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PATH", DefaultValue: "/metrics", ItemType: &StringType{}, Editable: true},
{Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, {Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true},
{Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, {Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true},

View File

@ -157,4 +157,9 @@ const (
// DefaultGCTimeWindowHours is the reserve blob time window used by GC, default is 2 hours // DefaultGCTimeWindowHours is the reserve blob time window used by GC, default is 2 hours
DefaultGCTimeWindowHours = int64(2) DefaultGCTimeWindowHours = int64(2)
// Metric setting items
MetricEnable = "metric_enable"
MetricPort = "metric_port"
MetricPath = "metric_path"
) )

View File

@ -0,0 +1,22 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
// Metric wraps the configurations to access UAA service
type Metric struct {
Enabled bool
Port int
Path string
}

View File

@ -487,3 +487,12 @@ func GetGCTimeWindow() int64 {
} }
return common.DefaultGCTimeWindowHours return common.DefaultGCTimeWindowHours
} }
// Metric returns the overall metric settings
func Metric() *models.Metric {
return &models.Metric{
Enabled: cfgMgr.Get(common.MetricEnable).GetBool(),
Port: cfgMgr.Get(common.MetricPort).GetInt(),
Path: cfgMgr.Get(common.MetricPath).GetString(),
}
}

View File

@ -43,6 +43,7 @@ import (
"github.com/goharbor/harbor/src/core/middlewares" "github.com/goharbor/harbor/src/core/middlewares"
"github.com/goharbor/harbor/src/core/service/token" "github.com/goharbor/harbor/src/core/service/token"
"github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/metric"
"github.com/goharbor/harbor/src/migration" "github.com/goharbor/harbor/src/migration"
"github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/notification"
_ "github.com/goharbor/harbor/src/pkg/notifier/topic" _ "github.com/goharbor/harbor/src/pkg/notifier/topic"
@ -162,6 +163,11 @@ func main() {
log.Info("initializing configurations...") log.Info("initializing configurations...")
config.Init() config.Init()
log.Info("configurations initialization completed") log.Info("configurations initialization completed")
metricCfg := config.Metric()
if metricCfg.Enabled {
metric.RegisterCollectors()
go metric.ServeProm(metricCfg.Path, metricCfg.Port)
}
token.InitCreators() token.InitCreators()
database, err := config.Database() database, err := config.Database()
if err != nil { if err != nil {

View File

@ -24,6 +24,7 @@ import (
"github.com/goharbor/harbor/src/server/middleware/artifactinfo" "github.com/goharbor/harbor/src/server/middleware/artifactinfo"
"github.com/goharbor/harbor/src/server/middleware/csrf" "github.com/goharbor/harbor/src/server/middleware/csrf"
"github.com/goharbor/harbor/src/server/middleware/log" "github.com/goharbor/harbor/src/server/middleware/log"
"github.com/goharbor/harbor/src/server/middleware/metric"
"github.com/goharbor/harbor/src/server/middleware/notification" "github.com/goharbor/harbor/src/server/middleware/notification"
"github.com/goharbor/harbor/src/server/middleware/orm" "github.com/goharbor/harbor/src/server/middleware/orm"
"github.com/goharbor/harbor/src/server/middleware/readonly" "github.com/goharbor/harbor/src/server/middleware/readonly"
@ -68,6 +69,7 @@ var (
// MiddleWares returns global middlewares // MiddleWares returns global middlewares
func MiddleWares() []beego.MiddleWare { func MiddleWares() []beego.MiddleWare {
return []beego.MiddleWare{ return []beego.MiddleWare{
metric.Middleware(),
requestid.Middleware(), requestid.Middleware(),
log.Middleware(), log.Middleware(),
session.Middleware(), session.Middleware(),

View File

@ -59,6 +59,7 @@ require (
github.com/opentracing/opentracing-go v1.1.0 // indirect github.com/opentracing/opentracing-go v1.1.0 // indirect
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
github.com/prometheus/client_golang v1.0.0
github.com/robfig/cron v1.0.0 github.com/robfig/cron v1.0.0
github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect
github.com/spf13/viper v1.4.0 // indirect github.com/spf13/viper v1.4.0 // indirect
@ -67,7 +68,7 @@ require (
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 golang.org/x/net v0.0.0-20200202094626-16171245cfb2
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175
gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect
gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect
gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect

View File

@ -0,0 +1,51 @@
package metric
import (
"os"
"github.com/prometheus/client_golang/prometheus"
)
// RegisterCollectors register all the common static collector
func RegisterCollectors() {
prometheus.MustRegister([]prometheus.Collector{
TotalInFlightGauge,
TotalReqCnt,
TotalReqDurSummary,
}...)
}
var (
// TotalInFlightGauge used to collect total in flight number
TotalInFlightGauge = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: os.Getenv(NamespaceEnvKey),
Subsystem: os.Getenv(SubsystemEnvKey),
Name: "http_request_inflight",
Help: "The total number of requests",
},
[]string{"url"},
)
// TotalReqCnt used to collect total request counter
TotalReqCnt = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: os.Getenv(NamespaceEnvKey),
Subsystem: os.Getenv(SubsystemEnvKey),
Name: "http_request",
Help: "The total number of requests",
},
[]string{"method", "code", "url"},
)
// TotalReqDurSummary used to collect total request duration summaries
TotalReqDurSummary = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: os.Getenv(NamespaceEnvKey),
Subsystem: os.Getenv(SubsystemEnvKey),
Name: "http_request_duration_seconds",
Help: "The time duration of the requests",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
[]string{"method", "url"})
)

24
src/lib/metric/server.go Normal file
View File

@ -0,0 +1,24 @@
package metric
import (
"fmt"
"net/http"
"github.com/goharbor/harbor/src/lib/log"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
const (
// NamespaceEnvKey is the metric namespace key in environment
NamespaceEnvKey = "METRIC_NAMESPACE"
// SubsystemEnvKey is the metric subsystem key in environment
SubsystemEnvKey = "METRIC_SUBSYSTEM"
)
// ServeProm return a server to serve prometheus metrics
func ServeProm(path string, port int) {
mux := http.NewServeMux()
mux.Handle(path, promhttp.Handler())
log.Infof("Prometheus metric server running on port %v", port)
log.Errorf("Promethus metrcis server down with %s", http.ListenAndServe(fmt.Sprintf(":%v", port), mux))
}

View File

@ -0,0 +1,44 @@
package metric
import (
"net/http"
"strconv"
"time"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/metric"
)
func instrumentHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
now, url, code := time.Now(), r.URL.EscapedPath(), "0"
metric.TotalInFlightGauge.WithLabelValues(url).Inc()
defer metric.TotalInFlightGauge.WithLabelValues(url).Dec()
next.ServeHTTP(w, r)
if r.Response != nil {
code = strconv.Itoa(r.Response.StatusCode)
}
metric.TotalReqDurSummary.WithLabelValues(r.Method, url).Observe(time.Since(now).Seconds())
metric.TotalReqCnt.WithLabelValues(r.Method, code, url).Inc()
})
}
// Middleware returns a middleware for handling requests
func Middleware() func(http.Handler) http.Handler {
if config.Metric().Enabled {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
next = instrumentHandler(next)
next.ServeHTTP(rw, req)
})
}
}
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
next.ServeHTTP(rw, req)
})
}
}

View File

@ -427,6 +427,7 @@ github.com/pmezard/go-difflib/difflib
github.com/pquerna/cachecontrol github.com/pquerna/cachecontrol
github.com/pquerna/cachecontrol/cacheobject github.com/pquerna/cachecontrol/cacheobject
# github.com/prometheus/client_golang v1.0.0 # github.com/prometheus/client_golang v1.0.0
## explicit
github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus
github.com/prometheus/client_golang/prometheus/internal github.com/prometheus/client_golang/prometheus/internal
github.com/prometheus/client_golang/prometheus/promhttp github.com/prometheus/client_golang/prometheus/promhttp