From 2624e25c943adc9ee17a5f54630ed3d0b1fa7c3a Mon Sep 17 00:00:00 2001 From: Slawomir CALUCH Date: Sat, 9 Nov 2024 18:43:51 +0000 Subject: [PATCH] feat: add prometheus metrics (#342) --- cmd/mc-router/main.go | 2 +- cmd/mc-router/metrics.go | 80 ++++++++++++++++++++++++++++++++++------ go.mod | 11 +++++- go.sum | 18 +++++++++ server/api_server.go | 3 ++ server/connector.go | 13 ++++--- 6 files changed, 106 insertions(+), 21 deletions(-) diff --git a/cmd/mc-router/main.go b/cmd/mc-router/main.go index be00bc9..2dcc6a7 100644 --- a/cmd/mc-router/main.go +++ b/cmd/mc-router/main.go @@ -45,7 +45,7 @@ type Config struct { DockerSocket string `default:"unix:///var/run/docker.sock" usage:"Path to Docker socket to use"` DockerTimeout int `default:"0" usage:"Timeout configuration in seconds for the Docker integrations"` DockerRefreshInterval int `default:"15" usage:"Refresh interval in seconds for the Docker integrations"` - MetricsBackend string `default:"discard" usage:"Backend to use for metrics exposure/publishing: discard,expvar,influxdb"` + MetricsBackend string `default:"discard" usage:"Backend to use for metrics exposure/publishing: discard,expvar,influxdb,prometheus"` UseProxyProtocol bool `default:"false" usage:"Send PROXY protocol to backend servers"` ReceiveProxyProtocol bool `default:"false" usage:"Receive PROXY protocol from backend servers, by default trusts every proxy header that it receives, combine with -trusted-proxies to specify a list of trusted proxies"` TrustedProxies []string `usage:"Comma delimited list of CIDR notation IP blocks to trust when receiving PROXY protocol"` diff --git a/cmd/mc-router/metrics.go b/cmd/mc-router/metrics.go index fc6b2b3..9cc4a7c 100644 --- a/cmd/mc-router/metrics.go +++ b/cmd/mc-router/metrics.go @@ -11,8 +11,11 @@ import ( discardMetrics "github.com/go-kit/kit/metrics/discard" expvarMetrics "github.com/go-kit/kit/metrics/expvar" kitinflux "github.com/go-kit/kit/metrics/influx" + prometheusMetrics "github.com/go-kit/kit/metrics/prometheus" influx "github.com/influxdata/influxdb1-client/v2" "github.com/itzg/mc-router/server" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/sirupsen/logrus" ) @@ -25,6 +28,8 @@ func NewMetricsBuilder(backend string, config *MetricsBackendConfig) MetricsBuil switch strings.ToLower(backend) { case "expvar": return &expvarMetricsBuilder{} + case "prometheus": + return &prometheusMetricsBuilder{} case "influxdb": return &influxMetricsBuilder{config: config} default: @@ -41,11 +46,13 @@ func (b expvarMetricsBuilder) Start(ctx context.Context) error { } func (b expvarMetricsBuilder) BuildConnectorMetrics() *server.ConnectorMetrics { + c := expvarMetrics.NewCounter("connections") return &server.ConnectorMetrics{ - Errors: expvarMetrics.NewCounter("errors").With("subsystem", "connector"), - BytesTransmitted: expvarMetrics.NewCounter("bytes"), - Connections: expvarMetrics.NewCounter("connections"), - ActiveConnections: expvarMetrics.NewGauge("active_connections"), + Errors: expvarMetrics.NewCounter("errors").With("subsystem", "connector"), + BytesTransmitted: expvarMetrics.NewCounter("bytes"), + ConnectionsFrontend: c, + ConnectionsBackend: c, + ActiveConnections: expvarMetrics.NewGauge("active_connections"), } } @@ -59,10 +66,11 @@ func (b discardMetricsBuilder) Start(ctx context.Context) error { func (b discardMetricsBuilder) BuildConnectorMetrics() *server.ConnectorMetrics { return &server.ConnectorMetrics{ - Errors: discardMetrics.NewCounter(), - BytesTransmitted: discardMetrics.NewCounter(), - Connections: discardMetrics.NewCounter(), - ActiveConnections: discardMetrics.NewGauge(), + Errors: discardMetrics.NewCounter(), + BytesTransmitted: discardMetrics.NewCounter(), + ConnectionsFrontend: discardMetrics.NewCounter(), + ConnectionsBackend: discardMetrics.NewCounter(), + ActiveConnections: discardMetrics.NewGauge(), } } @@ -105,10 +113,58 @@ func (b *influxMetricsBuilder) BuildConnectorMetrics() *server.ConnectorMetrics b.metrics = metrics + c := metrics.NewCounter("mc_router_connections") return &server.ConnectorMetrics{ - Errors: metrics.NewCounter("mc_router_errors"), - BytesTransmitted: metrics.NewCounter("mc_router_transmitted_bytes"), - Connections: metrics.NewCounter("mc_router_connections"), - ActiveConnections: metrics.NewGauge("mc_router_connections_active"), + Errors: metrics.NewCounter("mc_router_errors"), + BytesTransmitted: metrics.NewCounter("mc_router_transmitted_bytes"), + ConnectionsFrontend: c.With("side", "frontend"), + ConnectionsBackend: c.With("side", "backend"), + ActiveConnections: metrics.NewGauge("mc_router_connections_active"), + } +} + +type prometheusMetricsBuilder struct { +} + +var pcv *prometheusMetrics.Counter + +func (b prometheusMetricsBuilder) Start(ctx context.Context) error { + + // nothing needed + return nil +} + +func (b prometheusMetricsBuilder) BuildConnectorMetrics() *server.ConnectorMetrics { + pcv = prometheusMetrics.NewCounter(promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mc_router", + Name: "errors", + Help: "The total number of errors", + }, []string{"type"})) + return &server.ConnectorMetrics{ + Errors: pcv, + BytesTransmitted: prometheusMetrics.NewCounter(promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mc_router", + Name: "bytes", + Help: "The total number of bytes transmitted", + }, nil)), + ConnectionsFrontend: prometheusMetrics.NewCounter(promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mc_router", + Subsystem: "frontend", + Name: "connections", + Help: "The total number of connections", + ConstLabels: prometheus.Labels{"side": "frontend"}, + }, nil)), + ConnectionsBackend: prometheusMetrics.NewCounter(promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "mc_router", + Subsystem: "backend", + Name: "connections", + Help: "The total number of backend connections", + ConstLabels: prometheus.Labels{"side": "backend"}, + }, []string{"host"})), + ActiveConnections: prometheusMetrics.NewGauge(promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "mc_router", + Name: "active_connections", + Help: "The number of active connections", + }, nil)), } } diff --git a/go.mod b/go.mod index ebd5b62..b160130 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/juju/ratelimit v1.0.2 github.com/pires/go-proxyproto v0.8.0 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.20.5 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 golang.ngrok.com/ngrok v1.11.0 @@ -22,6 +23,8 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/distribution/reference v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect @@ -37,11 +40,15 @@ require ( github.com/inconshreveable/log15/v3 v3.0.0-testing.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect go.opentelemetry.io/otel v1.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect @@ -64,7 +71,7 @@ require ( github.com/docker/docker v27.3.1+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/go-kit/log v0.2.0 // indirect + github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -83,7 +90,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.28.0 // indirect - golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sys v0.24.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index e8d90aa..6b27ead 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,12 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VM github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -29,6 +33,8 @@ github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg= github.com/go-kit/log v0.2.0 h1:7i2K3eKTos3Vc0enKCfnVcgHh2olr/MyfboYq7cAcFw= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -95,6 +101,8 @@ github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -137,6 +145,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -197,6 +213,8 @@ golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/server/api_server.go b/server/api_server.go index be9528a..afd8ef5 100644 --- a/server/api_server.go +++ b/server/api_server.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" ) @@ -15,6 +16,8 @@ func StartApiServer(apiBinding string) { apiRoutes.Path("/vars").Handler(expvar.Handler()) + apiRoutes.Path("/metrics").Handler(promhttp.Handler()) + go func() { logrus.WithError( http.ListenAndServe(apiBinding, apiRoutes)).Error("API server failed") diff --git a/server/connector.go b/server/connector.go index a63e27e..a31b694 100644 --- a/server/connector.go +++ b/server/connector.go @@ -26,10 +26,11 @@ const ( var noDeadline time.Time type ConnectorMetrics struct { - Errors metrics.Counter - BytesTransmitted metrics.Counter - Connections metrics.Counter - ActiveConnections metrics.Gauge + Errors metrics.Counter + BytesTransmitted metrics.Counter + ConnectionsFrontend metrics.Counter + ConnectionsBackend metrics.Counter + ActiveConnections metrics.Gauge } func NewConnector(metrics *ConnectorMetrics, sendProxyProto bool, receiveProxyProto bool, trustedProxyNets []*net.IPNet) *Connector { @@ -158,7 +159,7 @@ func (c *Connector) acceptConnections(ctx context.Context, ln net.Listener, conn } func (c *Connector) HandleConnection(ctx context.Context, frontendConn net.Conn) { - c.metrics.Connections.With("side", "frontend").Add(1) + c.metrics.ConnectionsFrontend.Add(1) //noinspection GoUnhandledErrorResult defer frontendConn.Close() @@ -276,7 +277,7 @@ func (c *Connector) findAndConnectBackend(ctx context.Context, frontendConn net. return } - c.metrics.Connections.With("side", "backend", "host", resolvedHost).Add(1) + c.metrics.ConnectionsBackend.With("host", resolvedHost).Add(1) c.metrics.ActiveConnections.Set(float64( atomic.AddInt32(&c.activeConnections, 1)))