feat: add prometheus metrics (#342)

cmd/mc-router/main.go

@@ -45,7 +45,7 @@ type Config struct {
 	DockerSocket          string            `default:"unix:///var/run/docker.sock" usage:"Path to Docker socket to use"`
 	DockerTimeout         int               `default:"0" usage:"Timeout configuration in seconds for the Docker integrations"`
 	DockerRefreshInterval int               `default:"15" usage:"Refresh interval in seconds for the Docker integrations"`
-	MetricsBackend        string            `default:"discard" usage:"Backend to use for metrics exposure/publishing: discard,expvar,influxdb"`
+	MetricsBackend        string            `default:"discard" usage:"Backend to use for metrics exposure/publishing: discard,expvar,influxdb,prometheus"`
 	UseProxyProtocol      bool              `default:"false" usage:"Send PROXY protocol to backend servers"`
 	ReceiveProxyProtocol  bool              `default:"false" usage:"Receive PROXY protocol from backend servers, by default trusts every proxy header that it receives, combine with -trusted-proxies to specify a list of trusted proxies"`
 	TrustedProxies        []string          `usage:"Comma delimited list of CIDR notation IP blocks to trust when receiving PROXY protocol"`

cmd/mc-router/metrics.go

@@ -11,8 +11,11 @@ import (
 	discardMetrics "github.com/go-kit/kit/metrics/discard"
 	expvarMetrics "github.com/go-kit/kit/metrics/expvar"
 	kitinflux "github.com/go-kit/kit/metrics/influx"
+	prometheusMetrics "github.com/go-kit/kit/metrics/prometheus"
 	influx "github.com/influxdata/influxdb1-client/v2"
 	"github.com/itzg/mc-router/server"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/sirupsen/logrus"
 )
 
@@ -25,6 +28,8 @@ func NewMetricsBuilder(backend string, config *MetricsBackendConfig) MetricsBuil
 	switch strings.ToLower(backend) {
 	case "expvar":
 		return &expvarMetricsBuilder{}
+	case "prometheus":
+		return &prometheusMetricsBuilder{}
 	case "influxdb":
 		return &influxMetricsBuilder{config: config}
 	default:
@@ -41,11 +46,13 @@ func (b expvarMetricsBuilder) Start(ctx context.Context) error {
 }
 
 func (b expvarMetricsBuilder) BuildConnectorMetrics() *server.ConnectorMetrics {
+	c := expvarMetrics.NewCounter("connections")
 	return &server.ConnectorMetrics{
-		Errors:            expvarMetrics.NewCounter("errors").With("subsystem", "connector"),
+		Errors:              expvarMetrics.NewCounter("errors").With("subsystem", "connector"),
-		BytesTransmitted:  expvarMetrics.NewCounter("bytes"),
+		BytesTransmitted:    expvarMetrics.NewCounter("bytes"),
-		Connections:       expvarMetrics.NewCounter("connections"),
+		ConnectionsFrontend: c,
-		ActiveConnections: expvarMetrics.NewGauge("active_connections"),
+		ConnectionsBackend:  c,
+		ActiveConnections:   expvarMetrics.NewGauge("active_connections"),
 	}
 }
 
@@ -59,10 +66,11 @@ func (b discardMetricsBuilder) Start(ctx context.Context) error {
 
 func (b discardMetricsBuilder) BuildConnectorMetrics() *server.ConnectorMetrics {
 	return &server.ConnectorMetrics{
-		Errors:            discardMetrics.NewCounter(),
+		Errors:              discardMetrics.NewCounter(),
-		BytesTransmitted:  discardMetrics.NewCounter(),
+		BytesTransmitted:    discardMetrics.NewCounter(),
-		Connections:       discardMetrics.NewCounter(),
+		ConnectionsFrontend: discardMetrics.NewCounter(),
-		ActiveConnections: discardMetrics.NewGauge(),
+		ConnectionsBackend:  discardMetrics.NewCounter(),
+		ActiveConnections:   discardMetrics.NewGauge(),
 	}
 }
 
@@ -105,10 +113,58 @@ func (b *influxMetricsBuilder) BuildConnectorMetrics() *server.ConnectorMetrics
 
 	b.metrics = metrics
 
+	c := metrics.NewCounter("mc_router_connections")
 	return &server.ConnectorMetrics{
-		Errors:            metrics.NewCounter("mc_router_errors"),
+		Errors:              metrics.NewCounter("mc_router_errors"),
-		BytesTransmitted:  metrics.NewCounter("mc_router_transmitted_bytes"),
+		BytesTransmitted:    metrics.NewCounter("mc_router_transmitted_bytes"),
-		Connections:       metrics.NewCounter("mc_router_connections"),
+		ConnectionsFrontend: c.With("side", "frontend"),
-		ActiveConnections: metrics.NewGauge("mc_router_connections_active"),
+		ConnectionsBackend:  c.With("side", "backend"),
+		ActiveConnections:   metrics.NewGauge("mc_router_connections_active"),
+	}
+}
+
+type prometheusMetricsBuilder struct {
+}
+
+var pcv *prometheusMetrics.Counter
+
+func (b prometheusMetricsBuilder) Start(ctx context.Context) error {
+
+	// nothing needed
+	return nil
+}
+
+func (b prometheusMetricsBuilder) BuildConnectorMetrics() *server.ConnectorMetrics {
+	pcv = prometheusMetrics.NewCounter(promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "mc_router",
+		Name:      "errors",
+		Help:      "The total number of errors",
+	}, []string{"type"}))
+	return &server.ConnectorMetrics{
+		Errors: pcv,
+		BytesTransmitted: prometheusMetrics.NewCounter(promauto.NewCounterVec(prometheus.CounterOpts{
+			Namespace: "mc_router",
+			Name:      "bytes",
+			Help:      "The total number of bytes transmitted",
+		}, nil)),
+		ConnectionsFrontend: prometheusMetrics.NewCounter(promauto.NewCounterVec(prometheus.CounterOpts{
+			Namespace:   "mc_router",
+			Subsystem:   "frontend",
+			Name:        "connections",
+			Help:        "The total number of connections",
+			ConstLabels: prometheus.Labels{"side": "frontend"},
+		}, nil)),
+		ConnectionsBackend: prometheusMetrics.NewCounter(promauto.NewCounterVec(prometheus.CounterOpts{
+			Namespace:   "mc_router",
+			Subsystem:   "backend",
+			Name:        "connections",
+			Help:        "The total number of backend connections",
+			ConstLabels: prometheus.Labels{"side": "backend"},
+		}, []string{"host"})),
+		ActiveConnections: prometheusMetrics.NewGauge(promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "mc_router",
+			Name:      "active_connections",
+			Help:      "The number of active connections",
+		}, nil)),
 	}
 }

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/juju/ratelimit v1.0.2
 	github.com/pires/go-proxyproto v0.8.0
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.20.5
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
 	golang.ngrok.com/ngrok v1.11.0
@@ -22,6 +23,8 @@ require (
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
@@ -37,11 +40,15 @@ require (
 	github.com/inconshreveable/log15/v3 v3.0.0-testing.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
 	go.opentelemetry.io/otel v1.22.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect
@@ -64,7 +71,7 @@ require (
 	github.com/docker/docker v27.3.1+incompatible
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/go-kit/log v0.2.0 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -83,7 +90,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/net v0.28.0 // indirect
-	golang.org/x/oauth2 v0.8.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/sys v0.24.0 // indirect
 	golang.org/x/term v0.23.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

go.sum

@@ -4,8 +4,12 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VM
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -29,6 +33,8 @@ github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU=
 github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg=
 github.com/go-kit/log v0.2.0 h1:7i2K3eKTos3Vc0enKCfnVcgHh2olr/MyfboYq7cAcFw=
 github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
+github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
+github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -95,6 +101,8 @@ github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI=
 github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -137,6 +145,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@@ -197,6 +213,8 @@ golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
 golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
 golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
 golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
+golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
+golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

server/api_server.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 )
 
@@ -15,6 +16,8 @@ func StartApiServer(apiBinding string) {
 
 	apiRoutes.Path("/vars").Handler(expvar.Handler())
 
+	apiRoutes.Path("/metrics").Handler(promhttp.Handler())
+
 	go func() {
 		logrus.WithError(
 			http.ListenAndServe(apiBinding, apiRoutes)).Error("API server failed")

server/connector.go

@@ -26,10 +26,11 @@ const (
 var noDeadline time.Time
 
 type ConnectorMetrics struct {
-	Errors            metrics.Counter
+	Errors              metrics.Counter
-	BytesTransmitted  metrics.Counter
+	BytesTransmitted    metrics.Counter
-	Connections       metrics.Counter
+	ConnectionsFrontend metrics.Counter
-	ActiveConnections metrics.Gauge
+	ConnectionsBackend  metrics.Counter
+	ActiveConnections   metrics.Gauge
 }
 
 func NewConnector(metrics *ConnectorMetrics, sendProxyProto bool, receiveProxyProto bool, trustedProxyNets []*net.IPNet) *Connector {
@@ -158,7 +159,7 @@ func (c *Connector) acceptConnections(ctx context.Context, ln net.Listener, conn
 }
 
 func (c *Connector) HandleConnection(ctx context.Context, frontendConn net.Conn) {
-	c.metrics.Connections.With("side", "frontend").Add(1)
+	c.metrics.ConnectionsFrontend.Add(1)
 	//noinspection GoUnhandledErrorResult
 	defer frontendConn.Close()
 
@@ -276,7 +277,7 @@ func (c *Connector) findAndConnectBackend(ctx context.Context, frontendConn net.
 		return
 	}
 
-	c.metrics.Connections.With("side", "backend", "host", resolvedHost).Add(1)
+	c.metrics.ConnectionsBackend.With("host", resolvedHost).Add(1)
 
 	c.metrics.ActiveConnections.Set(float64(
 		atomic.AddInt32(&c.activeConnections, 1)))