diff --git a/Dockerfile b/Dockerfile index d3061dc..6e230bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20 as builder +FROM golang:1.22 AS builder WORKDIR /build diff --git a/README.md b/README.md index 8ae4da0..010432f 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,16 @@ Routes Minecraft client connections to backend servers based upon the requested Enable debug logs (env DEBUG) -default string host:port of a default Minecraft server to use when mapping not found (env DEFAULT) + -docker-socket + Path to Docker socket to use (env DOCKER_SOCKET) (default "unix:///var/run/docker.sock") -docker-refresh-interval int Refresh interval in seconds for the Docker Swarm integration (env DOCKER_REFRESH_INTERVAL) (default 15) -docker-timeout int Timeout configuration in seconds for the Docker Swarm integration (env DOCKER_TIMEOUT) + -in-docker + Use Docker service discovery (env IN_DOCKER) -in-docker-swarm - Use in-swarm Docker config (env IN_DOCKER_SWARM) + Use Docker Swarm service discovery (env IN_DOCKER_SWARM) -in-kube-cluster Use in-cluster Kubernetes config (env IN_KUBE_CLUSTER) -kube-config string @@ -114,6 +118,33 @@ To test out this example, add these two entries to my "hosts" file: 127.0.0.1 forge.example.com ``` +### Using Docker auto-discovery + +When running `mc-router` in a Docker environment you can pass the `--in-docker` or `--in-docker-swarm` +command-line argument and it will poll the Docker API periodically to find all the running +containers/services for Minecraft instances. To enable discovery you have to set the `mc-router.host` +label on the container. These are the labels scanned: + +- `mc-router.host`: Used to configure the hostname the Minecraft clients would use to + connect to the server. The container/service endpoint will be used as the routed backend. You can + use more than one hostname by splitting it with a comma. +- `mc-router.port`: This value must be set to the port the Minecraft server is listening on. + The default value is 25565. +- `mc-router.default`: Set this to a truthy value to make this server the default backend. + Please note that `mc-router.host` is still required to be set. +- `mc-router.network`: Specify the network you are using for the router if multiple are + present in the container/service. You can either use the network ID, it's full name or an alias. + +#### Example Docker deployment + +Refer to [this example docker-compose.yml](docs/sd-docker.docker-compose.yml) to see how to +configure two different Minecraft servers and a `mc-router` instance for use with Docker service discovery. + +#### Example Docker Swarm deployment + +Refer to [this example docker-compose.yml](docs/swarm.docker-compose.yml) to see how to +configure two different Minecraft servers and a `mc-router` instance for use with Docker Swarm service discovery. + ## Routing Configuration The routing configuration allows routing via a config file rather than a command. @@ -213,32 +244,6 @@ rules: verbs: ["watch","list","get","update"] ``` -## Docker Swarm Usage - -### Using Docker Swarm Service auto-discovery - -When running `mc-router` in a Docker Swarm environment you can pass the `--in-docker-swarm` -command-line argument and it will poll the Docker API periodically to find all the running -services for minecraft instances. To enable discovery you have to set the `mc-router.host` -label on the service. These are the labels scanned: - -- `mc-router.host`: Used to configure the hostname the Minecraft clients would use to - connect to the server. The service endpoint will be used as the routed backend. You can - use more than one hostname by splitting it with a comma. -- `mc-router.port`: This value must be set to the port the Minecraft server is listening on. - The default value is 25565. -- `mc-router.default`: Set this to a truthy value to make this server the deafult backend. - Please note that `mc-router.host` is still required to be set. -- `mc-router.network`: Specify the network you are using for the router if multiple are - present in the service. You can either use the network ID, it's full name or an alias. - -### Example Docker Swarm deployment - -Refer to [this example docker-compose.yml](docs/swarm.docker-compose.yml) to see how to -configure two different Minecraft servers and a `mc-router` instance. Notice how you don't -have to expose the Minecraft instances ports, but all the containers are required to be in -the same network. - ## REST API * `GET /routes` (with `Accept: application/json`) @@ -365,4 +370,4 @@ docker run -it --rm \ ## Related Projects -* https://github.com/haveachin/infrared \ No newline at end of file +* https://github.com/haveachin/infrared diff --git a/cmd/mc-router/main.go b/cmd/mc-router/main.go index e02dbad..be00bc9 100644 --- a/cmd/mc-router/main.go +++ b/cmd/mc-router/main.go @@ -40,9 +40,11 @@ type Config struct { InKubeCluster bool `usage:"Use in-cluster Kubernetes config"` KubeConfig string `usage:"The path to a Kubernetes configuration file"` AutoScaleUp bool `usage:"Increase Kubernetes StatefulSet Replicas (only) from 0 to 1 on respective backend servers when accessed"` - InDockerSwarm bool `usage:"Use in-swarm Docker config"` - DockerTimeout int `default:"0" usage:"Timeout configuration in seconds for the Docker Swarm integration"` - DockerRefreshInterval int `default:"15" usage:"Refresh interval in seconds for the Docker Swarm integration"` + InDocker bool `usage:"Use Docker service discovery"` + InDockerSwarm bool `usage:"Use Docker Swarm service discovery"` + DockerSocket string `default:"unix:///var/run/docker.sock" usage:"Path to Docker socket to use"` + DockerTimeout int `default:"0" usage:"Timeout configuration in seconds for the Docker integrations"` + DockerRefreshInterval int `default:"15" usage:"Refresh interval in seconds for the Docker integrations"` MetricsBackend string `default:"discard" usage:"Backend to use for metrics exposure/publishing: discard,expvar,influxdb"` UseProxyProtocol bool `default:"false" usage:"Send PROXY protocol to backend servers"` ReceiveProxyProtocol bool `default:"false" usage:"Receive PROXY protocol from backend servers, by default trusts every proxy header that it receives, combine with -trusted-proxies to specify a list of trusted proxies"` @@ -161,12 +163,21 @@ func main() { } } + if config.InDocker { + err = server.DockerWatcher.Start(config.DockerSocket, config.DockerTimeout, config.DockerRefreshInterval) + if err != nil { + logrus.WithError(err).Fatal("Unable to start docker integration") + } else { + defer server.DockerWatcher.Stop() + } + } + if config.InDockerSwarm { - err = server.DockerWatcher.StartInSwarm(config.DockerTimeout, config.DockerRefreshInterval) + err = server.DockerSwarmWatcher.Start(config.DockerSocket, config.DockerTimeout, config.DockerRefreshInterval) if err != nil { logrus.WithError(err).Fatal("Unable to start docker swarm integration") } else { - defer server.DockerWatcher.Stop() + defer server.DockerSwarmWatcher.Stop() } } diff --git a/docs/sd-docker.docker-compose.yml b/docs/sd-docker.docker-compose.yml new file mode 100644 index 0000000..b40edac --- /dev/null +++ b/docs/sd-docker.docker-compose.yml @@ -0,0 +1,34 @@ +services: + mcfoo: + image: itzg/minecraft-server + environment: + EULA: "TRUE" + MOTD: "foo" + volumes: + - mcfoodata:/data + labels: + mc-router.host: "foo.localhost" + mc-router.default: true + mcbar: + image: itzg/minecraft-server + environment: + EULA: "TRUE" + MOTD: "bar" + volumes: + - mcbardata:/data + labels: + mc-router.host: "bar.localhost" + router: + image: itzg/mc-router:latest + command: "-debug -in-docker" + depends_on: + - mcfoo + - mcbar + ports: + - "25565:25565" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + +volumes: + mcfoodata: + mcbardata: diff --git a/server/docker.go b/server/docker.go index b14a702..87688c4 100644 --- a/server/docker.go +++ b/server/docker.go @@ -3,26 +3,30 @@ package server import ( "context" "fmt" - "net" "strconv" "strings" "sync" "time" dockertypes "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types/swarm" - swarmtypes "github.com/docker/docker/api/types/swarm" - "github.com/docker/docker/api/types/versions" + "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" "github.com/sirupsen/logrus" ) type IDockerWatcher interface { - StartInSwarm(timeoutSeconds int, refreshIntervalSeconds int) error + Start(socket string, timeoutSeconds int, refreshIntervalSeconds int) error Stop() } +const ( + DockerAPIVersion = "1.24" + DockerRouterLabelHost = "mc-router.host" + DockerRouterLabelPort = "mc-router.port" + DockerRouterLabelDefault = "mc-router.default" + DockerRouterLabelNetwork = "mc-router.network" +) + var DockerWatcher IDockerWatcher = &dockerWatcherImpl{} type dockerWatcherImpl struct { @@ -31,29 +35,20 @@ type dockerWatcherImpl struct { contextCancel context.CancelFunc } -const ( - DockerConfigHost = "unix:///var/run/docker.sock" - DockerAPIVersion = "1.24" - DockerRouterLabelHost = "mc-router.host" - DockerRouterLabelPort = "mc-router.port" - DockerRouterLabelDefault = "mc-router.default" - DockerRouterLabelNetwork = "mc-router.network" -) - -func (w *dockerWatcherImpl) makeWakerFunc(service *routableService) func(ctx context.Context) error { +func (w *dockerWatcherImpl) makeWakerFunc(_ *routableContainer) func(ctx context.Context) error { return func(ctx context.Context) error { return nil } } -func (w *dockerWatcherImpl) StartInSwarm(timeoutSeconds int, refreshIntervalSeconds int) error { +func (w *dockerWatcherImpl) Start(socket string, timeoutSeconds int, refreshIntervalSeconds int) error { var err error timeout := time.Duration(timeoutSeconds) * time.Second refreshInterval := time.Duration(refreshIntervalSeconds) * time.Second opts := []client.Opt{ - client.WithHost(DockerConfigHost), + client.WithHost(socket), client.WithTimeout(timeout), client.WithHTTPHeaders(map[string]string{ "User-Agent": "mc-router ", @@ -67,22 +62,22 @@ func (w *dockerWatcherImpl) StartInSwarm(timeoutSeconds int, refreshIntervalSeco } ticker := time.NewTicker(refreshInterval) - serviceMap := map[string]*routableService{} + containerMap := map[string]*routableContainer{} var ctx context.Context ctx, w.contextCancel = context.WithCancel(context.Background()) - initialServices, err := w.listServices(ctx) + initialContainers, err := w.listContainers(ctx) if err != nil { return err } - for _, s := range initialServices { - serviceMap[s.externalServiceName] = s - if s.externalServiceName != "" { - Routes.CreateMapping(s.externalServiceName, s.containerEndpoint, w.makeWakerFunc(s)) + for _, c := range initialContainers { + containerMap[c.externalContainerName] = c + if c.externalContainerName != "" { + Routes.CreateMapping(c.externalContainerName, c.containerEndpoint, w.makeWakerFunc(c)) } else { - Routes.SetDefaultRoute(s.containerEndpoint) + Routes.SetDefaultRoute(c.containerEndpoint) } } @@ -90,43 +85,43 @@ func (w *dockerWatcherImpl) StartInSwarm(timeoutSeconds int, refreshIntervalSeco for { select { case <-ticker.C: - services, err := w.listServices(ctx) + containers, err := w.listContainers(ctx) if err != nil { - logrus.WithError(err).Error("Docker failed to list services") + logrus.WithError(err).Error("Docker failed to list containers") return } visited := map[string]struct{}{} - for _, rs := range services { - if oldRs, ok := serviceMap[rs.externalServiceName]; !ok { - serviceMap[rs.externalServiceName] = rs - logrus.WithField("routableService", rs).Debug("ADD") - if rs.externalServiceName != "" { - Routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, w.makeWakerFunc(rs)) + for _, rs := range containers { + if oldRs, ok := containerMap[rs.externalContainerName]; !ok { + containerMap[rs.externalContainerName] = rs + logrus.WithField("routableContainer", rs).Debug("ADD") + if rs.externalContainerName != "" { + Routes.CreateMapping(rs.externalContainerName, rs.containerEndpoint, w.makeWakerFunc(rs)) } else { Routes.SetDefaultRoute(rs.containerEndpoint) } } else if oldRs.containerEndpoint != rs.containerEndpoint { - serviceMap[rs.externalServiceName] = rs - if rs.externalServiceName != "" { - Routes.DeleteMapping(rs.externalServiceName) - Routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, w.makeWakerFunc(rs)) + containerMap[rs.externalContainerName] = rs + if rs.externalContainerName != "" { + Routes.DeleteMapping(rs.externalContainerName) + Routes.CreateMapping(rs.externalContainerName, rs.containerEndpoint, w.makeWakerFunc(rs)) } else { Routes.SetDefaultRoute(rs.containerEndpoint) } logrus.WithFields(logrus.Fields{"old": oldRs, "new": rs}).Debug("UPDATE") } - visited[rs.externalServiceName] = struct{}{} + visited[rs.externalContainerName] = struct{}{} } - for _, rs := range serviceMap { - if _, ok := visited[rs.externalServiceName]; !ok { - delete(serviceMap, rs.externalServiceName) - if rs.externalServiceName != "" { - Routes.DeleteMapping(rs.externalServiceName) + for _, rs := range containerMap { + if _, ok := visited[rs.externalContainerName]; !ok { + delete(containerMap, rs.externalContainerName) + if rs.externalContainerName != "" { + Routes.DeleteMapping(rs.externalContainerName) } else { Routes.SetDefaultRoute("") } - logrus.WithField("routableService", rs).Debug("DELETE") + logrus.WithField("routableContainer", rs).Debug("DELETE") } } @@ -137,64 +132,33 @@ func (w *dockerWatcherImpl) StartInSwarm(timeoutSeconds int, refreshIntervalSeco } }() - logrus.Info("Monitoring Docker for Minecraft services") + logrus.Info("Monitoring Docker for Minecraft containers") return nil } -func (w *dockerWatcherImpl) listServices(ctx context.Context) ([]*routableService, error) { - services, err := w.client.ServiceList(ctx, dockertypes.ServiceListOptions{}) +func (w *dockerWatcherImpl) listContainers(ctx context.Context) ([]*routableContainer, error) { + containers, err := w.client.ContainerList(ctx, container.ListOptions{}) if err != nil { return nil, err } - serverVersion, err := w.client.ServerVersion(ctx) - if err != nil { - return nil, err - } - - networkListArgs := filters.NewArgs() - // https://docs.docker.com/engine/api/v1.29/#tag/Network (Docker 17.06) - if versions.GreaterThanOrEqualTo(serverVersion.APIVersion, "1.29") { - networkListArgs.Add("scope", "swarm") - } else { - networkListArgs.Add("driver", "overlay") - } - - networkList, err := w.client.NetworkList(ctx, dockertypes.NetworkListOptions{Filters: networkListArgs}) - if err != nil { - return nil, err - } - - networkMap := make(map[string]*dockertypes.NetworkResource) - for _, network := range networkList { - networkToAdd := network - networkMap[network.ID] = &networkToAdd - } - - var result []*routableService - for _, service := range services { - if service.Spec.EndpointSpec.Mode != swarmtypes.ResolutionModeVIP { - continue - } - if len(service.Endpoint.VirtualIPs) == 0 { - continue - } - - data, ok := w.parseServiceData(&service, networkMap) + var result []*routableContainer + for _, container := range containers { + data, ok := w.parseContainerData(&container) if !ok { continue } for _, host := range data.hosts { - result = append(result, &routableService{ - containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), - externalServiceName: host, + result = append(result, &routableContainer{ + containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), + externalContainerName: host, }) } if data.def != nil && *data.def { - result = append(result, &routableService{ - containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), - externalServiceName: "", + result = append(result, &routableContainer{ + containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), + externalContainerName: "", }) } } @@ -202,28 +166,7 @@ func (w *dockerWatcherImpl) listServices(ctx context.Context) ([]*routableServic return result, nil } -func dockerCheckNetworkName(id string, name string, networkMap map[string]*dockertypes.NetworkResource, networkAliases map[string][]string) (bool, error) { - // we allow to specify the id instead - if id == name { - return true, nil - } - if network := networkMap[id]; network != nil { - if network.Name == name { - return true, nil - } - aliases := networkAliases[id] - for _, alias := range aliases { - if alias == name { - return true, nil - } - } - return false, nil - } - - return false, fmt.Errorf("network not found %s", id) -} - -type parsedDockerServiceData struct { +type parsedDockerContainerData struct { hosts []string port uint64 def *bool @@ -231,40 +174,36 @@ type parsedDockerServiceData struct { ip string } -func (w *dockerWatcherImpl) parseServiceData(service *swarm.Service, networkMap map[string]*dockertypes.NetworkResource) (data parsedDockerServiceData, ok bool) { - networkAliases := map[string][]string{} - for _, network := range service.Spec.TaskTemplate.Networks { - networkAliases[network.Target] = network.Aliases - } - - for key, value := range service.Spec.Labels { +func (w *dockerWatcherImpl) parseContainerData(container *dockertypes.Container) (data parsedDockerContainerData, ok bool) { + for key, value := range container.Labels { if key == DockerRouterLabelHost { if data.hosts != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service with duplicate %s", DockerRouterLabelHost) + logrus.WithFields(logrus.Fields{"containerId": container.ID, "containerNames": container.Names}). + Warnf("ignoring container with duplicate %s label", DockerRouterLabelHost) return } data.hosts = strings.Split(value, ",") } + if key == DockerRouterLabelPort { if data.port != 0 { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service with duplicate %s", DockerRouterLabelPort) + logrus.WithFields(logrus.Fields{"containerId": container.ID, "containerNames": container.Names}). + Warnf("ignoring container with duplicate %s label", DockerRouterLabelPort) return } var err error data.port, err = strconv.ParseUint(value, 10, 32) if err != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + logrus.WithFields(logrus.Fields{"containerId": container.ID, "containerNames": container.Names}). WithError(err). - Warnf("ignoring service with invalid %s", DockerRouterLabelPort) + Warnf("ignoring container with invalid %s label", DockerRouterLabelPort) return } } if key == DockerRouterLabelDefault { if data.def != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service with duplicate %s", DockerRouterLabelDefault) + logrus.WithFields(logrus.Fields{"containerId": container.ID, "containerNames": container.Names}). + Warnf("ignoring container with duplicate %s label", DockerRouterLabelDefault) return } data.def = new(bool) @@ -274,8 +213,8 @@ func (w *dockerWatcherImpl) parseServiceData(service *swarm.Service, networkMap } if key == DockerRouterLabelNetwork { if data.network != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service with duplicate %s", DockerRouterLabelNetwork) + logrus.WithFields(logrus.Fields{"containerId": container.ID, "containerNames": container.Names}). + Warnf("ignoring container with duplicate %s label", DockerRouterLabelNetwork) return } data.network = new(string) @@ -288,9 +227,9 @@ func (w *dockerWatcherImpl) parseServiceData(service *swarm.Service, networkMap return } - if len(service.Endpoint.VirtualIPs) == 0 { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service, no VirtualIPs found") + if len(container.NetworkSettings.Networks) == 0 { + logrus.WithFields(logrus.Fields{"containerId": container.ID, "containerNames": container.Names}). + Warnf("ignoring container, no networks found") return } @@ -298,32 +237,50 @@ func (w *dockerWatcherImpl) parseServiceData(service *swarm.Service, networkMap data.port = 25565 } - vipIndex := -1 if data.network != nil { - for i, vip := range service.Endpoint.VirtualIPs { - if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { - vipIndex = i + // Loop through all the container's networks and attempt to find one whose Network ID, Name, or Aliases match the + // specified network + for name, endpoint := range container.NetworkSettings.Networks { + if name == endpoint.NetworkID { + data.ip = endpoint.IPAddress + } + + if name == *data.network { + data.ip = endpoint.IPAddress break - } else if err != nil { - // we intentionally ignore name check errors - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Debugf("%v", err) + } + + for _, alias := range endpoint.Aliases { + if alias == name { + data.ip = endpoint.IPAddress + break + } } } - if vipIndex == -1 { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service, network %s not found", *data.network) + } else { + // If there's no endpoint specified we can just assume the only one is the network we should use. One caveat is + // if there's more than one network on this container, we should require that the user specifies a network to avoid + // weird problems. + if len(container.NetworkSettings.Networks) > 1 { + logrus.WithFields(logrus.Fields{"containerId": container.ID, "containerNames": container.Names}). + Warnf("ignoring container, multiple networks found and none specified using label %s", DockerRouterLabelNetwork) return } - } else { - // if network isn't specified assume it's the first one - vipIndex = 0 + + for _, endpoint := range container.NetworkSettings.Networks { + data.ip = endpoint.IPAddress + break + } + } + + if data.ip == "" { + logrus.WithFields(logrus.Fields{"containerId": container.ID, "containerNames": container.Names}). + Warnf("ignoring container, unable to find accessible ip address") + return } - virtualIP := service.Endpoint.VirtualIPs[vipIndex] - ip, _, _ := net.ParseCIDR(virtualIP.Addr) - data.ip = ip.String() ok = true + return } @@ -332,3 +289,8 @@ func (w *dockerWatcherImpl) Stop() { w.contextCancel() } } + +type routableContainer struct { + externalContainerName string + containerEndpoint string +} diff --git a/server/docker_swarm.go b/server/docker_swarm.go new file mode 100644 index 0000000..c5b1764 --- /dev/null +++ b/server/docker_swarm.go @@ -0,0 +1,321 @@ +package server + +import ( + "context" + "fmt" + "net" + "strconv" + "strings" + "sync" + "time" + + dockertypes "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/api/types/swarm" + swarmtypes "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/api/types/versions" + "github.com/docker/docker/client" + "github.com/sirupsen/logrus" +) + +var DockerSwarmWatcher IDockerWatcher = &dockerSwarmWatcherImpl{} + +type dockerSwarmWatcherImpl struct { + sync.RWMutex + client *client.Client + contextCancel context.CancelFunc +} + +func (w *dockerSwarmWatcherImpl) makeWakerFunc(_ *routableService) func(ctx context.Context) error { + return func(ctx context.Context) error { + return nil + } +} + +func (w *dockerSwarmWatcherImpl) Start(socket string, timeoutSeconds int, refreshIntervalSeconds int) error { + var err error + + timeout := time.Duration(timeoutSeconds) * time.Second + refreshInterval := time.Duration(refreshIntervalSeconds) * time.Second + + opts := []client.Opt{ + client.WithHost(socket), + client.WithTimeout(timeout), + client.WithHTTPHeaders(map[string]string{ + "User-Agent": "mc-router ", + }), + client.WithVersion(DockerAPIVersion), + } + + w.client, err = client.NewClientWithOpts(opts...) + if err != nil { + return err + } + + ticker := time.NewTicker(refreshInterval) + serviceMap := map[string]*routableService{} + + var ctx context.Context + ctx, w.contextCancel = context.WithCancel(context.Background()) + + initialServices, err := w.listServices(ctx) + if err != nil { + return err + } + + for _, s := range initialServices { + serviceMap[s.externalServiceName] = s + if s.externalServiceName != "" { + Routes.CreateMapping(s.externalServiceName, s.containerEndpoint, w.makeWakerFunc(s)) + } else { + Routes.SetDefaultRoute(s.containerEndpoint) + } + } + + go func() { + for { + select { + case <-ticker.C: + services, err := w.listServices(ctx) + if err != nil { + logrus.WithError(err).Error("Docker failed to list services") + return + } + + visited := map[string]struct{}{} + for _, rs := range services { + if oldRs, ok := serviceMap[rs.externalServiceName]; !ok { + serviceMap[rs.externalServiceName] = rs + logrus.WithField("routableService", rs).Debug("ADD") + if rs.externalServiceName != "" { + Routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, w.makeWakerFunc(rs)) + } else { + Routes.SetDefaultRoute(rs.containerEndpoint) + } + } else if oldRs.containerEndpoint != rs.containerEndpoint { + serviceMap[rs.externalServiceName] = rs + if rs.externalServiceName != "" { + Routes.DeleteMapping(rs.externalServiceName) + Routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, w.makeWakerFunc(rs)) + } else { + Routes.SetDefaultRoute(rs.containerEndpoint) + } + logrus.WithFields(logrus.Fields{"old": oldRs, "new": rs}).Debug("UPDATE") + } + visited[rs.externalServiceName] = struct{}{} + } + for _, rs := range serviceMap { + if _, ok := visited[rs.externalServiceName]; !ok { + delete(serviceMap, rs.externalServiceName) + if rs.externalServiceName != "" { + Routes.DeleteMapping(rs.externalServiceName) + } else { + Routes.SetDefaultRoute("") + } + logrus.WithField("routableService", rs).Debug("DELETE") + } + } + + case <-ctx.Done(): + ticker.Stop() + return + } + } + }() + + logrus.Info("Monitoring Docker Swarm for Minecraft services") + return nil +} + +func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableService, error) { + services, err := w.client.ServiceList(ctx, dockertypes.ServiceListOptions{}) + if err != nil { + return nil, err + } + + serverVersion, err := w.client.ServerVersion(ctx) + if err != nil { + return nil, err + } + + networkListArgs := filters.NewArgs() + // https://docs.docker.com/engine/api/v1.29/#tag/Network (Docker 17.06) + if versions.GreaterThanOrEqualTo(serverVersion.APIVersion, "1.29") { + networkListArgs.Add("scope", "swarm") + } else { + networkListArgs.Add("driver", "overlay") + } + + networkList, err := w.client.NetworkList(ctx, network.ListOptions{Filters: networkListArgs}) + if err != nil { + return nil, err + } + + networkMap := make(map[string]*network.Inspect) + for _, network := range networkList { + networkToAdd := network + networkMap[network.ID] = &networkToAdd + } + + var result []*routableService + for _, service := range services { + if service.Spec.EndpointSpec.Mode != swarmtypes.ResolutionModeVIP { + continue + } + if len(service.Endpoint.VirtualIPs) == 0 { + continue + } + + data, ok := w.parseServiceData(&service, networkMap) + if !ok { + continue + } + + for _, host := range data.hosts { + result = append(result, &routableService{ + containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), + externalServiceName: host, + }) + } + if data.def != nil && *data.def { + result = append(result, &routableService{ + containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), + externalServiceName: "", + }) + } + } + + return result, nil +} + +func dockerCheckNetworkName(id string, name string, networkMap map[string]*network.Inspect, networkAliases map[string][]string) (bool, error) { + // we allow to specify the id instead + if id == name { + return true, nil + } + if network := networkMap[id]; network != nil { + if network.Name == name { + return true, nil + } + aliases := networkAliases[id] + for _, alias := range aliases { + if alias == name { + return true, nil + } + } + return false, nil + } + + return false, fmt.Errorf("network not found %s", id) +} + +type parsedDockerServiceData struct { + hosts []string + port uint64 + def *bool + network *string + ip string +} + +func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { + networkAliases := map[string][]string{} + for _, network := range service.Spec.TaskTemplate.Networks { + networkAliases[network.Target] = network.Aliases + } + + for key, value := range service.Spec.Labels { + if key == DockerRouterLabelHost { + if data.hosts != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service with duplicate %s", DockerRouterLabelHost) + return + } + data.hosts = strings.Split(value, ",") + } + if key == DockerRouterLabelPort { + if data.port != 0 { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service with duplicate %s", DockerRouterLabelPort) + return + } + var err error + data.port, err = strconv.ParseUint(value, 10, 32) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid %s", DockerRouterLabelPort) + return + } + } + if key == DockerRouterLabelDefault { + if data.def != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service with duplicate %s", DockerRouterLabelDefault) + return + } + data.def = new(bool) + + lowerValue := strings.TrimSpace(strings.ToLower(value)) + *data.def = lowerValue != "" && lowerValue != "0" && lowerValue != "false" && lowerValue != "no" + } + if key == DockerRouterLabelNetwork { + if data.network != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service with duplicate %s", DockerRouterLabelNetwork) + return + } + data.network = new(string) + *data.network = value + } + } + + // probably not minecraft related + if len(data.hosts) == 0 { + return + } + + if len(service.Endpoint.VirtualIPs) == 0 { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service, no VirtualIPs found") + return + } + + if data.port == 0 { + data.port = 25565 + } + + vipIndex := -1 + if data.network != nil { + for i, vip := range service.Endpoint.VirtualIPs { + if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { + vipIndex = i + break + } else if err != nil { + // we intentionally ignore name check errors + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Debugf("%v", err) + } + } + if vipIndex == -1 { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service, network %s not found", *data.network) + return + } + } else { + // if network isn't specified assume it's the first one + vipIndex = 0 + } + + virtualIP := service.Endpoint.VirtualIPs[vipIndex] + ip, _, _ := net.ParseCIDR(virtualIP.Addr) + data.ip = ip.String() + ok = true + return +} + +func (w *dockerSwarmWatcherImpl) Stop() { + if w.contextCancel != nil { + w.contextCancel() + } +}