Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-22 02:05:41 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #15505 from ninjadq/add_distributed_tracing

Browse Source 
Add distributed tracing
This commit is contained in:
stonezdj(Daojun Zhang) 2021-09-22 14:16:19 +08:00 committed by GitHub
commit 972fa0880b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
790 changed files with 87233 additions and 21939 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
make/harbor.yml.tmpl
View File

@ -199,3 +199,30 @@ proxy:
# enabled: false
# port: 9090
# path: /metrics
# Trace related config
# only can enable one trace provider(jaeger or otel) at the same time,
# and when using jaeger as provider, can only enable it with agent mode or collector mode.
# if using jaeger collector mode, uncomment endpoint and uncomment username, password if needed
# if using jaeger agetn mode uncomment agent_host and agent_port
# trace:
# enabled: true
# # set sample_rate to 1 if you wanna sampling 100% of trace data; set 0.5 if you wanna sampling 50% of trace data, and so forth
# sample_rate: 1
# # # namespace used to differenciate different harbor services
# # namespace:
# # # attributes is a key value dict contains user defined attributes used to initialize trace provider
# # attributes:
# # application: harbor
# # jaeger:
# # endpoint: http://hostname:14268/api/traces
# # username:
# # password:
# # agent_host: hostname
# # agent_port: 6832
# # otel:
# # endpoint: hostname:4318
# # url_path: /v1/traces
# # compression: false
# # insecure: true
# # timeout: 10s

2
make/photon/prepare/migrations/version_2_4_0/harbor.yml.jinja
View File

@ -494,7 +494,7 @@ trace:
# enabled: true
# # set sample_rate to 1 if you wanna sampling 100% of trace data; set 0.5 if you wanna sampling 50% of trace data, and so forth
# sample_rate: 1
# # # namespace used to diferenciate different harbor services
# # # namespace used to differenciate different harbor services
# # namespace:
# # # attributes is a key value dict contains user defined attributes used to initialize trace provider
# # attributes:

62
make/photon/prepare/models.py
View File

@ -103,11 +103,10 @@ class InternalTLS:
return
raise Exception('cert file {} should include SAN'.format(filename))
def validate(self) -> bool:
def validate(self):
if not self.enabled:
# pass the validation if not enabled
return True
return
if not internal_tls_dir.exists():
raise Exception('Internal dir for tls {} not exist'.format(internal_tls_dir))
@ -115,8 +114,6 @@ class InternalTLS:
for filename in self.required_filenames:
self._check(filename)
return True
def prepare(self):
"""
Prepare moves certs in tls file to data volume with correct permission.
@ -140,7 +137,6 @@ class InternalTLS:
else:
os.chown(file, DEFAULT_UID, DEFAULT_GID)
class Metric:
def __init__(self, enabled: bool = False, port: int = 8080, path: str = "metrics" ):
self.enabled = enabled
@ -149,4 +145,56 @@ class Metric:
def validate(self):
if not port_number_valid(self.port):
raise Exception('Port number in metrics is not valid')
raise Exception('Port number in metrics is not valid')
class JaegerExporter:
def __init__(self, config: dict):
if not config:
self.enabled = False
return
self.enabled = True
self.endpoint = config.get('endpoint')
self.username = config.get('username')
self.password = config.get('password')
self.agent_host = config.get('agent_host')
self.agent_port = config.get('agent_port')
def validate(self):
if not self.endpoint and self.agent_host is None:
raise Exception('Jaeger Colector Endpoint or Agent host not set')
class OtelExporter:
def __init__(self, config: dict):
if not config:
self.enabled = False
return
self.enabled = True
self.endpoint = config.get('endpoint')
self.url_path = config.get('url_path')
self.compression = config.get('compression') or False
self.insecure = config.get('insecure') or False
self.timeout = config.get('timeout') or '10s'
def validate(self):
if not self.endpoint:
raise Exception('Trace endpoint not set')
if not self.url_path:
raise Exception('Trace url path not set')
class Trace:
def __init__(self, config: dict):
self.enabled = config.get('enabled') or False
self.sample_rate = config.get('sample_rate', 1)
self.namespace = config.get('namespace') or ''
self.jaeger = JaegerExporter(config.get('jaeger'))
self.otel = OtelExporter(config.get('otel'))
self.attributes = config.get('attributes') or {}
def validate(self):
if not self.jaeger.enabled and not self.enabled:
raise Exception('Trace enabled but no trace exporter set')
if self.jaeger.enabled:
JaegerExporter(self.jaeger).validate()
if self.otel.enabled:
OtelExporter(self.otel).validate()

22
make/photon/prepare/templates/core/env.jinja
View File

@ -62,3 +62,25 @@ METRIC_PORT={{ metric.port }}
METRIC_NAMESPACE=harbor
METRIC_SUBSYSTEM=core
{% endif %}
{% if trace.enabled %}
TRACE_ENABLED=true
TRACE_SERVICE_NAME=harbor-core
TRACE_SAMPLE_RATE={{ trace.sample_rate }}
TRACE_NAMESPACE={{ trace.namespace }}
TRACE_ATTRIBUTES={{ trace.attributes | to_json | safe }}
{% if trace.jaeger.enabled %}
TRACE_JAEGER_ENDPOINT={{ trace.jaeger.endpoint if trace.jaeger.endpoint else '' }}
TRACE_JAEGER_USERNAME={{ trace.jaeger.username if trace.jaeger.username else '' }}
TRACE_JAEGER_PASSWORD={{ trace.jaeger.password if trace.jaeger.password else '' }}
TRACE_JAEGER_AGENT_HOSTNAME={{ trace.jaeger.agent_host if trace.jaeger.agent_host else '' }}
TRACE_JAEGER_AGENT_PORT={{ trace.jaeger.agent_port if trace.jaeger.agent_port else '' }}
{% endif %}
{%if trace.otel.enabled %}
TRACE_OTEL_ENDPOINT={{ trace.otel.endpoint }}
TRACE_OTEL_URL_PATH={{ trace.otel.url_path if trace.otel.url_path else '' }}
TRACE_OTEL_COMPRESSION={{ trace.otel.compression }}
TRACE_OTEL_TIMEOUT={{ trace.otel.timeout }}
TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
{% endif %}
{% endif %}

22
make/photon/prepare/templates/jobservice/env.jinja
View File

@ -25,3 +25,25 @@ REGISTRY_CREDENTIAL_PASSWORD={{registry_password}}
METRIC_NAMESPACE=harbor
METRIC_SUBSYSTEM=jobservice
{% endif %}
{% if trace.enabled %}
TRACE_ENABLED=true
TRACE_SERVICE_NAME=harbor-jobservice
TRACE_SAMPLE_RATE={{ trace.sample_rate }}
TRACE_NAMESPACE={{ trace.namespace }}
TRACE_ATTRIBUTES={{ trace.attributes | to_json | safe }}
{% if trace.jaeger.enabled %}
TRACE_JAEGER_ENDPOINT={{ trace.jaeger.endpoint if trace.jaeger.endpoint else '' }}
TRACE_JAEGER_USERNAME={{ trace.jaeger.username if trace.jaeger.username else '' }}
TRACE_JAEGER_PASSWORD={{ trace.jaeger.password if trace.jaeger.password else '' }}
TRACE_JAEGER_AGENT_HOSTNAME={{ trace.jaeger.agent_host if trace.jaeger.agent_host else '' }}
TRACE_JAEGER_AGENT_PORT={{ trace.jaeger.agent_port if trace.jaeger.agent_port else '' }}
{% endif %}
{%if trace.otel.enabled %}
TRACE_OTEL_ENDPOINT={{ trace.otel.endpoint }}
TRACE_OTEL_URL_PATH={{ trace.otel.url_path if trace.otel.url_path else '' }}
TRACE_OTEL_COMPRESSION={{ trace.otel.compression }}
TRACE_OTEL_TIMEOUT={{ trace.otel.timeout }}
TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
{% endif %}
{% endif %}

21
make/photon/prepare/templates/registryctl/env.jinja
View File

@ -9,3 +9,24 @@ INTERNAL_TLS_CERT_PATH=/etc/harbor/ssl/registryctl.crt
{% if internal_tls.verify_client_cert %}
INTERNAL_VERIFY_CLIENT_CERT=true
{% endif %}
{% if trace.enabled %}
TRACE_ENABLED=true
TRACE_SERVICE_NAME=harbor-registryctl
TRACE_SAMPLE_RATE={{ trace.sample_rate }}
TRACE_NAMESPACE={{ trace.namespace }}
TRACE_ATTRIBUTES={{ trace.attributes | to_json | safe }}
{% if trace.jaeger.enabled %}
TRACE_JAEGER_ENDPOINT={{ trace.jaeger.endpoint if trace.jaeger.endpoint else '' }}
TRACE_JAEGER_USERNAME={{ trace.jaeger.username if trace.jaeger.username else '' }}
TRACE_JAEGER_PASSWORD={{ trace.jaeger.password if trace.jaeger.password else '' }}
TRACE_JAEGER_AGENT_HOSTNAME={{ trace.jaeger.agent_host if trace.jaeger.agent_host else '' }}
TRACE_JAEGER_AGENT_PORT={{ trace.jaeger.agent_port if trace.jaeger.agent_port else '' }}
{% endif %}
{%if trace.otel.enabled %}
TRACE_OTEL_ENDPOINT={{ trace.otel.endpoint }}
TRACE_OTEL_URL_PATH={{ trace.otel.url_path if trace.otel.url_path else '' }}
TRACE_OTEL_COMPRESSION={{ trace.otel.compression }}
TRACE_OTEL_TIMEOUT={{ trace.otel.timeout }}
TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
{% endif %}
{% endif %}

6
make/photon/prepare/utils/configs.py
View File

@ -3,7 +3,7 @@ import os
import yaml
from urllib.parse import urlencode
from g import versions_file_path, host_root_dir, DEFAULT_UID, INTERNAL_NO_PROXY_DN
from models import InternalTLS, Metric
from models import InternalTLS, Metric, Trace
from utils.misc import generate_random_string, owner_can_read, other_can_read
default_db_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
@ -327,6 +327,10 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
else:
config_dict['metric'] = Metric()
# trace configs
trace_config = configs.get('trace')
config_dict['trace'] = Trace(trace_config or {})
if config_dict['internal_tls'].enabled:
config_dict['portal_url'] = 'https://portal:8443'
config_dict['registry_url'] = 'https://registry:5443'

8
make/photon/prepare/utils/jinja.py
View File

@ -1,8 +1,16 @@
import json
from jinja2 import Environment, FileSystemLoader
from .misc import mark_file
jinja_env = Environment(loader=FileSystemLoader('/'), trim_blocks=True, lstrip_blocks=True)
def to_json(value):
return json.dumps(value)
jinja_env.filters['to_json'] = to_json
def render_jinja(src, dest,mode=0o640, uid=0, gid=0, **kw):
t = jinja_env.get_template(src)
with open(dest, 'w') as f:

2
make/photon/prepare/utils/registry_ctl.py
View File

@ -1,4 +1,4 @@
import os, shutil
import os
from g import config_dir, templates_dir, DEFAULT_GID, DEFAULT_UID
from utils.misc import prepare_dir

15
src/chartserver/client.go
View File

@ -10,8 +10,11 @@ import (
"sync"
"time"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/errors"
tracelib "github.com/goharbor/harbor/src/lib/trace"
)
const (
@ -22,7 +25,7 @@ const (
var (
once sync.Once
chartTransport *http.Transport
chartTransport http.RoundTripper
)
// ChartClient is a http client to get the content from the external http server
@ -38,9 +41,13 @@ type ChartClient struct {
// credential can be nil
func NewChartClient(credential *Credential) *ChartClient { // Create http client with customized timeouts
once.Do(func() {
chartTransport = commonhttp.GetHTTPTransport(commonhttp.SecureTransport).Clone()
chartTransport.MaxIdleConns = maxIdleConnections
chartTransport.IdleConnTimeout = idleConnectionTimeout
chartTransport = commonhttp.NewTransport(
commonhttp.WithMaxIdleConns(maxIdleConnections),
commonhttp.WithIdleconnectionTimeout(idleConnectionTimeout),
)
if tracelib.Enabled() {
chartTransport = otelhttp.NewTransport(chartTransport)
}
})
client := &http.Client{

2
src/chartserver/reverse_proxy.go
View File

@ -53,7 +53,7 @@ func NewProxyEngine(target *url.URL, cred *Credential, middlewares ...func(http.
director(target, cred, req)
},
ModifyResponse: modifyResponse,
Transport: commonhttp.GetHTTPTransport(commonhttp.SecureTransport),
Transport: commonhttp.GetHTTPTransport(),
}
if len(middlewares) > 0 {

5
src/cmd/exporter/main.go
View File

@ -1,11 +1,12 @@
package main
import (
"github.com/goharbor/harbor/src/common/models"
"net/http"
"os"
"strings"
"github.com/goharbor/harbor/src/common/models"
_ "github.com/lib/pq"
"github.com/prometheus/client_golang/prometheus"
"github.com/spf13/viper"
@ -43,7 +44,7 @@ func main() {
HarborHost: viper.GetString("service.host"),
HarborPort: viper.GetInt("service.port"),
Client: &http.Client{
Transport: commonthttp.GetHTTPTransport(commonthttp.SecureTransport),
Transport: commonthttp.GetHTTPTransport(),
},
})

17
src/common/const.go
View File

@ -169,4 +169,21 @@ const (
MetricEnable = "metric_enable"
MetricPort = "metric_port"
MetricPath = "metric_path"
// Trace setting items
TraceEnabled = "trace_enabled"
TraceServiceName = "trace_service_name"
TraceSampleRate = "trace_sample_rate"
TraceNamespace = "trace_namespace"
TraceAttributes = "trace_attribute"
TraceJaegerEndpoint = "trace_jaeger_endpoint"
TraceJaegerUsername = "trace_jaeger_username"
TraceJaegerPassword = "trace_jaeger_password"
TraceJaegerAgentHost = "trace_jaeger_agent_host"
TraceJaegerAgentPort = "trace_jaeger_agent_port"
TraceOtelEndpoint = "trace_otel_endpoint"
TraceOtelURLPath = "trace_otel_url_path"
TraceOtelCompression = "trace_otel_compression"
TraceOtelInsecure = "trace_otel_insecure"
TraceOtelTimeout = "trace_otel_timeout"
)

70
src/common/http/client.go
View File

@ -16,65 +16,18 @@ package http
import (
"bytes"
"crypto/tls"
"encoding/json"
"errors"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"reflect"
"time"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/lib"
)
const (
// InsecureTransport used to get the insecure http Transport
InsecureTransport = iota
// SecureTransport used to get the external secure http Transport
SecureTransport
)
var (
secureHTTPTransport *http.Transport
insecureHTTPTransport *http.Transport
)
func init() {
secureHTTPTransport = newDefaultTransport()
insecureHTTPTransport = newDefaultTransport()
insecureHTTPTransport.TLSClientConfig.InsecureSkipVerify = true
if InternalTLSEnabled() {
tlsConfig, err := GetInternalTLSConfig()
if err != nil {
panic(err)
}
secureHTTPTransport.TLSClientConfig = tlsConfig
}
}
// Use this instead of Default Transport in library because it sets ForceAttemptHTTP2 to true
// And that options introduced in go 1.13 will cause the https requests hang forever in replication environment
func newDefaultTransport() *http.Transport {
return &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
TLSClientConfig: &tls.Config{},
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
}
// Client is a util for common HTTP operations, such Get, Head, Post, Put and Delete.
// Use Do instead if those methods can not meet your requirement
type Client struct {
@ -87,27 +40,6 @@ func (c *Client) GetClient() *http.Client {
return c.client
}
// GetHTTPTransport returns HttpTransport based on insecure configuration
func GetHTTPTransport(clientType uint) *http.Transport {
switch clientType {
case SecureTransport:
return secureHTTPTransport
case InsecureTransport:
return insecureHTTPTransport
default:
// default Transport is secure one
return secureHTTPTransport
}
}
// GetHTTPTransportByInsecure returns a insecure HttpTransport if insecure is true or it returns secure one
func GetHTTPTransportByInsecure(insecure bool) *http.Transport {
if insecure {
return insecureHTTPTransport
}
return secureHTTPTransport
}
// NewClient creates an instance of Client.
// Use net/http.Client as the default value if c is nil.
// Modifiers modify the request before sending it.
@ -117,7 +49,7 @@ func NewClient(c *http.Client, modifiers ...modifier.Modifier) *Client {
}
if client.client == nil {
client.client = &http.Client{
Transport: GetHTTPTransport(SecureTransport),
Transport: GetHTTPTransport(),
}
}
if len(modifiers) > 0 {

14
src/common/http/client_test.go
View File

@ -1,14 +0,0 @@
package http
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetHTTPTransport(t *testing.T) {
transport := GetHTTPTransport(InsecureTransport)
assert.True(t, transport.TLSClientConfig.InsecureSkipVerify)
transport = GetHTTPTransport(SecureTransport)
assert.False(t, transport.TLSClientConfig.InsecureSkipVerify)
}

136
src/common/http/transport.go Normal file
View File

@ -0,0 +1,136 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package http
import (
"crypto/tls"
"net"
"net/http"
"time"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)
const (
// InsecureTransport used to get the insecure http Transport
InsecureTransport = iota
// SecureTransport used to get the external secure http Transport
SecureTransport
)
var (
secureHTTPTransport http.RoundTripper
insecureHTTPTransport http.RoundTripper
)
func init() {
insecureHTTPTransport = NewTransport(WithInsecureSkipVerify(true))
if InternalTLSEnabled() {
secureHTTPTransport = NewTransport(WithInternalTLSConfig())
} else {
secureHTTPTransport = NewTransport()
}
}
func AddTracingWithGlobalTransport() {
insecureHTTPTransport = otelhttp.NewTransport(insecureHTTPTransport)
secureHTTPTransport = otelhttp.NewTransport(secureHTTPTransport)
}
// Use this instead of Default Transport in library because it sets ForceAttemptHTTP2 to true
// And that options introduced in go 1.13 will cause the https requests hang forever in replication environment
func newDefaultTransport() *http.Transport {
return &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
TLSClientConfig: &tls.Config{},
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
}
// WithInternalTLSConfig returns a TransportOption that configures the transport to use the internal TLS configuration
func WithInternalTLSConfig() func(*http.Transport) {
return func(tr *http.Transport) {
tlsConfig, err := GetInternalTLSConfig()
if err != nil {
panic(err)
}
tr.TLSClientConfig = tlsConfig
}
}
// WithInsecureSkipVerify returns a TransportOption that configures the transport to skip verification of the server's certificate
func WithInsecureSkipVerify(skipVerify bool) func(*http.Transport) {
return func(tr *http.Transport) {
tr.TLSClientConfig.InsecureSkipVerify = skipVerify
}
}
// WithMaxIdleConnsPerHost returns a TransportOption that configures the transport to use the specified number of idle connections per host
func WithMaxIdleConns(maxIdleConns int) func(*http.Transport) {
return func(tr *http.Transport) {
tr.MaxIdleConns = maxIdleConns
}
}
// WithIdleConnTimeout returns a TransportOption that configures the transport to use the specified idle connection timeout
func WithIdleconnectionTimeout(idleConnectionTimeout time.Duration) func(*http.Transport) {
return func(tr *http.Transport) {
tr.IdleConnTimeout = idleConnectionTimeout
}
}
// NewTransport returns a new http.Transport with the specified options
func NewTransport(opts ...func(*http.Transport)) http.RoundTripper {
tr := newDefaultTransport()
for _, opt := range opts {
opt(tr)
}
return tr
}
// TransportConfig is the configuration for http transport
type TransportConfig struct {
Insecure bool
}
// TransportOption is the option for http transport
type TransportOption func(*TransportConfig)
// WithInsecure returns a TransportOption that configures the transport to skip verification of the server's certificate
func WithInsecure(skipVerify bool) TransportOption {
return func(cfg *TransportConfig) {
cfg.Insecure = skipVerify
}
}
// GetHTTPTransport returns HttpTransport based on insecure configuration
func GetHTTPTransport(opts ...TransportOption) http.RoundTripper {
cfg := &TransportConfig{}
for _, opt := range opts {
opt(cfg)
}
if cfg.Insecure {
return insecureHTTPTransport
}
return secureHTTPTransport
}

14
src/common/http/transport_test.go Normal file
View File

@ -0,0 +1,14 @@
package http
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetHTTPTransport(t *testing.T) {
transport := GetHTTPTransport()
assert.Equal(t, secureHTTPTransport, transport, "Transport should be secure")
transport = GetHTTPTransport(WithInsecure(true))
assert.Equal(t, insecureHTTPTransport, transport, "Transport should be insecure")
}

9
src/common/job/client.go
View File

@ -5,12 +5,13 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/goharbor/harbor/src/lib/config"
"io/ioutil"
"net/http"
"regexp"
"strings"
"github.com/goharbor/harbor/src/lib/config"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/job/models"
@ -61,7 +62,7 @@ type DefaultClient struct {
func NewDefaultClient(endpoint, secret string) *DefaultClient {
var c *commonhttp.Client
httpCli := &http.Client{
Transport: commonhttp.GetHTTPTransport(commonhttp.SecureTransport),
Transport: commonhttp.GetHTTPTransport(),
}
if len(secret) > 0 {
c = commonhttp.NewClient(httpCli, auth.NewSecretAuthorizer(secret))
@ -81,12 +82,12 @@ func NewReplicationClient(endpoint, secret string) *DefaultClient {
if len(secret) > 0 {
c = commonhttp.NewClient(&http.Client{
Transport: commonhttp.GetHTTPTransport(commonhttp.SecureTransport),
Transport: commonhttp.GetHTTPTransport(),
},
auth.NewSecretAuthorizer(secret))
} else {
c = commonhttp.NewClient(&http.Client{
Transport: commonhttp.GetHTTPTransport(commonhttp.SecureTransport),
Transport: commonhttp.GetHTTPTransport(),
})
}

4
src/controller/artifact/controller.go
View File

@ -204,7 +204,7 @@ func (c *controller) ensureArtifact(ctx context.Context, repository, digest stri
created = true
artifact.ID = id
return nil
})(ctx); err != nil {
})(orm.SetTransactionOpNameToContext(ctx, "tx-ensure-artifact")); err != nil {
// got error that isn't conflict error, return directly
if !errors.IsConflictErr(err) {
return false, nil, err
@ -376,7 +376,7 @@ func (c *controller) deleteDeeply(ctx context.Context, id int64, isRoot bool) er
Digest: art.Digest,
})
return err
})(ctx); err != nil && !errors.IsErr(err, errors.ConflictCode) {
})(orm.SetTransactionOpNameToContext(ctx, "tx-delete-artifact-deeply")); err != nil && !errors.IsErr(err, errors.ConflictCode) {
return err
}

5
src/controller/blob/controller.go
View File

@ -17,9 +17,10 @@ package blob
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/lib/q"
"time"
"github.com/goharbor/harbor/src/lib/q"
"github.com/docker/distribution"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
@ -303,7 +304,7 @@ func (c *controller) Sync(ctx context.Context, references []distribution.Descrip
}
return nil
})(ctx)
})(orm.SetTransactionOpNameToContext(ctx, "tx-sync-blob"))
}
if len(missing) > 0 {

2
src/controller/event/handler/internal/util.go
View File

@ -41,5 +41,5 @@ func autoScan(ctx context.Context, a *artifact.Artifact, tags ...string) error {
}
return scan.DefaultController.Scan(ctx, a, options...)
})(ctx)
})(orm.SetTransactionOpNameToContext(ctx, "tx-auto-scan"))
}

2
src/controller/health/checker.go
View File

@ -48,7 +48,7 @@ func HTTPStatusCodeHealthChecker(method string, url string, header http.Header,
}
client := httputil.NewClient(&http.Client{
Transport: httputil.GetHTTPTransport(httputil.SecureTransport),
Transport: httputil.GetHTTPTransport(),
Timeout: timeout,
})
resp, err := client.Do(req)

2
src/controller/project/controller.go
View File

@ -104,7 +104,7 @@ func (c *controller) Create(ctx context.Context, project *models.Project) (int64
return nil
}
if err := orm.WithTransaction(h)(ctx); err != nil {
if err := orm.WithTransaction(h)(orm.SetTransactionOpNameToContext(ctx, "tx-create-project")); err != nil {
return 0, err
}

8
src/controller/project/controller_test.go
View File

@ -50,16 +50,16 @@ func (suite *ControllerTestSuite) TestCreate() {
c := controller{projectMgr: mgr, allowlistMgr: allowlistMgr, metaMgr: metadataMgr}
{
metadataMgr.On("Add", ctx, mock.Anything, mock.Anything).Return(nil).Once()
mgr.On("Create", ctx, mock.Anything).Return(int64(2), nil).Once()
metadataMgr.On("Add", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
mgr.On("Create", mock.Anything, mock.Anything).Return(int64(2), nil).Once()
projectID, err := c.Create(ctx, &models.Project{OwnerID: 1, Metadata: map[string]string{"public": "true"}})
suite.Nil(err)
suite.Equal(int64(2), projectID)
}
{
metadataMgr.On("Add", ctx, mock.Anything, mock.Anything).Return(fmt.Errorf("oops")).Once()
mgr.On("Create", ctx, mock.Anything).Return(int64(2), nil).Once()
metadataMgr.On("Add", mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("oops")).Once()
mgr.On("Create", mock.Anything, mock.Anything).Return(int64(2), nil).Once()
projectID, err := c.Create(ctx, &models.Project{OwnerID: 1, Metadata: map[string]string{"public": "true"}})
suite.Error(err)
suite.Equal(int64(0), projectID)

2
src/controller/repository/controller.go
View File

@ -108,7 +108,7 @@ func (c *controller) Ensure(ctx context.Context, name string) (bool, int64, erro
}
created = true
return nil
})(ctx); err != nil {
})(orm.SetTransactionOpNameToContext(ctx, "tx-repository-ensure")); err != nil {
// isn't conflict error, return directly
if !errors.IsConflictErr(err) {
return false, 0, err

4
src/controller/scan/base_controller.go
View File

@ -379,7 +379,7 @@ func (bc *basicController) startScanAll(ctx context.Context, executionID int64)
return bc.Scan(ctx, artifact, WithExecutionID(executionID))
}
if err := orm.WithTransaction(scan)(bc.makeCtx()); err != nil {
if err := orm.WithTransaction(scan)(orm.SetTransactionOpNameToContext(bc.makeCtx(), "tx-start-scanall")); err != nil {
// Just logged
log.Errorf("failed to scan artifact %s, error %v", artifact, err)
@ -500,7 +500,7 @@ func (bc *basicController) makeReportPlaceholder(ctx context.Context, r *scanner
return nil
}
if err := orm.WithTransaction(create)(ctx); err != nil {
if err := orm.WithTransaction(create)(orm.SetTransactionOpNameToContext(ctx, "tx-make-report-placeholder")); err != nil {
return nil, err
}

5
src/controller/tag/controller.go
View File

@ -16,6 +16,8 @@ package tag
import (
"context"
"time"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
@ -28,7 +30,6 @@ import (
"github.com/goharbor/harbor/src/pkg/signature"
"github.com/goharbor/harbor/src/pkg/tag"
model_tag "github.com/goharbor/harbor/src/pkg/tag/model/tag"
"time"
)
var (
@ -115,7 +116,7 @@ func (c *controller) Ensure(ctx context.Context, repositoryID, artifactID int64,
tag.PushTime = time.Now()
_, err = c.Create(ctx, tag)
return err
})(ctx); err != nil && !errors.IsConflictErr(err) {
})(orm.SetTransactionOpNameToContext(ctx, "tx-tag-ensure")); err != nil && !errors.IsConflictErr(err) {
return err
}

31
src/core/main.go
View File

@ -26,15 +26,14 @@ import (
"syscall"
"time"
configCtl "github.com/goharbor/harbor/src/controller/config"
"github.com/goharbor/harbor/src/pkg/oidc"
"github.com/astaxie/beego"
_ "github.com/astaxie/beego/session/redis"
_ "github.com/astaxie/beego/session/redis_sentinel"
"github.com/goharbor/harbor/src/common/dao"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/models"
configCtl "github.com/goharbor/harbor/src/controller/config"
_ "github.com/goharbor/harbor/src/controller/event/handler"
"github.com/goharbor/harbor/src/controller/health"
"github.com/goharbor/harbor/src/controller/registry"
@ -53,9 +52,12 @@ import (
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/metric"
"github.com/goharbor/harbor/src/lib/orm"
tracelib "github.com/goharbor/harbor/src/lib/trace"
"github.com/goharbor/harbor/src/migration"
_ "github.com/goharbor/harbor/src/pkg/config/inmemory"
"github.com/goharbor/harbor/src/pkg/notification"
_ "github.com/goharbor/harbor/src/pkg/notifier/topic"
"github.com/goharbor/harbor/src/pkg/oidc"
"github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
pkguser "github.com/goharbor/harbor/src/pkg/user"
@ -86,17 +88,27 @@ func updateInitPassword(ctx context.Context, userID int, password string) error
return nil
}
func gracefulShutdown(closing, done chan struct{}) {
func gracefulShutdown(closing, done chan struct{}, shutdowns ...func()) {
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
log.Infof("capture system signal %s, to close \"closing\" channel", <-signals)
close(closing)
select {
case <-done:
shutdownChan := make(chan struct{}, 1)
go func() {
for _, s := range shutdowns {
s()
}
<-done
log.Infof("Goroutines exited normally")
shutdownChan <- struct{}{}
}()
select {
case <-shutdownChan:
log.Infof("all shutdown jobs done")
case <-time.After(time.Second * 3):
log.Infof("Timeout waiting goroutines to exit")
}
os.Exit(0)
}
@ -175,6 +187,9 @@ func main() {
metric.RegisterCollectors()
go metric.ServeProm(metricCfg.Path, metricCfg.Port)
}
ctx := context.Background()
config.InitTraceConfig(ctx)
shutdownTracerProvider := tracelib.InitGlobalTracer(ctx)
token.InitCreators()
database, err := config.Database()
if err != nil {
@ -186,7 +201,7 @@ func main() {
if err = migration.Migrate(database); err != nil {
log.Fatalf("failed to migrate: %v", err)
}
ctx := orm.Context()
ctx = orm.Clone(ctx)
if err := config.Load(ctx); err != nil {
log.Fatalf("failed to load config: %v", err)
}
@ -211,7 +226,7 @@ func main() {
closing := make(chan struct{})
done := make(chan struct{})
go gracefulShutdown(closing, done)
go gracefulShutdown(closing, done, shutdownTracerProvider)
// Start health checker for registries
go registry.Ctl.StartRegularHealthCheck(orm.Context(), closing, done)

2
src/core/middlewares/middlewares.go
View File

@ -32,6 +32,7 @@ import (
"github.com/goharbor/harbor/src/server/middleware/requestid"
"github.com/goharbor/harbor/src/server/middleware/security"
"github.com/goharbor/harbor/src/server/middleware/session"
"github.com/goharbor/harbor/src/server/middleware/trace"
"github.com/goharbor/harbor/src/server/middleware/transaction"
)
@ -79,6 +80,7 @@ var (
func MiddleWares() []beego.MiddleWare {
return []beego.MiddleWare{
mergeslash.Middleware(),
trace.Middleware(),
metric.Middleware(),
requestid.Middleware(),
log.Middleware(),

18
src/go.mod
View File

@ -17,7 +17,7 @@ require (
github.com/bugsnag/bugsnag-go v1.5.2 // indirect
github.com/bugsnag/panicwrap v1.2.0 // indirect
github.com/casbin/casbin v1.7.0
github.com/cenkalti/backoff/v4 v4.1.0
github.com/cenkalti/backoff/v4 v4.1.1
github.com/cloudflare/cfssl v0.0.0-20190510060611-9c027c93ba9e // indirect
github.com/coreos/go-oidc/v3 v3.0.0
github.com/denverdino/aliyungo v0.0.0-20191227032621-df38c6fa730c // indirect
@ -46,7 +46,7 @@ require (
github.com/google/uuid v1.1.2
github.com/gorilla/csrf v1.6.2
github.com/gorilla/handlers v1.4.2
github.com/gorilla/mux v1.7.4
github.com/gorilla/mux v1.8.0
github.com/graph-gophers/dataloader v5.0.0+incompatible
github.com/jinzhu/gorm v1.9.8 // indirect
github.com/jpillora/backoff v1.0.0
@ -67,10 +67,21 @@ require (
github.com/tencentcloud/tencentcloud-sdk-go v1.0.62
github.com/theupdateframework/notary v0.6.1
github.com/vmihailenco/msgpack/v5 v5.0.0-rc.2
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.22.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.22.0
go.opentelemetry.io/otel v1.0.0-RC3
go.opentelemetry.io/otel/exporters/jaeger v1.0.0-RC3
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.0-RC3 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.0.0-RC3
go.opentelemetry.io/otel/sdk v1.0.0-RC3
go.opentelemetry.io/otel/trace v1.0.0-RC3
go.uber.org/ratelimit v0.2.0
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
golang.org/x/net v0.0.0-20210902165921-8d991716f632
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect
gopkg.in/dancannon/gorethink.v3 v3.0.5 // indirect
gopkg.in/fatih/pool.v2 v2.0.0 // indirect
gopkg.in/gorethink/gorethink.v3 v3.0.5 // indirect
@ -86,5 +97,6 @@ require (
replace (
github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0+incompatible
github.com/goharbor/harbor => ../
go.opentelemetry.io/otel/exporters/otlp/otlptrace => github.com/ninjadq/opentelemetry-go/exporters/otlp/otlptrace v1.0.0-RC3-sp1
google.golang.org/api => google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff
)

107
src/go.sum
View File

@ -111,6 +111,7 @@ github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@ -166,8 +167,8 @@ github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwt
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc=
github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@ -183,6 +184,9 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cloudflare/cfssl v0.0.0-20190510060611-9c027c93ba9e h1:ZtyhUG4s94BMUCdgvRZySr/AXYL5CDcjxhIV/83xJog=
github.com/cloudflare/cfssl v0.0.0-20190510060611-9c027c93ba9e/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/cockroach-go v0.0.0-20190925194419-606b3d062051/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
@ -289,7 +293,12 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@ -299,6 +308,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZM
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o=
github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
@ -460,7 +471,6 @@ github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq
github.com/golang-migrate/migrate/v4 v4.11.0 h1:uqtd0ysK5WyBQ/T1K2uDIooJV0o2Obt6uPwP062DupQ=
github.com/golang-migrate/migrate/v4 v4.11.0/go.mod h1:nqbpDbckcYjsCD5I8q5+NI9Tkk7SVcmaF40Ax1eAWhg=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@ -485,8 +495,10 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@ -503,8 +515,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@ -539,8 +553,9 @@ github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
@ -557,6 +572,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
@ -779,6 +796,8 @@ github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/ninjadq/opentelemetry-go/exporters/otlp/otlptrace v1.0.0-RC3-sp1 h1:gRTp8Tf81b44CznxTQfj4JPByMVx+dEN1oGoctyndaU=
github.com/ninjadq/opentelemetry-go/exporters/otlp/otlptrace v1.0.0-RC3-sp1/go.mod h1:1tvDhRy/GCexiD9dQZzqwqGnI7/fnZOsi31DyvK3zyQ=
github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@ -880,6 +899,7 @@ github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qq
github.com/robfig/cron v1.0.0 h1:slmQxIUH6U9ruw4XoJ7C2pyyx4yYeiHx8S9pNootHsM=
github.com/robfig/cron v1.0.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@ -1003,6 +1023,7 @@ github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6Ut
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI=
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE=
@ -1030,6 +1051,38 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opentelemetry.io/contrib v0.22.0 h1:0F7gDEjgb1WGn4ODIjaCAg75hmqF+UN0LiVgwxsCodc=
go.opentelemetry.io/contrib v0.22.0/go.mod h1:EH4yDYeNoaTqn/8yCWQmfNB78VHfGX2Jt2bvnvzBlGM=
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.22.0 h1:Mfz1DMQ43mhQePKqiny6kUTnUrtin+395V67yAIyYhg=
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.22.0/go.mod h1:jpoprhHaffWHQ1KBpL0jI+w7979p4ijAL2auucLUj1E=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.22.0 h1:WHjZguqT+3UjTgFum33hWZYybDVnx8u9q5/kQDfaGTs=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.22.0/go.mod h1:o3MuU25bYroYnc2TOKe8mTk8f9X1oPFO6C5RCoPKtSU=
go.opentelemetry.io/contrib/propagators v0.22.0 h1:KGdv58M2//veiYLIhb31mofaI2LgkIPXXAZVeYVyfd8=
go.opentelemetry.io/contrib/propagators v0.22.0/go.mod h1:xGOuXr6lLIF9BXipA4pm6UuOSI0M98U6tsI3khbOiwU=
go.opentelemetry.io/otel v1.0.0-RC1/go.mod h1:x9tRa9HK4hSSq7jf2TKbqFbtt58/TGk0f9XiEYISI1I=
go.opentelemetry.io/otel v1.0.0-RC2/go.mod h1:w1thVQ7qbAy8MHb0IFj8a5Q2QU0l2ksf8u/CN8m3NOM=
go.opentelemetry.io/otel v1.0.0-RC3 h1:kvwiyEkiUT/JaadXzVLI/R1wDO934A7r3Bs2wEe6wqA=
go.opentelemetry.io/otel v1.0.0-RC3/go.mod h1:Ka5j3ua8tZs4Rkq4Ex3hwgBgOchyPVq5S6P2lz//nKQ=
go.opentelemetry.io/otel/exporters/jaeger v1.0.0-RC3 h1:pKXuRvOc+5NgM0vv05PVIUetreuM57mcC6QQAKkcqZA=
go.opentelemetry.io/otel/exporters/jaeger v1.0.0-RC3/go.mod h1:UbP19Xlhk9tcRZ+A3PfvyN5ld4X4YrSnzXaYzx1yNLc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.0.0-RC3 h1:a112vu2cfR0cM+jUZZDNwi+uRuYoEgskrbs6MU5NySg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.0.0-RC3/go.mod h1:ZX52wqONzDpcQArxoJ0J10raZQ7kb/tF/VEjKmWwMkU=
go.opentelemetry.io/otel/internal/metric v0.22.0 h1:Q9bS02XRykSRIbggaU4hVF9oWOP9PyILu26zJWoKmk0=
go.opentelemetry.io/otel/internal/metric v0.22.0/go.mod h1:7qVuMihW/ktMonEfOvBXuh6tfMvvEyoIDgeJNRloYbQ=
go.opentelemetry.io/otel/metric v0.22.0 h1:/qv10BzznqEifrXBwsTT370OCN1PRgt+mnjzMwxJKrQ=
go.opentelemetry.io/otel/metric v0.22.0/go.mod h1:KcsUkBiYGW003DJ+ugd2aqIRIfjabD9jeOUXqsAtrq0=
go.opentelemetry.io/otel/oteltest v1.0.0-RC1/go.mod h1:+eoIG0gdEOaPNftuy1YScLr1Gb4mL/9lpDkZ0JjMRq4=
go.opentelemetry.io/otel/oteltest v1.0.0-RC2 h1:xNKqMhlZYkASSyvF4JwObZFMq0jhFN3c3SP+2rCzVPk=
go.opentelemetry.io/otel/oteltest v1.0.0-RC2/go.mod h1:kiQ4tw5tAL4JLTbcOYwK1CWI1HkT5aiLzHovgOVnz/A=
go.opentelemetry.io/otel/sdk v1.0.0-RC3 h1:iRMkET+EmJUn5mW0hJzygBraXRmrUwzbOtNvTCh/oKs=
go.opentelemetry.io/otel/sdk v1.0.0-RC3/go.mod h1:78H6hyg2fka0NYT9fqGuFLvly2yCxiBXDJAgLKo/2Us=
go.opentelemetry.io/otel/trace v1.0.0-RC1/go.mod h1:86UHmyHWFEtWjfWPSbu0+d0Pf9Q6e1U+3ViBOc+NXAg=
go.opentelemetry.io/otel/trace v1.0.0-RC2/go.mod h1:JPQ+z6nNw9mqEGT8o3eoPTdnNI+Aj5JcxEsVGREIAy4=
go.opentelemetry.io/otel/trace v1.0.0-RC3 h1:9F0ayEvlxv8BmNmPbU005WK7hC+7KbOazCPZjNa1yME=
go.opentelemetry.io/otel/trace v1.0.0-RC3/go.mod h1:VUt2TUYd8S2/ZRX09ZDFZQwn2RqfMB5MzO17jBojGxo=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v0.9.0 h1:C0g6TWmQYvjKRnljRULLWUVJGy8Uvu0NEL/5frY2/t4=
go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg=
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc=
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@ -1093,6 +1146,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@ -1102,6 +1156,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -1137,10 +1192,13 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210902165921-8d991716f632 h1:900XJE4Rn/iPU+xD5ZznOe4GKKc4AdFK0IO1P6Z3/lQ=
golang.org/x/net v0.0.0-20210902165921-8d991716f632/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1156,8 +1214,9 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -1212,8 +1271,15 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210903071746-97244b99971b h1:3Dq0eVHn0uaQJmPO+/aYPI/fRMqdrVDbu7MQcku54gg=
golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg=
golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
@ -1223,8 +1289,11 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@ -1286,6 +1355,7 @@ golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1323,9 +1393,11 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8=
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 h1:NHN4wOCScVzKhPenJ2dt+BTs3X/XkBVI/Rh4iDt55T8=
google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@ -1336,10 +1408,15 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@ -1349,8 +1426,11 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@ -1390,6 +1470,7 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

10
src/jobservice/api/router.go
View File

@ -16,12 +16,15 @@ package api
import (
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"net/http"
"github.com/gorilla/mux"
"go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux"
"github.com/goharbor/harbor/src/jobservice/errs"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/gorilla/mux"
"github.com/goharbor/harbor/src/lib/errors"
tracelib "github.com/goharbor/harbor/src/lib/trace"
)
const (
@ -59,6 +62,9 @@ func NewBaseRouter(handler Handler, authenticator Authenticator) Router {
// Register routes here
br.registerRoutes()
if tracelib.Enabled() {
br.router.Use(otelmux.Middleware("serve-http"))
}
return br
}

4
src/jobservice/job/impl/notification/http_helper.go
View File

@ -25,9 +25,9 @@ func init() {
clients: map[string]*http.Client{},
}
httpHelper.clients[secure] = &http.Client{
Transport: commonhttp.GetHTTPTransport(commonhttp.SecureTransport),
Transport: commonhttp.GetHTTPTransport(),
}
httpHelper.clients[insecure] = &http.Client{
Transport: commonhttp.GetHTTPTransport(commonhttp.InsecureTransport),
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(true)),
}
}

6
src/jobservice/main.go
View File

@ -19,6 +19,7 @@ import (
"errors"
"flag"
"fmt"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/jobservice/common/utils"
"github.com/goharbor/harbor/src/jobservice/config"
@ -27,6 +28,8 @@ import (
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/jobservice/runtime"
cfgLib "github.com/goharbor/harbor/src/lib/config"
tracelib "github.com/goharbor/harbor/src/lib/trace"
_ "github.com/goharbor/harbor/src/pkg/config/inmemory"
_ "github.com/goharbor/harbor/src/pkg/config/rest"
)
@ -57,6 +60,9 @@ func main() {
panic(err)
}
cfgLib.InitTraceConfig(ctx)
defer tracelib.InitGlobalTracer(context.Background()).Shutdown()
// Set job context initializer
runtime.JobService.SetJobContextInitializer(func(ctx context.Context) (job.Context, error) {
secret := config.GetAuthSecret()

27
src/jobservice/runner/redis.go
View File

@ -15,6 +15,7 @@
package runner
import (
"context"
"fmt"
"runtime"
"time"
@ -28,10 +29,14 @@ import (
"github.com/goharbor/harbor/src/jobservice/period"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/metric"
tracelib "github.com/goharbor/harbor/src/lib/trace"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
)
const (
maxTrackRetries = 6
tracerName = "goharbor/harbor/src/jobservice/runner/redis"
)
// RedisJob is a job wrapper to wrap the job.Interface to the style which can be recognized by the redis worker.
@ -52,6 +57,9 @@ func NewRedisJob(job interface{}, ctx *env.Context, ctl lcm.Controller) *RedisJo
// Run the job
func (rj *RedisJob) Run(j *work.Job) (err error) {
_, span := tracelib.StartTrace(context.Background(), tracerName, "run-job")
defer span.End()
var (
runningJob job.Interface
execContext job.Context
@ -65,9 +73,10 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
// Check if the job is a periodic one as periodic job has its own ID format
if eID, yes := isPeriodicJobExecution(j); yes {
jID = eID
span.SetAttributes(attribute.Key("periodicJob").Bool(true))
logger.Infof("Start to run periodical job execution: %s", eID)
}
span.SetAttributes(attribute.Key("jobID").String(jID), attribute.Key("jobName").String(j.Name))
// As the job stats may not be ready when job executing sometimes (corner case),
// the track call here may get NOT_FOUND error. For that case, let's do retry to recovery.
@ -76,7 +85,6 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
if err == nil {
break
}
if errs.IsObjectNotFoundError(err) {
if retried < maxTrackRetries {
// Still have chance to re-track the given job.
@ -84,6 +92,7 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
b := backoff(retried)
logger.Errorf("Track job %s: stats may not have been ready yet, hold for %d ms and retry again", jID, b)
<-time.After(time.Duration(b) * time.Millisecond)
span.AddEvent("retrying to get job stat")
continue
} else {
// Exit and never try.
@ -94,7 +103,7 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
// Log error and exit
logger.Errorf("Job '%s:%s' exit with error: failed to get job tracker: %s", j.Name, j.ID, err)
tracelib.RecordError(span, err, "failed to get job tracker")
// ELSE:
// As tracker creation failed, there is no way to mark the job status change.
// Also a non nil error return consumes a fail. If all retries are failed here,
@ -113,8 +122,10 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
logger.Errorf("Job '%s:%s' exit with error: %s", j.Name, j.ID, err)
metric.JobserviceTotalTask.WithLabelValues(j.Name, "fail").Inc()
metric.JobservieTaskProcessTimeSummary.WithLabelValues(j.Name, "fail").Observe(time.Since(now).Seconds())
tracelib.RecordError(span, err, "job failed with err")
if er := tracker.Fail(); er != nil {
logger.Errorf("Error occurred when marking the status of job %s:%s to failure: %s", j.Name, j.ID, er)
span.RecordError(err)
}
return
@ -129,6 +140,7 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
// Logged
metric.JobserviceTotalTask.WithLabelValues(j.Name, "stop").Inc()
metric.JobservieTaskProcessTimeSummary.WithLabelValues(j.Name, "stop").Observe(time.Since(now).Seconds())
tracelib.RecordError(span, err, "job stopped")
logger.Infof("Job %s:%s is stopped", j.Name, j.ID)
return
}
@ -137,7 +149,9 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
metric.JobservieTaskProcessTimeSummary.WithLabelValues(j.Name, "success").Observe(time.Since(now).Seconds())
// Mark job status to success.
logger.Infof("Job '%s:%s' exit with success", j.Name, j.ID)
span.SetStatus(codes.Ok, "job exit with success")
if er := tracker.Succeed(); er != nil {
tracelib.RecordError(span, err, "failed to mark job success")
logger.Errorf("Error occurred when marking the status of job %s:%s to success: %s", j.Name, j.ID, er)
}
}()
@ -169,6 +183,7 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
// Reset job info.
if err = tracker.Reset(); err != nil {
// Log error and return the original error if existing
tracelib.RecordError(span, err, "reset job failed")
err = errors.Wrap(err, fmt.Sprintf("retrying %s job %s:%s failed", jStatus.String(), j.Name, j.ID))
if len(j.LastErr) > 0 {
@ -184,7 +199,9 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
// do nothing
return nil
default:
return errors.Errorf("mismatch status for running job: expected %s/%s but got %s", job.PendingStatus, job.ScheduledStatus, jStatus.String())
err = errors.Errorf("mismatch status for running job: expected %s/%s but got %s", job.PendingStatus, job.ScheduledStatus, jStatus.String())
tracelib.RecordError(span, err, "status mismatch")
return err
}
// Build job context
@ -197,6 +214,7 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
// Close open io stream first
if closer, ok := execContext.GetLogger().(logger.Closer); ok {
if er := closer.Close(); er != nil {
tracelib.RecordError(span, er, "close job logger failed")
logger.Errorf("Close job logger failed: %s", er)
}
}
@ -206,6 +224,7 @@ func (rj *RedisJob) Run(j *work.Job) (err error) {
runningJob = Wrap(rj.job)
// Set status to run
if err = tracker.Run(); err != nil {
tracelib.RecordError(span, err, "failed set status to run")
return
}
// Run the job

19
src/lib/config/metadata/metadatalist.go
View File

@ -156,11 +156,28 @@ var (
{Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "30", ItemType: &IntType{}, Editable: true, Description: `The robot account token duration in days`},
{Name: common.RobotNamePrefix, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_NAME_PREFIX", DefaultValue: "robot$", ItemType: &StringType{}, Editable: true, Description: `The rebot account name prefix`},
{Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true, Description: `Enable notification`},
{Name: common.MetricEnable, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_ENABLE", DefaultValue: "false", ItemType: &BoolType{}, Editable: true},
{Name: common.MetricPort, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PORT", DefaultValue: "9090", ItemType: &IntType{}, Editable: true},
{Name: common.MetricPort, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PORT", DefaultValue: "9090", ItemType: &PortType{}, Editable: true},
{Name: common.MetricPath, Scope: SystemScope, Group: BasicGroup, EnvKey: "METRIC_PATH", DefaultValue: "/metrics", ItemType: &StringType{}, Editable: true},
{Name: common.QuotaPerProjectEnable, Scope: UserScope, Group: QuotaGroup, EnvKey: "QUOTA_PER_PROJECT_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true, Description: `Enable quota per project`},
{Name: common.StoragePerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "STORAGE_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true, Description: `The storage quota per project`},
{Name: common.TraceEnabled, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_ENABLED", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `Enable trace`},
{Name: common.TraceServiceName, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_SERVICE_NAME", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The service name of the trace`},
{Name: common.TraceNamespace, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_NAMESPACE", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The namespace of the trace`},
{Name: common.TraceSampleRate, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_SAMPLE_RATE", DefaultValue: "1", ItemType: &Float64Type{}, Editable: false, Description: `The sample rate of the trace`},
{Name: common.TraceAttributes, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_ATTRIBUTES", DefaultValue: "", ItemType: &StringToStringMapType{}, Editable: false, Description: `The attribute of the trace`},
{Name: common.TraceJaegerEndpoint, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_JAEGER_ENDPOINT", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The endpoint of the Jaeger`},
{Name: common.TraceJaegerUsername, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_JAEGER_USERNAME", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The username of the Jaeger`},
{Name: common.TraceJaegerPassword, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_JAEGER_PASSWORD", DefaultValue: "", ItemType: &PasswordType{}, Editable: false, Description: `The password of the Jaeger`},
{Name: common.TraceJaegerAgentHost, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_JAEGER_AGENT_HOSTNAME", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The agent host of the Jaeger`},
{Name: common.TraceJaegerAgentPort, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_JAEGER_AGENT_PORT", DefaultValue: "6831", ItemType: &StringType{}, Editable: false, Description: `The agent port of the Jaeger`},
{Name: common.TraceOtelEndpoint, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_OTEL_ENDPOINT", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The endpoint of the Otel`},
{Name: common.TraceOtelURLPath, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_OTEL_URL_PATH", DefaultValue: "", ItemType: &StringType{}, Editable: false, Description: `The URL path of the Otel`},
{Name: common.TraceOtelCompression, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_OTEL_COMPRESSION", DefaultValue: "", ItemType: &BoolType{}, Editable: false, Description: `The compression of the Otel`},
{Name: common.TraceOtelInsecure, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_OTEL_INSECURE", DefaultValue: "", ItemType: &BoolType{}, Editable: false, Description: `The insecure of the Otel`},
{Name: common.TraceOtelTimeout, Scope: SystemScope, Group: BasicGroup, EnvKey: "TRACE_OTEL_TIMEOUT", DefaultValue: "", ItemType: &IntType{}, Editable: false, Description: `The timeout of the Otel`},
}
)

20
src/lib/config/metadata/type.go
View File

@ -148,6 +148,17 @@ func (t *Int64Type) get(str string) (interface{}, error) {
return parseInt64(str)
}
type Float64Type struct{}
func (f *Float64Type) validate(str string) error {
_, err := parseFloat64(str)
return err
}
func (f *Float64Type) get(str string) (interface{}, error) {
return parseFloat64(str)
}
// BoolType ...
type BoolType struct {
}
@ -251,3 +262,12 @@ func parseInt(str string) (int, error) {
return 0, fmt.Errorf("invalid int string: %s", str)
}
func parseFloat64(str string) (float64, error) {
val, err := strconv.ParseFloat(str, 64)
if err == nil {
return val, nil
}
return 0, fmt.Errorf("invalid float64 string: %s", str)
}

16
src/lib/config/metadata/value.go
View File

@ -95,6 +95,22 @@ func (c *ConfigureValue) GetInt64() int64 {
return 0
}
// GetFloat64 - return the float64 value of current value
func (c *ConfigureValue) GetFloat64() float64 {
if item, ok := Instance().GetByName(c.Name); ok {
val, err := item.ItemType.get(c.Value)
if err != nil {
log.Errorf("GetFloat64 failed, error: %+v", err)
return 0
}
if float64Value, suc := val.(float64); suc {
return float64Value
}
}
log.Errorf("GetFloat64 failed, the current value's metadata is not defined, %+v", c)
return 0
}
// GetBool - return the bool value of current setting
func (c *ConfigureValue) GetBool() bool {
if item, ok := Instance().GetByName(c.Name); ok {

9
src/lib/config/metadata/value_test.go
View File

@ -16,8 +16,9 @@ package metadata
import (
"fmt"
"github.com/stretchr/testify/assert"
"testing"
"github.com/stretchr/testify/assert"
)
var testingMetaDataArray = []Item{
@ -27,6 +28,7 @@ var testingMetaDataArray = []Item{
{Name: "ldap_verify_cert", ItemType: &BoolType{}, Scope: "user", Group: "ldapbasic"},
{Name: "sample_map_setting", ItemType: &MapType{}, Scope: "user", Group: "ldapbasic"},
{Name: "scan_all_policy", ItemType: &MapType{}, Scope: "user", Group: "basic"},
{Name: "sample_rate", ItemType: &Float64Type{}, Scope: "system", Group: "basic"},
}
// createCfgValue ... Create a ConfigureValue object, only used in test
@ -67,6 +69,11 @@ func TestConfigureValue_GetInt64(t *testing.T) {
assert.Equal(t, createCfgValue("ulimit", "99999").GetInt64(), int64(99999))
}
func TestConfigureValue_GetFloat64(t *testing.T) {
Instance().initFromArray(testingMetaDataArray)
assert.Equal(t, createCfgValue("sample_rate", "0.5").GetFloat64(), float64(0.5))
}
func TestNewScanAllPolicy(t *testing.T) {
Instance().initFromArray(testingMetaDataArray)
value, err := NewCfgValue("scan_all_policy", `{"parameter":{"daily_time":0},"type":"daily"}`)

37
src/lib/config/trace.go Normal file
View File

@ -0,0 +1,37 @@
package config
import (
"context"
"github.com/goharbor/harbor/src/common"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/log"
tracelib "github.com/goharbor/harbor/src/lib/trace"
)
func InitTraceConfig(ctx context.Context) {
cfgMgr, err := GetManager(common.InMemoryCfgManager)
if err != nil {
log.Fatalf("failed to get config manager: %v", err)
}
tracelib.InitGlobalConfig(
tracelib.WithEnabled(cfgMgr.Get(ctx, common.TraceEnabled).GetBool()),
tracelib.WithServiceName(cfgMgr.Get(ctx, common.TraceServiceName).GetString()),
tracelib.WithNamespace(cfgMgr.Get(ctx, common.TraceNamespace).GetString()),
tracelib.WithSampleRate(cfgMgr.Get(ctx, common.TraceSampleRate).GetFloat64()),
tracelib.WithAttributes(cfgMgr.Get(ctx, common.TraceAttributes).GetStringToStringMap()),
tracelib.WithJaegerEndpoint(cfgMgr.Get(ctx, common.TraceJaegerEndpoint).GetString()),
tracelib.WithJaegerUsername(cfgMgr.Get(ctx, common.TraceJaegerUsername).GetString()),
tracelib.WithJaegerPassword(cfgMgr.Get(ctx, common.TraceJaegerPassword).GetString()),
tracelib.WithJaegerAgentHost(cfgMgr.Get(ctx, common.TraceJaegerAgentHost).GetString()),
tracelib.WithJaegerAgentPort(cfgMgr.Get(ctx, common.TraceJaegerAgentPort).GetString()),
tracelib.WithOtelEndpoint(cfgMgr.Get(ctx, common.TraceOtelEndpoint).GetString()),
tracelib.WithOtelURLPath(cfgMgr.Get(ctx, common.TraceOtelURLPath).GetString()),
tracelib.WithOtelCompression(cfgMgr.Get(ctx, common.TraceOtelCompression).GetBool()),
tracelib.WithOtelInsecure(cfgMgr.Get(ctx, common.TraceOtelInsecure).GetBool()),
tracelib.WithOtelTimeout(cfgMgr.Get(ctx, common.TraceOtelTimeout).GetInt()),
)
if tracelib.Enabled() {
commonhttp.AddTracingWithGlobalTransport()
}
}

40
src/lib/orm/orm.go
View File

@ -23,7 +23,9 @@ import (
"strings"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/lib/log"
tracelib "github.com/goharbor/harbor/src/lib/trace"
)
// NewCondition alias function of orm.NewCondition
@ -48,6 +50,11 @@ func RegisterModel(models ...interface{}) {
type ormKey struct{}
const (
tracerName = "goharbor/harbor/src/lib/orm"
defaultTranscationOpName = "start-transaction"
)
func init() {
if os.Getenv("ORM_DEBUG") == "true" {
orm.Debug = true
@ -81,30 +88,59 @@ func Clone(ctx context.Context) context.Context {
return NewContext(ctx, orm.NewOrm())
}
type operationNameKey struct{}
// SetTransactionOpName sets the transaction operation name
func SetTransactionOpNameToContext(ctx context.Context, name string) context.Context {
if ctx == nil {
ctx = context.Background()
}
return context.WithValue(ctx, operationNameKey{}, name)
}
// GetTransactionOpNameFromContext returns the transaction operation name from context
func GetTransactionOpNameFromContext(ctx context.Context) string {
opName, ok := ctx.Value(operationNameKey{}).(string)
if !ok {
return defaultTranscationOpName
}
if opName == "" {
return defaultTranscationOpName
}
return opName
}
// WithTransaction a decorator which make f run in transaction
func WithTransaction(f func(ctx context.Context) error) func(ctx context.Context) error {
return func(ctx context.Context) error {
cx, span := tracelib.StartTrace(ctx, tracerName, GetTransactionOpNameFromContext(ctx))
defer span.End()
o, err := FromContext(ctx)
if err != nil {
tracelib.RecordError(span, err, "get orm from ctx failed")
return err
}
tx := ormerTx{Ormer: o}
if err := tx.Begin(); err != nil {
tracelib.RecordError(span, err, "begin transaction failed")
log.Errorf("begin transaction failed: %v", err)
return err
}
if err := f(ctx); err != nil {
if err := f(cx); err != nil {
span.AddEvent("rollback transaction")
if e := tx.Rollback(); e != nil {
tracelib.RecordError(span, e, "rollback transaction failed")
log.Errorf("rollback transaction failed: %v", e)
return e
}
return err
}
span.AddEvent("commit transaction")
if err := tx.Commit(); err != nil {
tracelib.RecordError(span, err, "commit transaction failed")
log.Errorf("commit transaction failed: %v", err)
return err
}

182
src/lib/trace/config.go Normal file
View File

@ -0,0 +1,182 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package trace
import (
"fmt"
)
const (
TraceEnvPrefix = "trace"
)
// C is the global configuration for trace
var C Config
func InitGlobalConfig(opts ...Option) {
C = NewConfig(opts...)
}
// OtelConfig is the configuration for otel
type OtelConfig struct {
Endpoint string `mapstructure:"otel_trace_endpoint"`
URLPath string `mapstructure:"otel_trace_url_path"`
Compression bool `mapstructure:"otel_trace_compression"`
Insecure bool `mapstructure:"otel_trace_insecure"`
Timeout int `mapstructure:"otel_trace_timeout"`
}
func (c *OtelConfig) String() string {
return fmt.Sprintf("endpoint: %s, url_path: %s, compression: %t, insecure: %t, timeout: %d",
c.Endpoint, c.URLPath, c.Compression, c.Insecure, c.Timeout)
}
// JaegerConfig is the configuration for Jaeger
type JaegerConfig struct {
Endpoint string `mapstructure:"jaeger_endpoint"`
Username string `mapstructure:"jaeger_username"`
Password string `mapstructure:"jaeger_password"`
AgentHost string `mapstructure:"jaeger_agent_host"`
AgentPort string `mapstructure:"jaeger_agent_port"`
}
func (c *JaegerConfig) String() string {
return fmt.Sprintf("endpoint: %s, username: %s, password: %s, agent_host: %s, agent_port: %s",
c.Endpoint, c.Username, c.Password, c.AgentHost, c.AgentPort)
}
// Config is the configuration for trace
type Config struct {
Enabled bool `mapstructure:"enabled"`
SampleRate float64 `mapstructure:"sample_rate"`
Namespace string `mapstructure:"namespace"`
ServiceName string `mapstructure:"service_name"`
Jaeger JaegerConfig
Otel OtelConfig
Attributes map[string]string
}
func (c *Config) String() string {
return fmt.Sprintf("{Enabled: %v, ServiceName: %v, SampleRate: %v, Namespace: %v, ServiceName: %v, Jaeger: %v, Otel: %v}", c.Enabled, c.ServiceName, c.SampleRate, c.Namespace, c.ServiceName, c.Jaeger, c.Otel)
}
type Option func(*Config)
func WithEnabled(enabled bool) Option {
return func(c *Config) {
c.Enabled = enabled
}
}
func WithSampleRate(sampleRate float64) Option {
return func(c *Config) {
c.SampleRate = sampleRate
}
}
func WithNamespace(namespace string) Option {
return func(c *Config) {
c.Namespace = namespace
}
}
func WithServiceName(serviceName string) Option {
return func(c *Config) {
c.ServiceName = serviceName
}
}
func WithAttributes(attributes map[string]string) Option {
return func(c *Config) {
c.Attributes = attributes
}
}
func WithJaegerEndpoint(endpoint string) Option {
return func(c *Config) {
c.Jaeger.Endpoint = endpoint
}
}
func WithJaegerUsername(username string) Option {
return func(c *Config) {
c.Jaeger.Username = username
}
}
func WithJaegerPassword(password string) Option {
return func(c *Config) {
c.Jaeger.Password = password
}
}
func WithJaegerAgentHost(host string) Option {
return func(c *Config) {
c.Jaeger.AgentHost = host
}
}
func WithJaegerAgentPort(port string) Option {
return func(c *Config) {
c.Jaeger.AgentPort = port
}
}
func WithOtelEndpoint(endpoint string) Option {
return func(c *Config) {
c.Otel.Endpoint = endpoint
}
}
func WithOtelURLPath(urlPath string) Option {
return func(c *Config) {
c.Otel.URLPath = urlPath
}
}
func WithOtelCompression(compression bool) Option {
return func(c *Config) {
c.Otel.Compression = compression
}
}
func WithOtelInsecure(insecure bool) Option {
return func(c *Config) {
c.Otel.Insecure = insecure
}
}
func WithOtelTimeout(timeout int) Option {
return func(c *Config) {
c.Otel.Timeout = timeout
}
}
func NewConfig(opts ...Option) Config {
c := Config{Otel: OtelConfig{}, Jaeger: JaegerConfig{}}
for _, opt := range opts {
opt(&c)
}
return c
}
// GetConfig returns the global configuration for trace
func GetGlobalConfig() Config {
return C
}
// Enabled returns whether trace is enabled
func Enabled() bool {
return C.Enabled
}

64
src/lib/trace/helper.go Normal file
View File

@ -0,0 +1,64 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package trace
import (
"context"
"net/http"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/codes"
oteltrace "go.opentelemetry.io/otel/trace"
)
// GetGlobalTracer returns the global tracer.
func GetGlobalTracer(instrumentationName string, opts ...oteltrace.TracerOption) oteltrace.Tracer {
return otel.GetTracerProvider().Tracer(instrumentationName, opts...)
}
// StartSpan starts a span with the given name.
func StartSpan(ctx context.Context, name string) (context.Context, oteltrace.Span) {
return otel.Tracer("goharbor/harbor/src/lib/trace").Start(ctx, name)
}
// SpanFromContext returns the span from the context.
func SpanFromHTTPRequest(req *http.Request) oteltrace.Span {
ctx := req.Context()
return oteltrace.SpanFromContext(ctx)
}
// RecordError records the error in the span from context.
func RecordError(span oteltrace.Span, err error, description string) {
if span == nil {
return
}
span.RecordError(err)
span.SetStatus(codes.Error, description)
}
// NewHandler returns a handler that wraps the given handler with tracing.
func NewHandler(h http.Handler, operation string) http.Handler {
httpOptions := []otelhttp.Option{
otelhttp.WithTracerProvider(otel.GetTracerProvider()),
otelhttp.WithPropagators(otel.GetTextMapPropagator()),
}
return otelhttp.NewHandler(h, operation, httpOptions...)
}
// StarTrace returns a new span with the given name.
func StartTrace(ctx context.Context, tracerName string, spanName string, opts ...oteltrace.SpanStartOption) (context.Context, oteltrace.Span) {
return otel.Tracer(tracerName).Start(ctx, spanName, opts...)
}

137
src/lib/trace/trace.go Normal file
View File

@ -0,0 +1,137 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package trace
import (
"context"
"time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/jaeger"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
tracesdk "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
oteltrace "go.opentelemetry.io/otel/trace"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/version"
)
func initExporter(ctx context.Context) (tracesdk.SpanExporter, error) {
var err error
var exp tracesdk.SpanExporter
cfg := GetGlobalConfig()
if len(cfg.Jaeger.Endpoint) != 0 {
// Jaeger collector exporter
log.Infof("init trace provider jaeger collector on %s with user %s", cfg.Jaeger.Endpoint, cfg.Jaeger.Username)
exp, err = jaeger.New(jaeger.WithCollectorEndpoint(
jaeger.WithEndpoint(cfg.Jaeger.Endpoint),
jaeger.WithUsername(cfg.Jaeger.Username),
jaeger.WithPassword(cfg.Jaeger.Password),
))
} else if len(cfg.Jaeger.AgentHost) != 0 {
// Jaeger agent exporter
log.Infof("init trace provider jaeger agent on %s", cfg.Jaeger.AgentHost)
exp, err = jaeger.New(jaeger.WithAgentEndpoint(
jaeger.WithAgentHost(cfg.Jaeger.AgentHost),
jaeger.WithAgentPort(cfg.Jaeger.AgentPort),
))
} else if len(cfg.Otel.Endpoint) != 0 {
// Otel exporter
log.Infof("init trace provider otel on %s/%s", cfg.Otel.Endpoint, cfg.Otel.URLPath)
opts := []otlptracehttp.Option{
otlptracehttp.WithEndpoint(cfg.Otel.Endpoint),
otlptracehttp.WithURLPath(cfg.Otel.URLPath),
otlptracehttp.WithTimeout(time.Duration(cfg.Otel.Timeout) * time.Second),
}
if cfg.Otel.Compression {
opts = append(opts, otlptracehttp.WithCompression(otlptracehttp.GzipCompression))
}
if cfg.Otel.Insecure {
opts = append(opts, otlptracehttp.WithInsecure())
}
exp, err = otlptracehttp.New(ctx, opts...)
} else {
log.Fatalf("Trace is enabled but no tracer provider is specified")
}
return exp, err
}
func initProvider(exp tracesdk.SpanExporter) *tracesdk.TracerProvider {
cfg := GetGlobalConfig()
// prepare attribute resources
attriSlice := []attribute.KeyValue{
semconv.ServiceNameKey.String(cfg.ServiceName),
}
if len(version.ReleaseVersion) != 0 {
attriSlice = append(attriSlice, semconv.ServiceVersionKey.String(version.ReleaseVersion))
}
if cfg.Namespace != "" {
attriSlice = append(attriSlice, semconv.ServiceNamespaceKey.String(cfg.Namespace))
}
if cfg.Attributes != nil {
for i, a := range cfg.Attributes {
attriSlice = append(attriSlice, attribute.String(i, a))
}
}
// prepare tp options
ops := make([]tracesdk.TracerProviderOption, 0, 4)
ops = append(ops,
// Always be sure to batch in production.
// tracesdk.WithBatcher(exp),
tracesdk.WithBatcher(exp),
// Record information about this application in an Resource.
tracesdk.WithResource(resource.NewWithAttributes(semconv.SchemaURL, attriSlice...)),
tracesdk.WithSampler(tracesdk.TraceIDRatioBased(cfg.SampleRate)),
)
// init trace provider
return tracesdk.NewTracerProvider(ops...)
}
// ShutdownFunc is a function to shutdown the trace provider
type ShutdownFunc func()
// Shutdown shutdown the trace provider
func (s ShutdownFunc) Shutdown() {
s()
}
// Init initializes the trace provider
func InitGlobalTracer(ctx context.Context) ShutdownFunc {
if !Enabled() {
otel.SetTracerProvider(oteltrace.NewNoopTracerProvider())
return func() {}
}
exp, err := initExporter(ctx)
handleErr(err, "fail in exporter initialization")
tp := initProvider(exp)
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
return func() {
log.Infof("shutdown trace provider")
handleErr(tp.Shutdown(ctx), "fail in tracer shutdown")
}
}
func handleErr(err error, message string) {
if err != nil {
log.Fatalf("%s: %v", message, err)
}
}

6
src/pkg/config/rest/rest.go
View File

@ -17,7 +17,6 @@ package rest
import (
"context"
"errors"
"net/http"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
@ -32,11 +31,6 @@ type Driver struct {
// NewRESTDriver - Create Driver
func NewRESTDriver(configRESTURL string, modifiers ...modifier.Modifier) *Driver {
if commonhttp.InternalTLSEnabled() {
tr := commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
return &Driver{configRESTURL: configRESTURL, client: commonhttp.NewClient(&http.Client{Transport: tr}, modifiers...)}
}
return &Driver{configRESTURL: configRESTURL, client: commonhttp.NewClient(nil, modifiers...)}
}

7
src/pkg/notification/policy/manager.go
View File

@ -3,6 +3,9 @@ package policy
import (
"context"
"fmt"
"net/http"
"time"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
@ -10,8 +13,6 @@ import (
"github.com/goharbor/harbor/src/pkg/notification/policy/dao"
"github.com/goharbor/harbor/src/pkg/notification/policy/model"
notifier_model "github.com/goharbor/harbor/src/pkg/notifier/model"
"net/http"
"time"
)
var (
@ -159,7 +160,7 @@ func (m *manager) policyHTTPTest(address string, skipCertVerify bool) error {
req.Header.Set("Content-Type", "application/json")
client := http.Client{
Transport: commonhttp.GetHTTPTransportByInsecure(skipCertVerify),
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(skipCertVerify)),
}
resp, err := client.Do(req)

2
src/pkg/p2p/preheat/dao/instance/dao.go
View File

@ -106,7 +106,7 @@ func (d *dao) Update(ctx context.Context, instance *provider.Instance, props ...
_, err = o.Update(instance, props...)
return
}
return orm.WithTransaction(trans)(ctx)
return orm.WithTransaction(trans)(orm.SetTransactionOpNameToContext(ctx, "tx-prehead-update"))
}

2
src/pkg/project/dao/dao.go
View File

@ -86,7 +86,7 @@ func (d *dao) Create(ctx context.Context, project *models.Project) (int64, error
return nil
}
if err := orm.WithTransaction(h)(ctx); err != nil {
if err := orm.WithTransaction(h)(orm.SetTransactionOpNameToContext(ctx, "tx-create-project")); err != nil {
return 0, err
}

4
src/pkg/project/metadata/manager.go
View File

@ -65,7 +65,7 @@ func (m *manager) Add(ctx context.Context, projectID int64, meta map[string]stri
}
return nil
}
return orm.WithTransaction(h)(ctx)
return orm.WithTransaction(h)(orm.SetTransactionOpNameToContext(ctx, "tx-add-project"))
}
// Delete metadatas whose keys are specified in parameter meta, if it is absent, delete all
@ -89,7 +89,7 @@ func (m *manager) Update(ctx context.Context, projectID int64, meta map[string]s
return nil
}
return orm.WithTransaction(h)(ctx)
return orm.WithTransaction(h)(orm.SetTransactionOpNameToContext(ctx, "tx-delete-project"))
}
// Get metadatas whose keys are specified in parameter meta, if it is absent, get all

4
src/pkg/quota/manager.go
View File

@ -74,7 +74,7 @@ func (m *manager) Create(ctx context.Context, reference, referenceID string, har
return err
}
err = orm.WithTransaction(h)(ctx)
err = orm.WithTransaction(h)(orm.SetTransactionOpNameToContext(ctx, "tx-create-quota"))
return id, err
}
@ -88,7 +88,7 @@ func (m *manager) Delete(ctx context.Context, id int64) error {
return m.dao.Delete(ctx, id)
}
return orm.WithTransaction(h)(ctx)
return orm.WithTransaction(h)(orm.SetTransactionOpNameToContext(ctx, "tx-delete-quota"))
}
func (m *manager) Get(ctx context.Context, id int64) (*Quota, error) {

9
src/pkg/reg/adapter/aliacr/adapter.go
View File

@ -61,7 +61,7 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
return nil, err
}
credential := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret)
authorizer := bearer.NewAuthorizer(realm, service, credential, util.GetHTTPTransport(registry.Insecure))
authorizer := bearer.NewAuthorizer(realm, service, credential, commonhttp.GetHTTPTransport(commonhttp.WithInsecure(registry.Insecure)))
return &adapter{
region: region,
registry: registry,
@ -71,11 +71,8 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
}
func ping(registry *model.Registry) (string, string, error) {
client := &http.Client{}
if registry.Insecure {
client.Transport = commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
} else {
client.Transport = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
client := &http.Client{
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(registry.Insecure)),
}
resp, err := client.Get(registry.URL + "/v2/")

9
src/pkg/reg/adapter/artifacthub/client.go
View File

@ -3,11 +3,12 @@ package artifacthub
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
"io/ioutil"
"net/http"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/reg/model"
)
// Client is a client to interact with Artifact Hub
@ -19,7 +20,7 @@ type Client struct {
func newClient(registry *model.Registry) *Client {
return &Client{
httpClient: &http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
Transport: common_http.GetHTTPTransport(common_http.WithInsecure(registry.Insecure)),
},
}
}

25
src/pkg/reg/adapter/awsecr/auth.go
View File

@ -18,19 +18,21 @@ import (
"encoding/base64"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/lib/log"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
"github.com/aws/aws-sdk-go/aws/session"
awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/lib/log"
"net/http"
"net/url"
"strings"
"time"
)
// Credential ...
@ -103,17 +105,12 @@ func getAwsSvc(region, accessKey, accessSecret string, insecure bool, forceEndpo
} else {
cred = ec2rolecreds.NewCredentials(sess)
}
var tr *http.Transport
if insecure {
tr = commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
} else {
tr = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
}
config := &aws.Config{
Credentials: cred,
Region: &region,
HTTPClient: &http.Client{
Transport: tr,
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(insecure)),
},
}
if forceEndpoint != nil {

4
src/pkg/reg/adapter/dockerhub/client.go
View File

@ -8,9 +8,9 @@ import (
"io/ioutil"
"net/http"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
)
// Client is a client to interact with DockerHub
@ -26,7 +26,7 @@ func NewClient(registry *model.Registry) (*Client, error) {
client := &Client{
host: registry.URL,
client: &http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(registry.Insecure)),
},
}

3
src/pkg/reg/adapter/dtr/client.go
View File

@ -15,7 +15,6 @@ import (
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
)
// Client is a client to interact with DTR
@ -35,7 +34,7 @@ func NewClient(registry *model.Registry) *Client {
password: registry.Credential.AccessSecret,
client: common_http.NewClient(
&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
Transport: common_http.GetHTTPTransport(common_http.WithInsecure(registry.Insecure)),
}),
}
return client

3
src/pkg/reg/adapter/dtr/client_test.go
View File

@ -6,7 +6,6 @@ import (
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/pkg/reg/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -164,7 +163,7 @@ func TestProjects(t *testing.T) {
username: "test",
client: common_http.NewClient(
&http.Client{
Transport: util.GetHTTPTransport(true),
Transport: common_http.GetHTTPTransport(common_http.WithInsecure(true)),
}),
}

2
src/pkg/reg/adapter/githubcr/adapter.go
View File

@ -77,7 +77,7 @@ func newAdapter(registry *model.Registry) *adapter {
registry.Credential.AccessSecret)
}
var transport = util.GetHTTPTransport(registry.Insecure)
var transport = common_http.GetHTTPTransport(common_http.WithInsecure(registry.Insecure))
return &adapter{
Adapter: native.NewAdapter(registry),

15
src/pkg/reg/adapter/gitlab/client.go
View File

@ -4,17 +4,16 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/docker/distribution/registry/client/auth/challenge"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
"io"
"io/ioutil"
"net/http"
common_http "github.com/goharbor/harbor/src/common/http"
"net/url"
"reflect"
"github.com/docker/distribution/registry/client/auth/challenge"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/reg/model"
)
const (
@ -33,7 +32,7 @@ type Client struct {
func NewClient(registry *model.Registry) (*Client, error) {
realm, _, err := ping(&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
Transport: common_http.GetHTTPTransport(common_http.WithInsecure(registry.Insecure)),
}, registry.URL)
if err != nil {
return nil, err
@ -51,7 +50,7 @@ func NewClient(registry *model.Registry) (*Client, error) {
token: registry.Credential.AccessSecret,
client: common_http.NewClient(
&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
Transport: common_http.GetHTTPTransport(common_http.WithInsecure(registry.Insecure)),
}),
}
return client, nil

12
src/pkg/reg/adapter/gitlab/client_test.go
View File

@ -1,13 +1,13 @@
package gitlab
import (
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/pkg/reg/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"testing"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestProjects(t *testing.T) {
@ -76,7 +76,7 @@ func TestProjects(t *testing.T) {
token: "test",
client: common_http.NewClient(
&http.Client{
Transport: util.GetHTTPTransport(true),
Transport: common_http.GetHTTPTransport(common_http.WithInsecure(true)),
}),
}
projects, e := client.getProjects()

4
src/pkg/reg/adapter/harbor/base/adapter.go
View File

@ -40,7 +40,7 @@ func New(registry *model.Registry) (*Adapter, error) {
// core, so insecure transport is ok
// If using the secure one, as we'll replace the URL with 127.0.0.1 and this will
// cause error "x509: cannot validate certificate for 127.0.0.1 because it doesn't contain any IP SANs"
Transport: common_http.GetHTTPTransport(common_http.InsecureTransport),
Transport: common_http.GetHTTPTransport(common_http.WithInsecure(true)),
}, authorizer)
client, err := NewClient(registry.URL, httpClient)
if err != nil {
@ -62,7 +62,7 @@ func New(registry *model.Registry) (*Adapter, error) {
registry.Credential.AccessSecret))
}
httpClient := common_http.NewClient(&http.Client{
Transport: common_http.GetHTTPTransportByInsecure(registry.Insecure),
Transport: common_http.GetHTTPTransport(common_http.WithInsecure(registry.Insecure)),
}, authorizers...)
client, err := NewClient(registry.URL, httpClient)
if err != nil {

9
src/pkg/reg/adapter/helmhub/client.go
View File

@ -3,11 +3,12 @@ package helmhub
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
"io/ioutil"
"net/http"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/reg/model"
)
// ErrHTTPNotFound defines the return error when receiving 404 response code
@ -22,7 +23,7 @@ type Client struct {
func NewClient(registry *model.Registry) *Client {
return &Client{
client: &http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(registry.Insecure)),
},
}
}

3
src/pkg/reg/adapter/huawei/huawei_adapter.go
View File

@ -30,7 +30,6 @@ import (
adp "github.com/goharbor/harbor/src/pkg/reg/adapter"
"github.com/goharbor/harbor/src/pkg/reg/adapter/native"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
)
func init() {
@ -270,7 +269,7 @@ func newAdapter(registry *model.Registry) (adp.Adapter, error) {
modifiers = append(modifiers, authorizer)
}
transport := util.GetHTTPTransport(registry.Insecure)
transport := common_http.GetHTTPTransport(common_http.WithInsecure(registry.Insecure))
return &adapter{
Adapter: native.NewAdapter(registry),
registry: registry,

3
src/pkg/reg/adapter/jfrog/client.go
View File

@ -23,7 +23,6 @@ import (
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
)
@ -48,7 +47,7 @@ func newClient(reg *model.Registry) *client {
return &client{
client: common_http.NewClient(
&http.Client{
Transport: util.GetHTTPTransport(reg.Insecure),
Transport: common_http.GetHTTPTransport(common_http.WithInsecure(reg.Insecure)),
},
basic.NewAuthorizer(username, password),
),

3
src/pkg/reg/adapter/quay/adapter.go
View File

@ -31,7 +31,6 @@ import (
"github.com/goharbor/harbor/src/pkg/reg/adapter/native"
qauth "github.com/goharbor/harbor/src/pkg/reg/adapter/quay/auth"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
)
var (
@ -87,7 +86,7 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
registry: registry,
client: common_http.NewClient(
&http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
Transport: common_http.GetHTTPTransport(common_http.WithInsecure(registry.Insecure)),
},
modifiers...,
),

5
src/pkg/reg/adapter/tencentcr/adapter.go
View File

@ -16,7 +16,6 @@ import (
adp "github.com/goharbor/harbor/src/pkg/reg/adapter"
"github.com/goharbor/harbor/src/pkg/reg/adapter/native"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
"github.com/goharbor/harbor/src/pkg/registry/auth/bearer"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
@ -147,7 +146,7 @@ func newAdapter(registry *model.Registry) (a *adapter, err error) {
}
var credential = NewAuth(instanceInfo.RegistryId, client)
var transport = util.GetHTTPTransport(registry.Insecure)
var transport = commonhttp.GetHTTPTransport(commonhttp.WithInsecure(registry.Insecure))
var authorizer = bearer.NewAuthorizer(realm, service, credential, transport)
return &adapter{
@ -168,7 +167,7 @@ func newAdapter(registry *model.Registry) (a *adapter, err error) {
func ping(registry *model.Registry) (string, string, error) {
client := &http.Client{
Transport: util.GetHTTPTransport(registry.Insecure),
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(registry.Insecure)),
}
resp, err := client.Get(registry.URL + "/v2/")

11
src/pkg/reg/util/util.go
View File

@ -15,20 +15,9 @@
package util
import (
"net/http"
"strings"
commonhttp "github.com/goharbor/harbor/src/common/http"
)
// GetHTTPTransport can be used to share the common HTTP transport
func GetHTTPTransport(insecure bool) *http.Transport {
if insecure {
return commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
}
return commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
}
// ParseRepository parses the "repository" provided into two parts: namespace and the rest
// the string before the last "/" is the namespace part
// c -> [,c]

7
src/pkg/reg/util/util_test.go
View File

@ -20,13 +20,6 @@ import (
"github.com/stretchr/testify/assert"
)
func TestGetHTTPTransport(t *testing.T) {
transport := GetHTTPTransport(true)
assert.True(t, transport.TLSClientConfig.InsecureSkipVerify)
transport = GetHTTPTransport(false)
assert.False(t, transport.TLSClientConfig.InsecureSkipVerify)
}
func TestParseRepository(t *testing.T) {
// empty repository
repository := ""

4
src/pkg/registry/auth/authorizer.go
View File

@ -36,7 +36,7 @@ func NewAuthorizer(username, password string, insecure bool) lib.Authorizer {
username: username,
password: password,
client: &http.Client{
Transport: commonhttp.GetHTTPTransportByInsecure(insecure),
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(insecure)),
},
}
}
@ -101,7 +101,7 @@ func (a *authorizer) initialize(u *url.URL) error {
if challenge, exist := cm["bearer"]; exist {
a.authorizer = bearer.NewAuthorizer(challenge.Parameters["realm"],
challenge.Parameters["service"], basic.NewAuthorizer(a.username, a.password),
a.client.Transport.(*http.Transport))
a.client.Transport)
return nil
}
if _, exist := cm["basic"]; exist {

14
src/pkg/registry/auth/bearer/authorizer.go
View File

@ -17,11 +17,12 @@ package bearer
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/errors"
"io/ioutil"
"net/http"
"net/url"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/errors"
)
const (
@ -30,18 +31,15 @@ const (
// NewAuthorizer return a bearer token authorizer
// The parameter "a" is an authorizer used to fetch the token
func NewAuthorizer(realm, service string, a lib.Authorizer, transport ...*http.Transport) lib.Authorizer {
func NewAuthorizer(realm, service string, a lib.Authorizer, transport http.RoundTripper) lib.Authorizer {
authorizer := &authorizer{
realm: realm,
service: service,
authorizer: a,
cache: newCache(cacheCapacity),
}
tp := http.DefaultTransport
if len(transport) > 0 && transport[0] != nil {
tp = transport[0]
}
authorizer.client = &http.Client{Transport: tp}
authorizer.client = &http.Client{Transport: transport}
return authorizer
}

15
src/pkg/registry/auth/bearer/authorizer_test.go
View File

@ -16,13 +16,16 @@ package bearer
import (
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
)
func TestModify(t *testing.T) {
@ -39,7 +42,7 @@ func TestModify(t *testing.T) {
// invalid credential
a := basic.NewAuthorizer("username", "invalid_password")
authorizer := NewAuthorizer(server.URL, "service", a)
authorizer := NewAuthorizer(server.URL, "service", a, commonhttp.NewTransport())
req, _ := http.NewRequest(http.MethodGet, server.URL, nil)
err := authorizer.Modify(req)
require.NotNil(t, err)
@ -47,7 +50,7 @@ func TestModify(t *testing.T) {
// valid credential
a = basic.NewAuthorizer("username", "password")
authorizer = NewAuthorizer(server.URL, "service", a)
authorizer = NewAuthorizer(server.URL, "service", a, commonhttp.NewTransport())
req, _ = http.NewRequest(http.MethodGet, server.URL, nil)
err = authorizer.Modify(req)
require.Nil(t, err)

14
src/pkg/registry/client.go
View File

@ -18,7 +18,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/lib/config"
"io"
"io/ioutil"
"net/http"
@ -27,6 +26,8 @@ import (
"strings"
"time"
"github.com/goharbor/harbor/src/lib/config"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/manifestlist"
_ "github.com/docker/distribution/manifest/ocischema" // register oci manifest unmarshal function
@ -105,7 +106,7 @@ func NewClient(url, username, password string, insecure bool) Client {
url: url,
authorizer: auth.NewAuthorizer(username, password, insecure),
client: &http.Client{
Transport: commonhttp.GetHTTPTransportByInsecure(insecure),
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(insecure)),
Timeout: 30 * time.Minute,
},
}
@ -113,18 +114,11 @@ func NewClient(url, username, password string, insecure bool) Client {
// NewClientWithAuthorizer creates a registry client with the provided authorizer
func NewClientWithAuthorizer(url string, authorizer lib.Authorizer, insecure bool) Client {
var transportType uint
if insecure {
transportType = commonhttp.InsecureTransport
} else {
transportType = commonhttp.SecureTransport
}
return &client{
url: url,
authorizer: authorizer,
client: &http.Client{
Transport: commonhttp.GetHTTPTransport(transportType),
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(insecure)),
},
}
}

2
src/pkg/scan/dao/scan/vulnerability.go
View File

@ -140,7 +140,7 @@ func (v *vulnerabilityRecordDao) InsertForReport(ctx context.Context, reportUUID
return err
}
if err := orm.WithTransaction(h)(ctx); err != nil {
if err := orm.WithTransaction(h)(orm.SetTransactionOpNameToContext(ctx, "tx-insert-for-report")); err != nil {
fields := log.Fields{
"error": err,
"report": reportUUID,

2
src/pkg/scan/dao/scanner/registration.go
View File

@ -167,7 +167,7 @@ func SetDefaultRegistration(ctx context.Context, UUID string) error {
return err
}
return orm.WithTransaction(f)(ctx)
return orm.WithTransaction(f)(orm.SetTransactionOpNameToContext(ctx, "tx-scan-set-default-registration"))
}
// GetDefaultRegistration gets the default registration

2
src/pkg/scan/job.go
View File

@ -536,7 +536,7 @@ func makeBearerAuthorization(robotAccount *model.Robot, tokenURL string, reposit
req.Header.Set("Authorization", auth)
client := &http.Client{
Transport: commonhttp.GetHTTPTransportByInsecure(true),
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(true)),
}
resp, err := client.Do(req)

10
src/pkg/signature/notary/helper.go
View File

@ -18,23 +18,23 @@ import (
"context"
"encoding/hex"
"fmt"
"github.com/goharbor/harbor/src/lib/config"
"net/http"
"os"
"path"
"strings"
"github.com/docker/distribution/registry/auth/token"
commonhttp "github.com/goharbor/harbor/src/common/http"
tokenutil "github.com/goharbor/harbor/src/core/service/token"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/log"
model2 "github.com/goharbor/harbor/src/pkg/signature/notary/model"
"github.com/docker/distribution/registry/auth/token"
digest "github.com/opencontainers/go-digest"
"github.com/theupdateframework/notary"
"github.com/theupdateframework/notary/client"
"github.com/theupdateframework/notary/trustpinning"
"github.com/theupdateframework/notary/tuf/data"
digest "github.com/opencontainers/go-digest"
)
var (
@ -83,7 +83,7 @@ func GetTargets(ctx context.Context, notaryEndpoint string, username string, fqR
authorizer := &notaryAuthorizer{
token: t.Token,
}
tr := NewTransport(commonhttp.GetHTTPTransport(commonhttp.SecureTransport), authorizer)
tr := NewTransport(commonhttp.GetHTTPTransport(), authorizer)
gun := data.GUN(fqRepo)
notaryRepo, err := client.NewFileCachedRepository(notaryCachePath, gun, notaryEndpoint, tr, mockRetriever, trustPin)
if err != nil {

22
src/registryctl/api/registry/blob/blob.go
View File

@ -2,14 +2,21 @@ package blob
import (
"errors"
"net/http"
"github.com/goharbor/harbor/src/lib/log"
tracelib "github.com/goharbor/harbor/src/lib/trace"
"github.com/goharbor/harbor/src/registryctl/api"
"github.com/docker/distribution/registry/storage"
storagedriver "github.com/docker/distribution/registry/storage/driver"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/registryctl/api"
"github.com/gorilla/mux"
"net/http"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
const tracerName = "goharbor/harbor/src/registryctl/api/registry/blob"
// NewHandler returns the handler to handler blob request
func NewHandler(storageDriver storagedriver.StorageDriver) http.Handler {
return &handler{
@ -33,14 +40,19 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// DeleteBlob ...
func (h *handler) delete(w http.ResponseWriter, r *http.Request) {
ctx, span := tracelib.StartTrace(r.Context(), tracerName, "delete-blob", trace.WithAttributes(attribute.Key("method").String(r.Method)))
defer span.End()
ref := mux.Vars(r)["reference"]
if ref == "" {
api.HandleBadRequest(w, errors.New("no reference specified"))
err := errors.New("no reference specified")
tracelib.RecordError(span, err, "no reference specified")
api.HandleBadRequest(w, err)
return
}
// don't parse the reference here as RemoveBlob does.
cleaner := storage.NewVacuum(r.Context(), h.storageDriver)
cleaner := storage.NewVacuum(ctx, h.storageDriver)
if err := cleaner.RemoveBlob(ref); err != nil {
tracelib.RecordError(span, err, "failed to remove blob")
log.Infof("failed to remove blob: %s, with error:%v", ref, err)
api.HandleError(w, err)
return

7
src/registryctl/api/registry/blob/blob_test.go
View File

@ -1,14 +1,15 @@
package blob
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/docker/distribution/registry/storage/driver/inmemory"
"github.com/docker/distribution/testutil"
"github.com/goharbor/harbor/src/registryctl/api/registry/test"
"github.com/gorilla/mux"
"github.com/stretchr/testify/assert"
"net/http"
"net/http/httptest"
"testing"
)
func TestDeletionBlob(t *testing.T) {

31
src/registryctl/api/registry/manifest/manifest.go
View File

@ -1,16 +1,23 @@
package manifest
import (
"github.com/docker/distribution/registry/storage"
storagedriver "github.com/docker/distribution/registry/storage/driver"
"net/http"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/registryctl/api"
"github.com/docker/distribution/registry/storage"
storagedriver "github.com/docker/distribution/registry/storage/driver"
tracelib "github.com/goharbor/harbor/src/lib/trace"
"github.com/gorilla/mux"
"github.com/opencontainers/go-digest"
"net/http"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
const tracerName = "goharbor/harbor/src/registryctl/api/registry/manifest"
// NewHandler returns the handler to handler manifest request
func NewHandler(storageDriver storagedriver.StorageDriver) http.Handler {
return &handler{
@ -34,25 +41,37 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// delete deletes manifest ...
func (h *handler) delete(w http.ResponseWriter, r *http.Request) {
var span trace.Span
ctx := r.Context()
ref := mux.Vars(r)["reference"]
if tracelib.Enabled() {
ctx, span = tracelib.StartTrace(ctx, tracerName, "delete-manifest", trace.WithAttributes(attribute.Key("method").String(r.Method)))
defer span.End()
}
if ref == "" {
api.HandleBadRequest(w, errors.New("no reference specified"))
err := errors.New("no reference specified")
tracelib.RecordError(span, err, "no reference specified ")
api.HandleBadRequest(w, err)
return
}
dgst, err := digest.Parse(ref)
if err != nil {
tracelib.RecordError(span, err, "invalid reference")
api.HandleBadRequest(w, errors.Wrap(err, "not supported reference"))
return
}
repoName := mux.Vars(r)["name"]
if repoName == "" {
api.HandleBadRequest(w, errors.New("no repository name specified"))
err := errors.New("no repository name specified")
tracelib.RecordError(span, err, "no repository name specified")
api.HandleBadRequest(w, err)
return
}
// let the tags as empty here, as it non-blocking GC. The tags deletion will be handled via DELETE /v2/manifest
var tags []string
cleaner := storage.NewVacuum(r.Context(), h.storageDriver)
cleaner := storage.NewVacuum(ctx, h.storageDriver)
if err := cleaner.RemoveManifest(repoName, dgst, tags); err != nil {
tracelib.RecordError(span, err, "failed to remove manifest")
log.Infof("failed to remove manifest: %s, with error:%v", ref, err)
api.HandleError(w, err)
return

6
src/registryctl/client/client.go
View File

@ -16,7 +16,6 @@ package client
import (
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
"io/ioutil"
"net/http"
"strings"
@ -24,6 +23,7 @@ import (
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/lib/errors"
)
// const definition
@ -62,9 +62,7 @@ func NewClient(baseURL string, cfg *Config) Client {
}
if cfg != nil {
authorizer := auth.NewSecretAuthorizer(cfg.Secret)
client.client = common_http.NewClient(&http.Client{
Transport: common_http.GetHTTPTransport(common_http.SecureTransport),
}, authorizer)
client.client = common_http.NewClient(nil, authorizer)
}
return client
}

7
src/registryctl/handlers/handler.go
View File

@ -18,10 +18,12 @@ import (
"net/http"
"os"
gorilla_handlers "github.com/gorilla/handlers"
"github.com/goharbor/harbor/src/lib/log"
tracelib "github.com/goharbor/harbor/src/lib/trace"
"github.com/goharbor/harbor/src/registryctl/auth"
"github.com/goharbor/harbor/src/registryctl/config"
gorilla_handlers "github.com/gorilla/handlers"
)
// NewHandlerChain returns a gorilla router which is wrapped by authenticate handler
@ -36,6 +38,9 @@ func NewHandlerChain(conf config.Configuration) http.Handler {
}
h = newAuthHandler(auth.NewSecretHandler(secrets), h, insecureAPIs)
h = gorilla_handlers.LoggingHandler(os.Stdout, h)
if tracelib.Enabled() {
h = tracelib.NewHandler(h, "serve-http")
}
return h
}

39
src/registryctl/main.go
View File

@ -15,14 +15,14 @@
package main
import (
"context"
"crypto/tls"
"flag"
"net/http"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/registryctl/config"
"github.com/goharbor/harbor/src/registryctl/handlers"
"os"
"os/signal"
"syscall"
"time"
_ "github.com/docker/distribution/registry/storage/driver/azure"
_ "github.com/docker/distribution/registry/storage/driver/filesystem"
@ -33,6 +33,14 @@ import (
_ "github.com/docker/distribution/registry/storage/driver/oss"
_ "github.com/docker/distribution/registry/storage/driver/s3-aws"
_ "github.com/docker/distribution/registry/storage/driver/swift"
common_http "github.com/goharbor/harbor/src/common/http"
cfgLib "github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/log"
tracelib "github.com/goharbor/harbor/src/lib/trace"
_ "github.com/goharbor/harbor/src/pkg/config/inmemory"
"github.com/goharbor/harbor/src/registryctl/config"
"github.com/goharbor/harbor/src/registryctl/handlers"
)
// RegistryCtl for registry controller
@ -48,6 +56,21 @@ func (s *RegistryCtl) Start() {
Handler: s.Handler,
TLSConfig: common_http.NewServerTLSConfig(),
}
ctx := context.Background()
regCtl.RegisterOnShutdown(tracelib.InitGlobalTracer(ctx))
// graceful shutdown
go func() {
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
<-sig
context, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
log.Infof("Got an interrupt, shutting down...")
if err := regCtl.Shutdown(context); err != nil {
log.Fatalf("Failed to shutdown registry controller: %v", err)
}
log.Infof("Registry controller is shut down properly")
}()
var err error
if s.ServerConf.Protocol == "https" {
@ -58,12 +81,10 @@ func (s *RegistryCtl) Start() {
} else {
err = regCtl.ListenAndServe()
}
<-ctx.Done()
if err != nil {
log.Fatal(err)
}
return
}
func main() {
@ -78,6 +99,8 @@ func main() {
log.Fatalf("Failed to load configurations with error: %s\n", err)
}
cfgLib.InitTraceConfig(context.Background())
regCtl := &RegistryCtl{
ServerConf: *config.DefaultConfig,
Handler: handlers.NewHandlerChain(*config.DefaultConfig),

2
src/server/middleware/blob/put_blob_upload.go
View File

@ -78,7 +78,7 @@ func PutBlobUploadMiddleware() func(http.Handler) http.Handler {
return nil
}
return orm.WithTransaction(h)(ctx)
return orm.WithTransaction(h)(orm.SetTransactionOpNameToContext(ctx, "tx-put-blob-mw"))
})
return middleware.Chain(before, after)

8
src/server/middleware/requestid/requestid.go
View File

@ -17,7 +17,11 @@ package requestid
import (
"net/http"
tracelib "github.com/goharbor/harbor/src/lib/trace"
"github.com/goharbor/harbor/src/server/middleware"
"go.opentelemetry.io/otel/attribute"
oteltrace "go.opentelemetry.io/otel/trace"
"github.com/google/uuid"
)
@ -32,8 +36,10 @@ func Middleware(skippers ...middleware.Skipper) func(http.Handler) http.Handler
rid = uuid.New().String()
r.Header.Set(HeaderXRequestID, rid)
}
w.Header().Set(HeaderXRequestID, rid)
if tracelib.Enabled() {
oteltrace.SpanFromContext(r.Context()).SetAttributes(attribute.Key(HeaderXRequestID).String(rid))
}
next.ServeHTTP(w, r)
}, skippers...)
}

35
src/server/middleware/trace/trace.go Normal file
View File

@ -0,0 +1,35 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package trace
import (
"net/http"
tracelib "github.com/goharbor/harbor/src/lib/trace"
)
func traceHandler(next http.Handler) http.Handler {
if tracelib.Enabled() {
return tracelib.NewHandler(next, "handle-http-request")
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r)
})
}
// Middleware returns a middleware for handling requests
func Middleware() func(http.Handler) http.Handler {
return traceHandler
}

5
src/server/registry/proxy.go
View File

@ -16,11 +16,12 @@ package registry
import (
"fmt"
"github.com/goharbor/harbor/src/lib/config"
"net/http"
"net/http/httputil"
"net/url"
"github.com/goharbor/harbor/src/lib/config"
commonhttp "github.com/goharbor/harbor/src/common/http"
)
@ -34,7 +35,7 @@ func newProxy() http.Handler {
}
proxy := httputil.NewSingleHostReverseProxy(url)
if commonhttp.InternalTLSEnabled() {
proxy.Transport = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
proxy.Transport = commonhttp.GetHTTPTransport()
}
proxy.Director = basicAuthDirector(proxy.Director)

6
src/vendor/github.com/cenkalti/backoff/v4/context.go generated vendored
View File

@ -57,10 +57,6 @@ func (b *backOffContext) NextBackOff() time.Duration {
case <-b.ctx.Done():
return Stop
default:
return b.BackOff.NextBackOff()
}
next := b.BackOff.NextBackOff()
if deadline, ok := b.ctx.Deadline(); ok && deadline.Sub(time.Now()) < next { // nolint: gosimple
return Stop
}
return next
}

4
src/vendor/github.com/cenkalti/backoff/v4/retry.go generated vendored
View File

@ -62,6 +62,10 @@ func RetryNotifyWithTimer(operation Operation, b BackOff, notify Notify, t Timer
}
if next = b.NextBackOff(); next == Stop {
if cerr := ctx.Err(); cerr != nil {
return cerr
}
return err
}

0
src/vendor/github.com/felixge/httpsnoop/.gitignore generated vendored Normal file
View File

6
src/vendor/github.com/felixge/httpsnoop/.travis.yml generated vendored Normal file
View File

@ -0,0 +1,6 @@
language: go
go:
- 1.6
- 1.7
- 1.8

19
src/vendor/github.com/felixge/httpsnoop/LICENSE.txt generated vendored Normal file
View File

@ -0,0 +1,19 @@
Copyright (c) 2016 Felix Geisendörfer (felix@debuggable.com)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

10
src/vendor/github.com/felixge/httpsnoop/Makefile generated vendored Normal file
View File

@ -0,0 +1,10 @@
.PHONY: ci generate clean
ci: clean generate
go test -v ./...
generate:
go generate .
clean:
rm -rf *_generated*.go

95
src/vendor/github.com/felixge/httpsnoop/README.md generated vendored Normal file
View File

@ -0,0 +1,95 @@
# httpsnoop
Package httpsnoop provides an easy way to capture http related metrics (i.e.
response time, bytes written, and http status code) from your application's
http.Handlers.
Doing this requires non-trivial wrapping of the http.ResponseWriter interface,
which is also exposed for users interested in a more low-level API.
[![GoDoc](https://godoc.org/github.com/felixge/httpsnoop?status.svg)](https://godoc.org/github.com/felixge/httpsnoop)
[![Build Status](https://travis-ci.org/felixge/httpsnoop.svg?branch=master)](https://travis-ci.org/felixge/httpsnoop)
## Usage Example
```go
// myH is your app's http handler, perhaps a http.ServeMux or similar.
var myH http.Handler
// wrappedH wraps myH in order to log every request.
wrappedH := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
m := httpsnoop.CaptureMetrics(myH, w, r)
log.Printf(
"%s %s (code=%d dt=%s written=%d)",
r.Method,
r.URL,
m.Code,
m.Duration,
m.Written,
)
})
http.ListenAndServe(":8080", wrappedH)
```
## Why this package exists
Instrumenting an application's http.Handler is surprisingly difficult.
However if you google for e.g. "capture ResponseWriter status code" you'll find
lots of advise and code examples that suggest it to be a fairly trivial
undertaking. Unfortunately everything I've seen so far has a high chance of
breaking your application.
The main problem is that a `http.ResponseWriter` often implements additional
interfaces such as `http.Flusher`, `http.CloseNotifier`, `http.Hijacker`, `http.Pusher`, and
`io.ReaderFrom`. So the naive approach of just wrapping `http.ResponseWriter`
in your own struct that also implements the `http.ResponseWriter` interface
will hide the additional interfaces mentioned above. This has a high change of
introducing subtle bugs into any non-trivial application.
Another approach I've seen people take is to return a struct that implements
all of the interfaces above. However, that's also problematic, because it's
difficult to fake some of these interfaces behaviors when the underlying
`http.ResponseWriter` doesn't have an implementation. It's also dangerous,
because an application may choose to operate differently, merely because it
detects the presence of these additional interfaces.
This package solves this problem by checking which additional interfaces a
`http.ResponseWriter` implements, returning a wrapped version implementing the
exact same set of interfaces.
Additionally this package properly handles edge cases such as `WriteHeader` not
being called, or called more than once, as well as concurrent calls to
`http.ResponseWriter` methods, and even calls happening after the wrapped
`ServeHTTP` has already returned.
Unfortunately this package is not perfect either. It's possible that it is
still missing some interfaces provided by the go core (let me know if you find
one), and it won't work for applications adding their own interfaces into the
mix. You can however use `httpsnoop.Unwrap(w)` to access the underlying
`http.ResponseWriter` and type-assert the result to its other interfaces.
However, hopefully the explanation above has sufficiently scared you of rolling
your own solution to this problem. httpsnoop may still break your application,
but at least it tries to avoid it as much as possible.
Anyway, the real problem here is that smuggling additional interfaces inside
`http.ResponseWriter` is a problematic design choice, but it probably goes as
deep as the Go language specification itself. But that's okay, I still prefer
Go over the alternatives ;).
## Performance
```
BenchmarkBaseline-8 20000 94912 ns/op
BenchmarkCaptureMetrics-8 20000 95461 ns/op
```
As you can see, using `CaptureMetrics` on a vanilla http.Handler introduces an
overhead of ~500 ns per http request on my machine. However, the margin of
error appears to be larger than that, therefor it should be reasonable to
assume that the overhead introduced by `CaptureMetrics` is absolutely
negligible.
## License
MIT

79
src/vendor/github.com/felixge/httpsnoop/capture_metrics.go generated vendored Normal file
View File

@ -0,0 +1,79 @@
package httpsnoop
import (
"io"
"net/http"
"time"
)
// Metrics holds metrics captured from CaptureMetrics.
type Metrics struct {
// Code is the first http response code passed to the WriteHeader func of
// the ResponseWriter. If no such call is made, a default code of 200 is
// assumed instead.
Code int
// Duration is the time it took to execute the handler.
Duration time.Duration
// Written is the number of bytes successfully written by the Write or
// ReadFrom function of the ResponseWriter. ResponseWriters may also write
// data to their underlaying connection directly (e.g. headers), but those
// are not tracked. Therefor the number of Written bytes will usually match
// the size of the response body.
Written int64
}
// CaptureMetrics wraps the given hnd, executes it with the given w and r, and
// returns the metrics it captured from it.
func CaptureMetrics(hnd http.Handler, w http.ResponseWriter, r *http.Request) Metrics {
return CaptureMetricsFn(w, func(ww http.ResponseWriter) {
hnd.ServeHTTP(ww, r)
})
}
// CaptureMetricsFn wraps w and calls fn with the wrapped w and returns the
// resulting metrics. This is very similar to CaptureMetrics (which is just
// sugar on top of this func), but is a more usable interface if your
// application doesn't use the Go http.Handler interface.
func CaptureMetricsFn(w http.ResponseWriter, fn func(http.ResponseWriter)) Metrics {
var (
start = time.Now()
m = Metrics{Code: http.StatusOK}
headerWritten bool
hooks = Hooks{
WriteHeader: func(next WriteHeaderFunc) WriteHeaderFunc {
return func(code int) {
next(code)
if !headerWritten {
m.Code = code
headerWritten = true
}
}
},
Write: func(next WriteFunc) WriteFunc {
return func(p []byte) (int, error) {
n, err := next(p)
m.Written += int64(n)
headerWritten = true
return n, err
}
},
ReadFrom: func(next ReadFromFunc) ReadFromFunc {
return func(src io.Reader) (int64, error) {
n, err := next(src)
headerWritten = true
m.Written += n
return n, err
}
},
}
)
fn(Wrap(w, hooks))
m.Duration = time.Since(start)
return m
}

10
src/vendor/github.com/felixge/httpsnoop/docs.go generated vendored Normal file
View File

@ -0,0 +1,10 @@
// Package httpsnoop provides an easy way to capture http related metrics (i.e.
// response time, bytes written, and http status code) from your application's
// http.Handlers.
//
// Doing this requires non-trivial wrapping of the http.ResponseWriter
// interface, which is also exposed for users interested in a more low-level
// API.
package httpsnoop
//go:generate go run codegen/main.go

3
src/vendor/github.com/felixge/httpsnoop/go.mod generated vendored Normal file
View File

@ -0,0 +1,3 @@
module github.com/felixge/httpsnoop
go 1.13

436
src/vendor/github.com/felixge/httpsnoop/wrap_generated_gteq_1.8.go generated vendored Normal file
View File

@ -0,0 +1,436 @@
// +build go1.8
// Code generated by "httpsnoop/codegen"; DO NOT EDIT
package httpsnoop
import (
"bufio"
"io"
"net"
"net/http"
)
// HeaderFunc is part of the http.ResponseWriter interface.
type HeaderFunc func() http.Header
// WriteHeaderFunc is part of the http.ResponseWriter interface.
type WriteHeaderFunc func(code int)
// WriteFunc is part of the http.ResponseWriter interface.
type WriteFunc func(b []byte) (int, error)
// FlushFunc is part of the http.Flusher interface.
type FlushFunc func()
// CloseNotifyFunc is part of the http.CloseNotifier interface.
type CloseNotifyFunc func() <-chan bool
// HijackFunc is part of the http.Hijacker interface.
type HijackFunc func() (net.Conn, *bufio.ReadWriter, error)
// ReadFromFunc is part of the io.ReaderFrom interface.
type ReadFromFunc func(src io.Reader) (int64, error)
// PushFunc is part of the http.Pusher interface.
type PushFunc func(target string, opts *http.PushOptions) error
// Hooks defines a set of method interceptors for methods included in
// http.ResponseWriter as well as some others. You can think of them as
// middleware for the function calls they target. See Wrap for more details.
type Hooks struct {
Header func(HeaderFunc) HeaderFunc
WriteHeader func(WriteHeaderFunc) WriteHeaderFunc
Write func(WriteFunc) WriteFunc
Flush func(FlushFunc) FlushFunc
CloseNotify func(CloseNotifyFunc) CloseNotifyFunc
Hijack func(HijackFunc) HijackFunc
ReadFrom func(ReadFromFunc) ReadFromFunc
Push func(PushFunc) PushFunc
}
// Wrap returns a wrapped version of w that provides the exact same interface
// as w. Specifically if w implements any combination of:
//
// - http.Flusher
// - http.CloseNotifier
// - http.Hijacker
// - io.ReaderFrom
// - http.Pusher
//
// The wrapped version will implement the exact same combination. If no hooks
// are set, the wrapped version also behaves exactly as w. Hooks targeting
// methods not supported by w are ignored. Any other hooks will intercept the
// method they target and may modify the call's arguments and/or return values.
// The CaptureMetrics implementation serves as a working example for how the
// hooks can be used.
func Wrap(w http.ResponseWriter, hooks Hooks) http.ResponseWriter {
rw := &rw{w: w, h: hooks}
_, i0 := w.(http.Flusher)
_, i1 := w.(http.CloseNotifier)
_, i2 := w.(http.Hijacker)
_, i3 := w.(io.ReaderFrom)
_, i4 := w.(http.Pusher)
switch {
// combination 1/32
case !i0 && !i1 && !i2 && !i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
}{rw, rw}
// combination 2/32
case !i0 && !i1 && !i2 && !i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Pusher
}{rw, rw, rw}
// combination 3/32
case !i0 && !i1 && !i2 && i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
io.ReaderFrom
}{rw, rw, rw}
// combination 4/32
case !i0 && !i1 && !i2 && i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
io.ReaderFrom
http.Pusher
}{rw, rw, rw, rw}
// combination 5/32
case !i0 && !i1 && i2 && !i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.Hijacker
}{rw, rw, rw}
// combination 6/32
case !i0 && !i1 && i2 && !i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Hijacker
http.Pusher
}{rw, rw, rw, rw}
// combination 7/32
case !i0 && !i1 && i2 && i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.Hijacker
io.ReaderFrom
}{rw, rw, rw, rw}
// combination 8/32
case !i0 && !i1 && i2 && i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Hijacker
io.ReaderFrom
http.Pusher
}{rw, rw, rw, rw, rw}
// combination 9/32
case !i0 && i1 && !i2 && !i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
}{rw, rw, rw}
// combination 10/32
case !i0 && i1 && !i2 && !i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
http.Pusher
}{rw, rw, rw, rw}
// combination 11/32
case !i0 && i1 && !i2 && i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
io.ReaderFrom
}{rw, rw, rw, rw}
// combination 12/32
case !i0 && i1 && !i2 && i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
io.ReaderFrom
http.Pusher
}{rw, rw, rw, rw, rw}
// combination 13/32
case !i0 && i1 && i2 && !i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
http.Hijacker
}{rw, rw, rw, rw}
// combination 14/32
case !i0 && i1 && i2 && !i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
http.Hijacker
http.Pusher
}{rw, rw, rw, rw, rw}
// combination 15/32
case !i0 && i1 && i2 && i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
http.Hijacker
io.ReaderFrom
}{rw, rw, rw, rw, rw}
// combination 16/32
case !i0 && i1 && i2 && i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
http.Hijacker
io.ReaderFrom
http.Pusher
}{rw, rw, rw, rw, rw, rw}
// combination 17/32
case i0 && !i1 && !i2 && !i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
}{rw, rw, rw}
// combination 18/32
case i0 && !i1 && !i2 && !i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.Pusher
}{rw, rw, rw, rw}
// combination 19/32
case i0 && !i1 && !i2 && i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
io.ReaderFrom
}{rw, rw, rw, rw}
// combination 20/32
case i0 && !i1 && !i2 && i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
io.ReaderFrom
http.Pusher
}{rw, rw, rw, rw, rw}
// combination 21/32
case i0 && !i1 && i2 && !i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.Hijacker
}{rw, rw, rw, rw}
// combination 22/32
case i0 && !i1 && i2 && !i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.Hijacker
http.Pusher
}{rw, rw, rw, rw, rw}
// combination 23/32
case i0 && !i1 && i2 && i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.Hijacker
io.ReaderFrom
}{rw, rw, rw, rw, rw}
// combination 24/32
case i0 && !i1 && i2 && i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.Hijacker
io.ReaderFrom
http.Pusher
}{rw, rw, rw, rw, rw, rw}
// combination 25/32
case i0 && i1 && !i2 && !i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
}{rw, rw, rw, rw}
// combination 26/32
case i0 && i1 && !i2 && !i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
http.Pusher
}{rw, rw, rw, rw, rw}
// combination 27/32
case i0 && i1 && !i2 && i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
io.ReaderFrom
}{rw, rw, rw, rw, rw}
// combination 28/32
case i0 && i1 && !i2 && i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
io.ReaderFrom
http.Pusher
}{rw, rw, rw, rw, rw, rw}
// combination 29/32
case i0 && i1 && i2 && !i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
http.Hijacker
}{rw, rw, rw, rw, rw}
// combination 30/32
case i0 && i1 && i2 && !i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
http.Hijacker
http.Pusher
}{rw, rw, rw, rw, rw, rw}
// combination 31/32
case i0 && i1 && i2 && i3 && !i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
http.Hijacker
io.ReaderFrom
}{rw, rw, rw, rw, rw, rw}
// combination 32/32
case i0 && i1 && i2 && i3 && i4:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
http.Hijacker
io.ReaderFrom
http.Pusher
}{rw, rw, rw, rw, rw, rw, rw}
}
panic("unreachable")
}
type rw struct {
w http.ResponseWriter
h Hooks
}
func (w *rw) Unwrap() http.ResponseWriter {
return w.w
}
func (w *rw) Header() http.Header {
f := w.w.(http.ResponseWriter).Header
if w.h.Header != nil {
f = w.h.Header(f)
}
return f()
}
func (w *rw) WriteHeader(code int) {
f := w.w.(http.ResponseWriter).WriteHeader
if w.h.WriteHeader != nil {
f = w.h.WriteHeader(f)
}
f(code)
}
func (w *rw) Write(b []byte) (int, error) {
f := w.w.(http.ResponseWriter).Write
if w.h.Write != nil {
f = w.h.Write(f)
}
return f(b)
}
func (w *rw) Flush() {
f := w.w.(http.Flusher).Flush
if w.h.Flush != nil {
f = w.h.Flush(f)
}
f()
}
func (w *rw) CloseNotify() <-chan bool {
f := w.w.(http.CloseNotifier).CloseNotify
if w.h.CloseNotify != nil {
f = w.h.CloseNotify(f)
}
return f()
}
func (w *rw) Hijack() (net.Conn, *bufio.ReadWriter, error) {
f := w.w.(http.Hijacker).Hijack
if w.h.Hijack != nil {
f = w.h.Hijack(f)
}
return f()
}
func (w *rw) ReadFrom(src io.Reader) (int64, error) {
f := w.w.(io.ReaderFrom).ReadFrom
if w.h.ReadFrom != nil {
f = w.h.ReadFrom(f)
}
return f(src)
}
func (w *rw) Push(target string, opts *http.PushOptions) error {
f := w.w.(http.Pusher).Push
if w.h.Push != nil {
f = w.h.Push(f)
}
return f(target, opts)
}
type Unwrapper interface {
Unwrap() http.ResponseWriter
}
// Unwrap returns the underlying http.ResponseWriter from within zero or more
// layers of httpsnoop wrappers.
func Unwrap(w http.ResponseWriter) http.ResponseWriter {
if rw, ok := w.(Unwrapper); ok {
// recurse until rw.Unwrap() returns a non-Unwrapper
return Unwrap(rw.Unwrap())
} else {
return w
}
}

278
src/vendor/github.com/felixge/httpsnoop/wrap_generated_lt_1.8.go generated vendored Normal file
View File

@ -0,0 +1,278 @@
// +build !go1.8
// Code generated by "httpsnoop/codegen"; DO NOT EDIT
package httpsnoop
import (
"bufio"
"io"
"net"
"net/http"
)
// HeaderFunc is part of the http.ResponseWriter interface.
type HeaderFunc func() http.Header
// WriteHeaderFunc is part of the http.ResponseWriter interface.
type WriteHeaderFunc func(code int)
// WriteFunc is part of the http.ResponseWriter interface.
type WriteFunc func(b []byte) (int, error)
// FlushFunc is part of the http.Flusher interface.
type FlushFunc func()
// CloseNotifyFunc is part of the http.CloseNotifier interface.
type CloseNotifyFunc func() <-chan bool
// HijackFunc is part of the http.Hijacker interface.
type HijackFunc func() (net.Conn, *bufio.ReadWriter, error)
// ReadFromFunc is part of the io.ReaderFrom interface.
type ReadFromFunc func(src io.Reader) (int64, error)
// Hooks defines a set of method interceptors for methods included in
// http.ResponseWriter as well as some others. You can think of them as
// middleware for the function calls they target. See Wrap for more details.
type Hooks struct {
Header func(HeaderFunc) HeaderFunc
WriteHeader func(WriteHeaderFunc) WriteHeaderFunc
Write func(WriteFunc) WriteFunc
Flush func(FlushFunc) FlushFunc
CloseNotify func(CloseNotifyFunc) CloseNotifyFunc
Hijack func(HijackFunc) HijackFunc
ReadFrom func(ReadFromFunc) ReadFromFunc
}
// Wrap returns a wrapped version of w that provides the exact same interface
// as w. Specifically if w implements any combination of:
//
// - http.Flusher
// - http.CloseNotifier
// - http.Hijacker
// - io.ReaderFrom
//
// The wrapped version will implement the exact same combination. If no hooks
// are set, the wrapped version also behaves exactly as w. Hooks targeting
// methods not supported by w are ignored. Any other hooks will intercept the
// method they target and may modify the call's arguments and/or return values.
// The CaptureMetrics implementation serves as a working example for how the
// hooks can be used.
func Wrap(w http.ResponseWriter, hooks Hooks) http.ResponseWriter {
rw := &rw{w: w, h: hooks}
_, i0 := w.(http.Flusher)
_, i1 := w.(http.CloseNotifier)
_, i2 := w.(http.Hijacker)
_, i3 := w.(io.ReaderFrom)
switch {
// combination 1/16
case !i0 && !i1 && !i2 && !i3:
return struct {
Unwrapper
http.ResponseWriter
}{rw, rw}
// combination 2/16
case !i0 && !i1 && !i2 && i3:
return struct {
Unwrapper
http.ResponseWriter
io.ReaderFrom
}{rw, rw, rw}
// combination 3/16
case !i0 && !i1 && i2 && !i3:
return struct {
Unwrapper
http.ResponseWriter
http.Hijacker
}{rw, rw, rw}
// combination 4/16
case !i0 && !i1 && i2 && i3:
return struct {
Unwrapper
http.ResponseWriter
http.Hijacker
io.ReaderFrom
}{rw, rw, rw, rw}
// combination 5/16
case !i0 && i1 && !i2 && !i3:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
}{rw, rw, rw}
// combination 6/16
case !i0 && i1 && !i2 && i3:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
io.ReaderFrom
}{rw, rw, rw, rw}
// combination 7/16
case !i0 && i1 && i2 && !i3:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
http.Hijacker
}{rw, rw, rw, rw}
// combination 8/16
case !i0 && i1 && i2 && i3:
return struct {
Unwrapper
http.ResponseWriter
http.CloseNotifier
http.Hijacker
io.ReaderFrom
}{rw, rw, rw, rw, rw}
// combination 9/16
case i0 && !i1 && !i2 && !i3:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
}{rw, rw, rw}
// combination 10/16
case i0 && !i1 && !i2 && i3:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
io.ReaderFrom
}{rw, rw, rw, rw}
// combination 11/16
case i0 && !i1 && i2 && !i3:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.Hijacker
}{rw, rw, rw, rw}
// combination 12/16
case i0 && !i1 && i2 && i3:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.Hijacker
io.ReaderFrom
}{rw, rw, rw, rw, rw}
// combination 13/16
case i0 && i1 && !i2 && !i3:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
}{rw, rw, rw, rw}
// combination 14/16
case i0 && i1 && !i2 && i3:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
io.ReaderFrom
}{rw, rw, rw, rw, rw}
// combination 15/16
case i0 && i1 && i2 && !i3:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
http.Hijacker
}{rw, rw, rw, rw, rw}
// combination 16/16
case i0 && i1 && i2 && i3:
return struct {
Unwrapper
http.ResponseWriter
http.Flusher
http.CloseNotifier
http.Hijacker
io.ReaderFrom
}{rw, rw, rw, rw, rw, rw}
}
panic("unreachable")
}
type rw struct {
w http.ResponseWriter
h Hooks
}
func (w *rw) Unwrap() http.ResponseWriter {
return w.w
}
func (w *rw) Header() http.Header {
f := w.w.(http.ResponseWriter).Header
if w.h.Header != nil {
f = w.h.Header(f)
}
return f()
}
func (w *rw) WriteHeader(code int) {
f := w.w.(http.ResponseWriter).WriteHeader
if w.h.WriteHeader != nil {
f = w.h.WriteHeader(f)
}
f(code)
}
func (w *rw) Write(b []byte) (int, error) {
f := w.w.(http.ResponseWriter).Write
if w.h.Write != nil {
f = w.h.Write(f)
}
return f(b)
}
func (w *rw) Flush() {
f := w.w.(http.Flusher).Flush
if w.h.Flush != nil {
f = w.h.Flush(f)
}
f()
}
func (w *rw) CloseNotify() <-chan bool {
f := w.w.(http.CloseNotifier).CloseNotify
if w.h.CloseNotify != nil {
f = w.h.CloseNotify(f)
}
return f()
}
func (w *rw) Hijack() (net.Conn, *bufio.ReadWriter, error) {
f := w.w.(http.Hijacker).Hijack
if w.h.Hijack != nil {
f = w.h.Hijack(f)
}
return f()
}
func (w *rw) ReadFrom(src io.Reader) (int64, error) {
f := w.w.(io.ReaderFrom).ReadFrom
if w.h.ReadFrom != nil {
f = w.h.ReadFrom(f)
}
return f(src)
}
type Unwrapper interface {
Unwrap() http.ResponseWriter
}
// Unwrap returns the underlying http.ResponseWriter from within zero or more
// layers of httpsnoop wrappers.
func Unwrap(w http.ResponseWriter) http.ResponseWriter {
if rw, ok := w.(Unwrapper); ok {
// recurse until rw.Unwrap() returns a non-Unwrapper
return Unwrap(rw.Unwrap())
} else {
return w
}
}

180
src/vendor/github.com/golang/protobuf/descriptor/descriptor.go generated vendored Normal file
View File

@ -0,0 +1,180 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package descriptor provides functions for obtaining the protocol buffer
// descriptors of generated Go types.
//
// Deprecated: See the "google.golang.org/protobuf/reflect/protoreflect" package
// for how to obtain an EnumDescriptor or MessageDescriptor in order to
// programatically interact with the protobuf type system.
package descriptor
import (
"bytes"
"compress/gzip"
"io/ioutil"
"sync"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/reflect/protodesc"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/runtime/protoimpl"
descriptorpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
)
// Message is proto.Message with a method to return its descriptor.
//
// Deprecated: The Descriptor method may not be generated by future
// versions of protoc-gen-go, meaning that this interface may not
// be implemented by many concrete message types.
type Message interface {
proto.Message
Descriptor() ([]byte, []int)
}
// ForMessage returns the file descriptor proto containing
// the message and the message descriptor proto for the message itself.
// The returned proto messages must not be mutated.
//
// Deprecated: Not all concrete message types satisfy the Message interface.
// Use MessageDescriptorProto instead. If possible, the calling code should
// be rewritten to use protobuf reflection instead.
// See package "google.golang.org/protobuf/reflect/protoreflect" for details.
func ForMessage(m Message) (*descriptorpb.FileDescriptorProto, *descriptorpb.DescriptorProto) {
return MessageDescriptorProto(m)
}
type rawDesc struct {
fileDesc []byte
indexes []int
}
var rawDescCache sync.Map // map[protoreflect.Descriptor]*rawDesc
func deriveRawDescriptor(d protoreflect.Descriptor) ([]byte, []int) {
// Fast-path: check whether raw descriptors are already cached.
origDesc := d
if v, ok := rawDescCache.Load(origDesc); ok {
return v.(*rawDesc).fileDesc, v.(*rawDesc).indexes
}
// Slow-path: derive the raw descriptor from the v2 descriptor.
// Start with the leaf (a given enum or message declaration) and
// ascend upwards until we hit the parent file descriptor.
var idxs []int
for {
idxs = append(idxs, d.Index())
d = d.Parent()
if d == nil {
// TODO: We could construct a FileDescriptor stub for standalone
// descriptors to satisfy the API.
return nil, nil
}
if _, ok := d.(protoreflect.FileDescriptor); ok {
break
}
}
// Obtain the raw file descriptor.
fd := d.(protoreflect.FileDescriptor)
b, _ := proto.Marshal(protodesc.ToFileDescriptorProto(fd))
file := protoimpl.X.CompressGZIP(b)
// Reverse the indexes, since we populated it in reverse.
for i, j := 0, len(idxs)-1; i < j; i, j = i+1, j-1 {
idxs[i], idxs[j] = idxs[j], idxs[i]
}
if v, ok := rawDescCache.LoadOrStore(origDesc, &rawDesc{file, idxs}); ok {
return v.(*rawDesc).fileDesc, v.(*rawDesc).indexes
}
return file, idxs
}
// EnumRawDescriptor returns the GZIP'd raw file descriptor representing
// the enum and the index path to reach the enum declaration.
// The returned slices must not be mutated.
func EnumRawDescriptor(e proto.GeneratedEnum) ([]byte, []int) {
if ev, ok := e.(interface{ EnumDescriptor() ([]byte, []int) }); ok {
return ev.EnumDescriptor()
}
ed := protoimpl.X.EnumTypeOf(e)
return deriveRawDescriptor(ed.Descriptor())
}
// MessageRawDescriptor returns the GZIP'd raw file descriptor representing
// the message and the index path to reach the message declaration.
// The returned slices must not be mutated.
func MessageRawDescriptor(m proto.GeneratedMessage) ([]byte, []int) {
if mv, ok := m.(interface{ Descriptor() ([]byte, []int) }); ok {
return mv.Descriptor()
}
md := protoimpl.X.MessageTypeOf(m)
return deriveRawDescriptor(md.Descriptor())
}
var fileDescCache sync.Map // map[*byte]*descriptorpb.FileDescriptorProto
func deriveFileDescriptor(rawDesc []byte) *descriptorpb.FileDescriptorProto {
// Fast-path: check whether descriptor protos are already cached.
if v, ok := fileDescCache.Load(&rawDesc[0]); ok {
return v.(*descriptorpb.FileDescriptorProto)
}
// Slow-path: derive the descriptor proto from the GZIP'd message.
zr, err := gzip.NewReader(bytes.NewReader(rawDesc))
if err != nil {
panic(err)
}
b, err := ioutil.ReadAll(zr)
if err != nil {
panic(err)
}
fd := new(descriptorpb.FileDescriptorProto)
if err := proto.Unmarshal(b, fd); err != nil {
panic(err)
}
if v, ok := fileDescCache.LoadOrStore(&rawDesc[0], fd); ok {
return v.(*descriptorpb.FileDescriptorProto)
}
return fd
}
// EnumDescriptorProto returns the file descriptor proto representing
// the enum and the enum descriptor proto for the enum itself.
// The returned proto messages must not be mutated.
func EnumDescriptorProto(e proto.GeneratedEnum) (*descriptorpb.FileDescriptorProto, *descriptorpb.EnumDescriptorProto) {
rawDesc, idxs := EnumRawDescriptor(e)
if rawDesc == nil || idxs == nil {
return nil, nil
}
fd := deriveFileDescriptor(rawDesc)
if len(idxs) == 1 {
return fd, fd.EnumType[idxs[0]]
}
md := fd.MessageType[idxs[0]]
for _, i := range idxs[1 : len(idxs)-1] {
md = md.NestedType[i]
}
ed := md.EnumType[idxs[len(idxs)-1]]
return fd, ed
}
// MessageDescriptorProto returns the file descriptor proto representing
// the message and the message descriptor proto for the message itself.
// The returned proto messages must not be mutated.
func MessageDescriptorProto(m proto.GeneratedMessage) (*descriptorpb.FileDescriptorProto, *descriptorpb.DescriptorProto) {
rawDesc, idxs := MessageRawDescriptor(m)
if rawDesc == nil || idxs == nil {
return nil, nil
}
fd := deriveFileDescriptor(rawDesc)
md := fd.MessageType[idxs[0]]
for _, i := range idxs[1:] {
md = md.NestedType[i]
}
return fd, md
}

524
src/vendor/github.com/golang/protobuf/jsonpb/decode.go generated vendored Normal file
View File

@ -0,0 +1,524 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsonpb
import (
"encoding/json"
"errors"
"fmt"
"io"
"math"
"reflect"
"strconv"
"strings"
"time"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/encoding/protojson"
protoV2 "google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
)
const wrapJSONUnmarshalV2 = false
// UnmarshalNext unmarshals the next JSON object from d into m.
func UnmarshalNext(d *json.Decoder, m proto.Message) error {
return new(Unmarshaler).UnmarshalNext(d, m)
}
// Unmarshal unmarshals a JSON object from r into m.
func Unmarshal(r io.Reader, m proto.Message) error {
return new(Unmarshaler).Unmarshal(r, m)
}
// UnmarshalString unmarshals a JSON object from s into m.
func UnmarshalString(s string, m proto.Message) error {
return new(Unmarshaler).Unmarshal(strings.NewReader(s), m)
}
// Unmarshaler is a configurable object for converting from a JSON
// representation to a protocol buffer object.
type Unmarshaler struct {
// AllowUnknownFields specifies whether to allow messages to contain
// unknown JSON fields, as opposed to failing to unmarshal.
AllowUnknownFields bool
// AnyResolver is used to resolve the google.protobuf.Any well-known type.
// If unset, the global registry is used by default.
AnyResolver AnyResolver
}
// JSONPBUnmarshaler is implemented by protobuf messages that customize the way
// they are unmarshaled from JSON. Messages that implement this should also
// implement JSONPBMarshaler so that the custom format can be produced.
//
// The JSON unmarshaling must follow the JSON to proto specification:
// https://developers.google.com/protocol-buffers/docs/proto3#json
//
// Deprecated: Custom types should implement protobuf reflection instead.
type JSONPBUnmarshaler interface {
UnmarshalJSONPB(*Unmarshaler, []byte) error
}
// Unmarshal unmarshals a JSON object from r into m.
func (u *Unmarshaler) Unmarshal(r io.Reader, m proto.Message) error {
return u.UnmarshalNext(json.NewDecoder(r), m)
}
// UnmarshalNext unmarshals the next JSON object from d into m.
func (u *Unmarshaler) UnmarshalNext(d *json.Decoder, m proto.Message) error {
if m == nil {
return errors.New("invalid nil message")
}
// Parse the next JSON object from the stream.
raw := json.RawMessage{}
if err := d.Decode(&raw); err != nil {
return err
}
// Check for custom unmarshalers first since they may not properly
// implement protobuf reflection that the logic below relies on.
if jsu, ok := m.(JSONPBUnmarshaler); ok {
return jsu.UnmarshalJSONPB(u, raw)
}
mr := proto.MessageReflect(m)
// NOTE: For historical reasons, a top-level null is treated as a noop.
// This is incorrect, but kept for compatibility.
if string(raw) == "null" && mr.Descriptor().FullName() != "google.protobuf.Value" {
return nil
}
if wrapJSONUnmarshalV2 {
// NOTE: If input message is non-empty, we need to preserve merge semantics
// of the old jsonpb implementation. These semantics are not supported by
// the protobuf JSON specification.
isEmpty := true
mr.Range(func(protoreflect.FieldDescriptor, protoreflect.Value) bool {
isEmpty = false // at least one iteration implies non-empty
return false
})
if !isEmpty {
// Perform unmarshaling into a newly allocated, empty message.
mr = mr.New()
// Use a defer to copy all unmarshaled fields into the original message.
dst := proto.MessageReflect(m)
defer mr.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
dst.Set(fd, v)
return true
})
}
// Unmarshal using the v2 JSON unmarshaler.
opts := protojson.UnmarshalOptions{
DiscardUnknown: u.AllowUnknownFields,
}
if u.AnyResolver != nil {
opts.Resolver = anyResolver{u.AnyResolver}
}
return opts.Unmarshal(raw, mr.Interface())
} else {
if err := u.unmarshalMessage(mr, raw); err != nil {
return err
}
return protoV2.CheckInitialized(mr.Interface())
}
}
func (u *Unmarshaler) unmarshalMessage(m protoreflect.Message, in []byte) error {
md := m.Descriptor()
fds := md.Fields()
if jsu, ok := proto.MessageV1(m.Interface()).(JSONPBUnmarshaler); ok {
return jsu.UnmarshalJSONPB(u, in)
}
if string(in) == "null" && md.FullName() != "google.protobuf.Value" {
return nil
}
switch wellKnownType(md.FullName()) {
case "Any":
var jsonObject map[string]json.RawMessage
if err := json.Unmarshal(in, &jsonObject); err != nil {
return err
}
rawTypeURL, ok := jsonObject["@type"]
if !ok {
return errors.New("Any JSON doesn't have '@type'")
}
typeURL, err := unquoteString(string(rawTypeURL))
if err != nil {
return fmt.Errorf("can't unmarshal Any's '@type': %q", rawTypeURL)
}
m.Set(fds.ByNumber(1), protoreflect.ValueOfString(typeURL))
var m2 protoreflect.Message
if u.AnyResolver != nil {
mi, err := u.AnyResolver.Resolve(typeURL)
if err != nil {
return err
}
m2 = proto.MessageReflect(mi)
} else {
mt, err := protoregistry.GlobalTypes.FindMessageByURL(typeURL)
if err != nil {
if err == protoregistry.NotFound {
return fmt.Errorf("could not resolve Any message type: %v", typeURL)
}
return err
}
m2 = mt.New()
}
if wellKnownType(m2.Descriptor().FullName()) != "" {
rawValue, ok := jsonObject["value"]
if !ok {
return errors.New("Any JSON doesn't have 'value'")
}
if err := u.unmarshalMessage(m2, rawValue); err != nil {
return fmt.Errorf("can't unmarshal Any nested proto %v: %v", typeURL, err)
}
} else {
delete(jsonObject, "@type")
rawJSON, err := json.Marshal(jsonObject)
if err != nil {
return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err)
}
if err = u.unmarshalMessage(m2, rawJSON); err != nil {
return fmt.Errorf("can't unmarshal Any nested proto %v: %v", typeURL, err)
}
}
rawWire, err := protoV2.Marshal(m2.Interface())
if err != nil {
return fmt.Errorf("can't marshal proto %v into Any.Value: %v", typeURL, err)
}
m.Set(fds.ByNumber(2), protoreflect.ValueOfBytes(rawWire))
return nil
case "BoolValue", "BytesValue", "StringValue",
"Int32Value", "UInt32Value", "FloatValue",
"Int64Value", "UInt64Value", "DoubleValue":
fd := fds.ByNumber(1)
v, err := u.unmarshalValue(m.NewField(fd), in, fd)
if err != nil {
return err
}
m.Set(fd, v)
return nil
case "Duration":
v, err := unquoteString(string(in))
if err != nil {
return err
}
d, err := time.ParseDuration(v)
if err != nil {
return fmt.Errorf("bad Duration: %v", err)
}
sec := d.Nanoseconds() / 1e9
nsec := d.Nanoseconds() % 1e9
m.Set(fds.ByNumber(1), protoreflect.ValueOfInt64(int64(sec)))
m.Set(fds.ByNumber(2), protoreflect.ValueOfInt32(int32(nsec)))
return nil
case "Timestamp":
v, err := unquoteString(string(in))
if err != nil {
return err
}
t, err := time.Parse(time.RFC3339Nano, v)
if err != nil {
return fmt.Errorf("bad Timestamp: %v", err)
}
sec := t.Unix()
nsec := t.Nanosecond()
m.Set(fds.ByNumber(1), protoreflect.ValueOfInt64(int64(sec)))
m.Set(fds.ByNumber(2), protoreflect.ValueOfInt32(int32(nsec)))
return nil
case "Value":
switch {
case string(in) == "null":
m.Set(fds.ByNumber(1), protoreflect.ValueOfEnum(0))
case string(in) == "true":
m.Set(fds.ByNumber(4), protoreflect.ValueOfBool(true))
case string(in) == "false":
m.Set(fds.ByNumber(4), protoreflect.ValueOfBool(false))
case hasPrefixAndSuffix('"', in, '"'):
s, err := unquoteString(string(in))
if err != nil {
return fmt.Errorf("unrecognized type for Value %q", in)
}
m.Set(fds.ByNumber(3), protoreflect.ValueOfString(s))
case hasPrefixAndSuffix('[', in, ']'):
v := m.Mutable(fds.ByNumber(6))
return u.unmarshalMessage(v.Message(), in)
case hasPrefixAndSuffix('{', in, '}'):
v := m.Mutable(fds.ByNumber(5))
return u.unmarshalMessage(v.Message(), in)
default:
f, err := strconv.ParseFloat(string(in), 0)
if err != nil {
return fmt.Errorf("unrecognized type for Value %q", in)
}
m.Set(fds.ByNumber(2), protoreflect.ValueOfFloat64(f))
}
return nil
case "ListValue":
var jsonArray []json.RawMessage
if err := json.Unmarshal(in, &jsonArray); err != nil {
return fmt.Errorf("bad ListValue: %v", err)
}
lv := m.Mutable(fds.ByNumber(1)).List()
for _, raw := range jsonArray {
ve := lv.NewElement()
if err := u.unmarshalMessage(ve.Message(), raw); err != nil {
return err
}
lv.Append(ve)
}
return nil
case "Struct":
var jsonObject map[string]json.RawMessage
if err := json.Unmarshal(in, &jsonObject); err != nil {
return fmt.Errorf("bad StructValue: %v", err)
}
mv := m.Mutable(fds.ByNumber(1)).Map()
for key, raw := range jsonObject {
kv := protoreflect.ValueOf(key).MapKey()
vv := mv.NewValue()
if err := u.unmarshalMessage(vv.Message(), raw); err != nil {
return fmt.Errorf("bad value in StructValue for key %q: %v", key, err)
}
mv.Set(kv, vv)
}
return nil
}
var jsonObject map[string]json.RawMessage
if err := json.Unmarshal(in, &jsonObject); err != nil {
return err
}
// Handle known fields.
for i := 0; i < fds.Len(); i++ {
fd := fds.Get(i)
if fd.IsWeak() && fd.Message().IsPlaceholder() {
continue // weak reference is not linked in
}
// Search for any raw JSON value associated with this field.
var raw json.RawMessage
name := string(fd.Name())
if fd.Kind() == protoreflect.GroupKind {
name = string(fd.Message().Name())
}
if v, ok := jsonObject[name]; ok {
delete(jsonObject, name)
raw = v
}
name = string(fd.JSONName())
if v, ok := jsonObject[name]; ok {
delete(jsonObject, name)
raw = v
}
field := m.NewField(fd)
// Unmarshal the field value.
if raw == nil || (string(raw) == "null" && !isSingularWellKnownValue(fd) && !isSingularJSONPBUnmarshaler(field, fd)) {
continue
}
v, err := u.unmarshalValue(field, raw, fd)
if err != nil {
return err
}
m.Set(fd, v)
}
// Handle extension fields.
for name, raw := range jsonObject {
if !strings.HasPrefix(name, "[") || !strings.HasSuffix(name, "]") {
continue
}
// Resolve the extension field by name.
xname := protoreflect.FullName(name[len("[") : len(name)-len("]")])
xt, _ := protoregistry.GlobalTypes.FindExtensionByName(xname)
if xt == nil && isMessageSet(md) {
xt, _ = protoregistry.GlobalTypes.FindExtensionByName(xname.Append("message_set_extension"))
}
if xt == nil {
continue
}
delete(jsonObject, name)
fd := xt.TypeDescriptor()
if fd.ContainingMessage().FullName() != m.Descriptor().FullName() {
return fmt.Errorf("extension field %q does not extend message %q", xname, m.Descriptor().FullName())
}
field := m.NewField(fd)
// Unmarshal the field value.
if raw == nil || (string(raw) == "null" && !isSingularWellKnownValue(fd) && !isSingularJSONPBUnmarshaler(field, fd)) {
continue
}
v, err := u.unmarshalValue(field, raw, fd)
if err != nil {
return err
}
m.Set(fd, v)
}
if !u.AllowUnknownFields && len(jsonObject) > 0 {
for name := range jsonObject {
return fmt.Errorf("unknown field %q in %v", name, md.FullName())
}
}
return nil
}
func isSingularWellKnownValue(fd protoreflect.FieldDescriptor) bool {
if md := fd.Message(); md != nil {
return md.FullName() == "google.protobuf.Value" && fd.Cardinality() != protoreflect.Repeated
}
return false
}
func isSingularJSONPBUnmarshaler(v protoreflect.Value, fd protoreflect.FieldDescriptor) bool {
if fd.Message() != nil && fd.Cardinality() != protoreflect.Repeated {
_, ok := proto.MessageV1(v.Interface()).(JSONPBUnmarshaler)
return ok
}
return false
}
func (u *Unmarshaler) unmarshalValue(v protoreflect.Value, in []byte, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
switch {
case fd.IsList():
var jsonArray []json.RawMessage
if err := json.Unmarshal(in, &jsonArray); err != nil {
return v, err
}
lv := v.List()
for _, raw := range jsonArray {
ve, err := u.unmarshalSingularValue(lv.NewElement(), raw, fd)
if err != nil {
return v, err
}
lv.Append(ve)
}
return v, nil
case fd.IsMap():
var jsonObject map[string]json.RawMessage
if err := json.Unmarshal(in, &jsonObject); err != nil {
return v, err
}
kfd := fd.MapKey()
vfd := fd.MapValue()
mv := v.Map()
for key, raw := range jsonObject {
var kv protoreflect.MapKey
if kfd.Kind() == protoreflect.StringKind {
kv = protoreflect.ValueOf(key).MapKey()
} else {
v, err := u.unmarshalSingularValue(kfd.Default(), []byte(key), kfd)
if err != nil {
return v, err
}
kv = v.MapKey()
}
vv, err := u.unmarshalSingularValue(mv.NewValue(), raw, vfd)
if err != nil {
return v, err
}
mv.Set(kv, vv)
}
return v, nil
default:
return u.unmarshalSingularValue(v, in, fd)
}
}
var nonFinite = map[string]float64{
`"NaN"`: math.NaN(),
`"Infinity"`: math.Inf(+1),
`"-Infinity"`: math.Inf(-1),
}
func (u *Unmarshaler) unmarshalSingularValue(v protoreflect.Value, in []byte, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
switch fd.Kind() {
case protoreflect.BoolKind:
return unmarshalValue(in, new(bool))
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
return unmarshalValue(trimQuote(in), new(int32))
case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
return unmarshalValue(trimQuote(in), new(int64))
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
return unmarshalValue(trimQuote(in), new(uint32))
case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
return unmarshalValue(trimQuote(in), new(uint64))
case protoreflect.FloatKind:
if f, ok := nonFinite[string(in)]; ok {
return protoreflect.ValueOfFloat32(float32(f)), nil
}
return unmarshalValue(trimQuote(in), new(float32))
case protoreflect.DoubleKind:
if f, ok := nonFinite[string(in)]; ok {
return protoreflect.ValueOfFloat64(float64(f)), nil
}
return unmarshalValue(trimQuote(in), new(float64))
case protoreflect.StringKind:
return unmarshalValue(in, new(string))
case protoreflect.BytesKind:
return unmarshalValue(in, new([]byte))
case protoreflect.EnumKind:
if hasPrefixAndSuffix('"', in, '"') {
vd := fd.Enum().Values().ByName(protoreflect.Name(trimQuote(in)))
if vd == nil {
return v, fmt.Errorf("unknown value %q for enum %s", in, fd.Enum().FullName())
}
return protoreflect.ValueOfEnum(vd.Number()), nil
}
return unmarshalValue(in, new(protoreflect.EnumNumber))
case protoreflect.MessageKind, protoreflect.GroupKind:
err := u.unmarshalMessage(v.Message(), in)
return v, err
default:
panic(fmt.Sprintf("invalid kind %v", fd.Kind()))
}
}
func unmarshalValue(in []byte, v interface{}) (protoreflect.Value, error) {
err := json.Unmarshal(in, v)
return protoreflect.ValueOf(reflect.ValueOf(v).Elem().Interface()), err
}
func unquoteString(in string) (out string, err error) {
err = json.Unmarshal([]byte(in), &out)
return out, err
}
func hasPrefixAndSuffix(prefix byte, in []byte, suffix byte) bool {
if len(in) >= 2 && in[0] == prefix && in[len(in)-1] == suffix {
return true
}
return false
}
// trimQuote is like unquoteString but simply strips surrounding quotes.
// This is incorrect, but is behavior done by the legacy implementation.
func trimQuote(in []byte) []byte {
if len(in) >= 2 && in[0] == '"' && in[len(in)-1] == '"' {
in = in[1 : len(in)-1]
}
return in
}

559
src/vendor/github.com/golang/protobuf/jsonpb/encode.go generated vendored Normal file
View File

@ -0,0 +1,559 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsonpb
import (
"encoding/json"
"errors"
"fmt"
"io"
"math"
"reflect"
"sort"
"strconv"
"strings"
"time"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/encoding/protojson"
protoV2 "google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
)
const wrapJSONMarshalV2 = false
// Marshaler is a configurable object for marshaling protocol buffer messages
// to the specified JSON representation.
type Marshaler struct {
// OrigName specifies whether to use the original protobuf name for fields.
OrigName bool
// EnumsAsInts specifies whether to render enum values as integers,
// as opposed to string values.
EnumsAsInts bool
// EmitDefaults specifies whether to render fields with zero values.
EmitDefaults bool
// Indent controls whether the output is compact or not.
// If empty, the output is compact JSON. Otherwise, every JSON object
// entry and JSON array value will be on its own line.
// Each line will be preceded by repeated copies of Indent, where the
// number of copies is the current indentation depth.
Indent string
// AnyResolver is used to resolve the google.protobuf.Any well-known type.
// If unset, the global registry is used by default.
AnyResolver AnyResolver
}
// JSONPBMarshaler is implemented by protobuf messages that customize the
// way they are marshaled to JSON. Messages that implement this should also
// implement JSONPBUnmarshaler so that the custom format can be parsed.
//
// The JSON marshaling must follow the proto to JSON specification:
// https://developers.google.com/protocol-buffers/docs/proto3#json
//
// Deprecated: Custom types should implement protobuf reflection instead.
type JSONPBMarshaler interface {
MarshalJSONPB(*Marshaler) ([]byte, error)
}
// Marshal serializes a protobuf message as JSON into w.
func (jm *Marshaler) Marshal(w io.Writer, m proto.Message) error {
b, err := jm.marshal(m)
if len(b) > 0 {
if _, err := w.Write(b); err != nil {
return err
}
}
return err
}
// MarshalToString serializes a protobuf message as JSON in string form.
func (jm *Marshaler) MarshalToString(m proto.Message) (string, error) {
b, err := jm.marshal(m)
if err != nil {
return "", err
}
return string(b), nil
}
func (jm *Marshaler) marshal(m proto.Message) ([]byte, error) {
v := reflect.ValueOf(m)
if m == nil || (v.Kind() == reflect.Ptr && v.IsNil()) {
return nil, errors.New("Marshal called with nil")
}
// Check for custom marshalers first since they may not properly
// implement protobuf reflection that the logic below relies on.
if jsm, ok := m.(JSONPBMarshaler); ok {
return jsm.MarshalJSONPB(jm)
}
if wrapJSONMarshalV2 {
opts := protojson.MarshalOptions{
UseProtoNames: jm.OrigName,
UseEnumNumbers: jm.EnumsAsInts,
EmitUnpopulated: jm.EmitDefaults,
Indent: jm.Indent,
}
if jm.AnyResolver != nil {
opts.Resolver = anyResolver{jm.AnyResolver}
}
return opts.Marshal(proto.MessageReflect(m).Interface())
} else {
// Check for unpopulated required fields first.
m2 := proto.MessageReflect(m)
if err := protoV2.CheckInitialized(m2.Interface()); err != nil {
return nil, err
}
w := jsonWriter{Marshaler: jm}
err := w.marshalMessage(m2, "", "")
return w.buf, err
}
}
type jsonWriter struct {
*Marshaler
buf []byte
}
func (w *jsonWriter) write(s string) {
w.buf = append(w.buf, s...)
}
func (w *jsonWriter) marshalMessage(m protoreflect.Message, indent, typeURL string) error {
if jsm, ok := proto.MessageV1(m.Interface()).(JSONPBMarshaler); ok {
b, err := jsm.MarshalJSONPB(w.Marshaler)
if err != nil {
return err
}
if typeURL != "" {
// we are marshaling this object to an Any type
var js map[string]*json.RawMessage
if err = json.Unmarshal(b, &js); err != nil {
return fmt.Errorf("type %T produced invalid JSON: %v", m.Interface(), err)
}
turl, err := json.Marshal(typeURL)
if err != nil {
return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err)
}
js["@type"] = (*json.RawMessage)(&turl)
if b, err = json.Marshal(js); err != nil {
return err
}
}
w.write(string(b))
return nil
}
md := m.Descriptor()
fds := md.Fields()
// Handle well-known types.
const secondInNanos = int64(time.Second / time.Nanosecond)
switch wellKnownType(md.FullName()) {
case "Any":
return w.marshalAny(m, indent)
case "BoolValue", "BytesValue", "StringValue",
"Int32Value", "UInt32Value", "FloatValue",
"Int64Value", "UInt64Value", "DoubleValue":
fd := fds.ByNumber(1)
return w.marshalValue(fd, m.Get(fd), indent)
case "Duration":
const maxSecondsInDuration = 315576000000
// "Generated output always contains 0, 3, 6, or 9 fractional digits,
// depending on required precision."
s := m.Get(fds.ByNumber(1)).Int()
ns := m.Get(fds.ByNumber(2)).Int()
if s < -maxSecondsInDuration || s > maxSecondsInDuration {
return fmt.Errorf("seconds out of range %v", s)
}
if ns <= -secondInNanos || ns >= secondInNanos {
return fmt.Errorf("ns out of range (%v, %v)", -secondInNanos, secondInNanos)
}
if (s > 0 && ns < 0) || (s < 0 && ns > 0) {
return errors.New("signs of seconds and nanos do not match")
}
var sign string
if s < 0 || ns < 0 {
sign, s, ns = "-", -1*s, -1*ns
}
x := fmt.Sprintf("%s%d.%09d", sign, s, ns)
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, ".000")
w.write(fmt.Sprintf(`"%vs"`, x))
return nil
case "Timestamp":
// "RFC 3339, where generated output will always be Z-normalized
// and uses 0, 3, 6 or 9 fractional digits."
s := m.Get(fds.ByNumber(1)).Int()
ns := m.Get(fds.ByNumber(2)).Int()
if ns < 0 || ns >= secondInNanos {
return fmt.Errorf("ns out of range [0, %v)", secondInNanos)
}
t := time.Unix(s, ns).UTC()
// time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits).
x := t.Format("2006-01-02T15:04:05.000000000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, ".000")
w.write(fmt.Sprintf(`"%vZ"`, x))
return nil
case "Value":
// JSON value; which is a null, number, string, bool, object, or array.
od := md.Oneofs().Get(0)
fd := m.WhichOneof(od)
if fd == nil {
return errors.New("nil Value")
}
return w.marshalValue(fd, m.Get(fd), indent)
case "Struct", "ListValue":
// JSON object or array.
fd := fds.ByNumber(1)
return w.marshalValue(fd, m.Get(fd), indent)
}
w.write("{")
if w.Indent != "" {
w.write("\n")
}
firstField := true
if typeURL != "" {
if err := w.marshalTypeURL(indent, typeURL); err != nil {
return err
}
firstField = false
}
for i := 0; i < fds.Len(); {
fd := fds.Get(i)
if od := fd.ContainingOneof(); od != nil {
fd = m.WhichOneof(od)
i += od.Fields().Len()
if fd == nil {
continue
}
} else {
i++
}
v := m.Get(fd)
if !m.Has(fd) {
if !w.EmitDefaults || fd.ContainingOneof() != nil {
continue
}
if fd.Cardinality() != protoreflect.Repeated && (fd.Message() != nil || fd.Syntax() == protoreflect.Proto2) {
v = protoreflect.Value{} // use "null" for singular messages or proto2 scalars
}
}
if !firstField {
w.writeComma()
}
if err := w.marshalField(fd, v, indent); err != nil {
return err
}
firstField = false
}
// Handle proto2 extensions.
if md.ExtensionRanges().Len() > 0 {
// Collect a sorted list of all extension descriptor and values.
type ext struct {
desc protoreflect.FieldDescriptor
val protoreflect.Value
}
var exts []ext
m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
if fd.IsExtension() {
exts = append(exts, ext{fd, v})
}
return true
})
sort.Slice(exts, func(i, j int) bool {
return exts[i].desc.Number() < exts[j].desc.Number()
})
for _, ext := range exts {
if !firstField {
w.writeComma()
}
if err := w.marshalField(ext.desc, ext.val, indent); err != nil {
return err
}
firstField = false
}
}
if w.Indent != "" {
w.write("\n")
w.write(indent)
}
w.write("}")
return nil
}
func (w *jsonWriter) writeComma() {
if w.Indent != "" {
w.write(",\n")
} else {
w.write(",")
}
}
func (w *jsonWriter) marshalAny(m protoreflect.Message, indent string) error {
// "If the Any contains a value that has a special JSON mapping,
// it will be converted as follows: {"@type": xxx, "value": yyy}.
// Otherwise, the value will be converted into a JSON object,
// and the "@type" field will be inserted to indicate the actual data type."
md := m.Descriptor()
typeURL := m.Get(md.Fields().ByNumber(1)).String()
rawVal := m.Get(md.Fields().ByNumber(2)).Bytes()
var m2 protoreflect.Message
if w.AnyResolver != nil {
mi, err := w.AnyResolver.Resolve(typeURL)
if err != nil {
return err
}
m2 = proto.MessageReflect(mi)
} else {
mt, err := protoregistry.GlobalTypes.FindMessageByURL(typeURL)
if err != nil {
return err
}
m2 = mt.New()
}
if err := protoV2.Unmarshal(rawVal, m2.Interface()); err != nil {
return err
}
if wellKnownType(m2.Descriptor().FullName()) == "" {
return w.marshalMessage(m2, indent, typeURL)
}
w.write("{")
if w.Indent != "" {
w.write("\n")
}
if err := w.marshalTypeURL(indent, typeURL); err != nil {
return err
}
w.writeComma()
if w.Indent != "" {
w.write(indent)
w.write(w.Indent)
w.write(`"value": `)
} else {
w.write(`"value":`)
}
if err := w.marshalMessage(m2, indent+w.Indent, ""); err != nil {
return err
}
if w.Indent != "" {
w.write("\n")
w.write(indent)
}
w.write("}")
return nil
}
func (w *jsonWriter) marshalTypeURL(indent, typeURL string) error {
if w.Indent != "" {
w.write(indent)
w.write(w.Indent)
}
w.write(`"@type":`)
if w.Indent != "" {
w.write(" ")
}
b, err := json.Marshal(typeURL)
if err != nil {
return err
}
w.write(string(b))
return nil
}
// marshalField writes field description and value to the Writer.
func (w *jsonWriter) marshalField(fd protoreflect.FieldDescriptor, v protoreflect.Value, indent string) error {
if w.Indent != "" {
w.write(indent)
w.write(w.Indent)
}
w.write(`"`)
switch {
case fd.IsExtension():
// For message set, use the fname of the message as the extension name.
name := string(fd.FullName())
if isMessageSet(fd.ContainingMessage()) {
name = strings.TrimSuffix(name, ".message_set_extension")
}
w.write("[" + name + "]")
case w.OrigName:
name := string(fd.Name())
if fd.Kind() == protoreflect.GroupKind {
name = string(fd.Message().Name())
}
w.write(name)
default:
w.write(string(fd.JSONName()))
}
w.write(`":`)
if w.Indent != "" {
w.write(" ")
}
return w.marshalValue(fd, v, indent)
}
func (w *jsonWriter) marshalValue(fd protoreflect.FieldDescriptor, v protoreflect.Value, indent string) error {
switch {
case fd.IsList():
w.write("[")
comma := ""
lv := v.List()
for i := 0; i < lv.Len(); i++ {
w.write(comma)
if w.Indent != "" {
w.write("\n")
w.write(indent)
w.write(w.Indent)
w.write(w.Indent)
}
if err := w.marshalSingularValue(fd, lv.Get(i), indent+w.Indent); err != nil {
return err
}
comma = ","
}
if w.Indent != "" {
w.write("\n")
w.write(indent)
w.write(w.Indent)
}
w.write("]")
return nil
case fd.IsMap():
kfd := fd.MapKey()
vfd := fd.MapValue()
mv := v.Map()
// Collect a sorted list of all map keys and values.
type entry struct{ key, val protoreflect.Value }
var entries []entry
mv.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool {
entries = append(entries, entry{k.Value(), v})
return true
})
sort.Slice(entries, func(i, j int) bool {
switch kfd.Kind() {
case protoreflect.BoolKind:
return !entries[i].key.Bool() && entries[j].key.Bool()
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind, protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
return entries[i].key.Int() < entries[j].key.Int()
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind, protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
return entries[i].key.Uint() < entries[j].key.Uint()
case protoreflect.StringKind:
return entries[i].key.String() < entries[j].key.String()
default:
panic("invalid kind")
}
})
w.write(`{`)
comma := ""
for _, entry := range entries {
w.write(comma)
if w.Indent != "" {
w.write("\n")
w.write(indent)
w.write(w.Indent)
w.write(w.Indent)
}
s := fmt.Sprint(entry.key.Interface())
b, err := json.Marshal(s)
if err != nil {
return err
}
w.write(string(b))
w.write(`:`)
if w.Indent != "" {
w.write(` `)
}
if err := w.marshalSingularValue(vfd, entry.val, indent+w.Indent); err != nil {
return err
}
comma = ","
}
if w.Indent != "" {
w.write("\n")
w.write(indent)
w.write(w.Indent)
}
w.write(`}`)
return nil
default:
return w.marshalSingularValue(fd, v, indent)
}
}
func (w *jsonWriter) marshalSingularValue(fd protoreflect.FieldDescriptor, v protoreflect.Value, indent string) error {
switch {
case !v.IsValid():
w.write("null")
return nil
case fd.Message() != nil:
return w.marshalMessage(v.Message(), indent+w.Indent, "")
case fd.Enum() != nil:
if fd.Enum().FullName() == "google.protobuf.NullValue" {
w.write("null")
return nil
}
vd := fd.Enum().Values().ByNumber(v.Enum())
if vd == nil || w.EnumsAsInts {
w.write(strconv.Itoa(int(v.Enum())))
} else {
w.write(`"` + string(vd.Name()) + `"`)
}
return nil
default:
switch v.Interface().(type) {
case float32, float64:
switch {
case math.IsInf(v.Float(), +1):
w.write(`"Infinity"`)
return nil
case math.IsInf(v.Float(), -1):
w.write(`"-Infinity"`)
return nil
case math.IsNaN(v.Float()):
w.write(`"NaN"`)
return nil
}
case int64, uint64:
w.write(fmt.Sprintf(`"%d"`, v.Interface()))
return nil
}
b, err := json.Marshal(v.Interface())
if err != nil {
return err
}
w.write(string(b))
return nil
}
}

Some files were not shown because too many files have changed in this diff Show More