perf: cache the metadata of the scanner (#14879)

1. Cache the metadata of scanner 30s.
2. Change the scanner client request timeout to 5s.

Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2021-05-14 14:27:37 +08:00 committed by GitHub
parent a19f6f8748
commit 6f3607cebd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 137 additions and 57 deletions

View File

@ -16,8 +16,13 @@ package scanner
import ( import (
"context" "context"
"fmt"
"sync"
"time"
"github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/lib/cache"
_ "github.com/goharbor/harbor/src/lib/cache/memory" // memory cache
"github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/q" "github.com/goharbor/harbor/src/lib/q"
@ -47,12 +52,24 @@ func New() Controller {
// basicController is default implementation of api.Controller interface // basicController is default implementation of api.Controller interface
type basicController struct { type basicController struct {
sync.Once
// Managers for managing the scanner registrations // Managers for managing the scanner registrations
manager rscanner.Manager manager rscanner.Manager
// For operating the project level configured scanner // For operating the project level configured scanner
proMetaMgr metadata.Manager proMetaMgr metadata.Manager
// Client pool for talking to adapters // Client pool for talking to adapters
clientPool v1.ClientPool clientPool v1.ClientPool
// Cache of the scanner metadata
cache cache.Cache
}
func (bc *basicController) Cache() cache.Cache {
bc.Do(func() {
bc.cache, _ = cache.New(cache.Memory, cache.Expiration(time.Second*30))
})
return bc.cache
} }
// ListRegistrations ... // ListRegistrations ...
@ -266,56 +283,28 @@ func (bc *basicController) Ping(ctx context.Context, registration *scanner.Regis
return nil, errors.New("nil registration to ping") return nil, errors.New("nil registration to ping")
} }
client, err := registration.Client(bc.clientPool) var (
err error
meta *v1.ScannerAdapterMetadata
)
if registration.ID > 0 {
meta, err = bc.getScannerAdapterMetadataWithCache(registration)
} else {
meta, err = bc.getScannerAdapterMetadata(registration)
}
if err != nil { if err != nil {
log.G(ctx).WithField("error", err).Error("failed to ping scanner")
return nil, errors.Wrap(err, "scanner controller: ping") return nil, errors.Wrap(err, "scanner controller: ping")
} }
meta, err := client.GetMetadata() if err := meta.Validate(); err != nil {
if err != nil { return nil, err
return nil, errors.Wrap(err, "scanner controller: ping")
} }
// Validate the required properties return meta, nil
if meta.Scanner == nil ||
len(meta.Scanner.Name) == 0 ||
len(meta.Scanner.Version) == 0 ||
len(meta.Scanner.Vendor) == 0 {
return nil, errors.New("invalid scanner in metadata")
}
if len(meta.Capabilities) == 0 {
return nil, errors.New("invalid capabilities in metadata")
}
for _, ca := range meta.Capabilities {
// v1.MimeTypeDockerArtifact is required now
found := false
for _, cm := range ca.ConsumesMimeTypes {
if cm == v1.MimeTypeDockerArtifact {
found = true
break
}
}
if !found {
return nil, errors.Errorf("missing %s in consumes_mime_types", v1.MimeTypeDockerArtifact)
}
// either of v1.MimeTypeNativeReport OR v1.MimeTypeGenericVulnerabilityReport is required
found = false
for _, pm := range ca.ProducesMimeTypes {
if pm == v1.MimeTypeNativeReport || pm == v1.MimeTypeGenericVulnerabilityReport {
found = true
break
}
}
if !found {
return nil, errors.Errorf("missing %s or %s in produces_mime_types", v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport)
}
}
return meta, err
} }
// GetMetadata ... // GetMetadata ...
@ -336,6 +325,35 @@ func (bc *basicController) GetMetadata(ctx context.Context, registrationUUID str
return bc.Ping(ctx, r) return bc.Ping(ctx, r)
} }
func (bc *basicController) getScannerAdapterMetadata(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) {
client, err := registration.Client(bc.clientPool)
if err != nil {
return nil, err
}
return client.GetMetadata()
}
func (bc *basicController) getScannerAdapterMetadataWithCache(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) {
key := fmt.Sprintf("reg:%d:metadata", registration.ID)
var result MetadataResult
err := cache.FetchOrSave(bc.Cache(), key, &result, func() (interface{}, error) {
meta, err := bc.getScannerAdapterMetadata(registration)
if err != nil {
return &MetadataResult{Error: err.Error()}, nil
}
return &MetadataResult{Metadata: meta}, nil
})
if err != nil {
return nil, err
}
return result.Unpack()
}
var ( var (
reservedNames = []string{"Trivy"} reservedNames = []string{"Trivy"}
) )
@ -349,3 +367,19 @@ func isReservedName(name string) bool {
return false return false
} }
// MetadataResult metadata or error saved in cache
type MetadataResult struct {
Metadata *v1.ScannerAdapterMetadata
Error string
}
// Unpack get ScannerAdapterMetadata and error from the result
func (m *MetadataResult) Unpack() (*v1.ScannerAdapterMetadata, error) {
var err error
if m.Error != "" {
err = fmt.Errorf(m.Error)
}
return m.Metadata, err
}

View File

@ -25,6 +25,15 @@ import (
"github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/log"
) )
const (
// Memory the cache name of memory
Memory = "memory"
// Redis the cache name of redis
Redis = "redis"
// RedisSentinel the cache name of redis sentinel
RedisSentinel = "redis+sentinel"
)
var ( var (
// ErrNotFound error returns when the key value not found in the cache // ErrNotFound error returns when the key value not found in the cache
ErrNotFound = errors.New("key not found") ErrNotFound = errors.New("key not found")

View File

@ -116,5 +116,5 @@ func New(opts cache.Options) (cache.Cache, error) {
} }
func init() { func init() {
cache.Register("memory", New) cache.Register(cache.Memory, New)
} }

View File

@ -127,6 +127,6 @@ func New(opts cache.Options) (cache.Cache, error) {
} }
func init() { func init() {
cache.Register("redis", New) cache.Register(cache.Redis, New)
cache.Register("redis+sentinel", New) cache.Register(cache.RedisSentinel, New)
} }

View File

@ -20,7 +20,6 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net"
"net/http" "net/http"
"strconv" "strconv"
"time" "time"
@ -83,14 +82,7 @@ type basicClient struct {
func NewClient(url, authType, accessCredential string, skipCertVerify bool) (Client, error) { func NewClient(url, authType, accessCredential string, skipCertVerify bool) (Client, error) {
transport := &http.Transport{ transport := &http.Transport{
Proxy: http.ProxyFromEnvironment, Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
MaxIdleConns: 100, MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{ TLSClientConfig: &tls.Config{
InsecureSkipVerify: skipCertVerify, InsecureSkipVerify: skipCertVerify,
}, },
@ -103,6 +95,7 @@ func NewClient(url, authType, accessCredential string, skipCertVerify bool) (Cli
return &basicClient{ return &basicClient{
httpClient: &http.Client{ httpClient: &http.Client{
Timeout: time.Second * 5,
Transport: transport, Transport: transport,
CheckRedirect: func(req *http.Request, via []*http.Request) error { CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse return http.ErrUseLastResponse

View File

@ -65,6 +65,50 @@ type ScannerAdapterMetadata struct {
Properties ScannerProperties `json:"properties"` Properties ScannerProperties `json:"properties"`
} }
// Validate validate the metadata
func (md *ScannerAdapterMetadata) Validate() error {
// Validate the required properties
if md.Scanner == nil ||
len(md.Scanner.Name) == 0 ||
len(md.Scanner.Version) == 0 ||
len(md.Scanner.Vendor) == 0 {
return errors.New("invalid scanner in metadata")
}
if len(md.Capabilities) == 0 {
return errors.New("invalid capabilities in metadata")
}
for _, ca := range md.Capabilities {
// v1.MimeTypeDockerArtifact is required now
found := false
for _, cm := range ca.ConsumesMimeTypes {
if cm == MimeTypeDockerArtifact {
found = true
break
}
}
if !found {
return errors.Errorf("missing %s in consumes_mime_types", MimeTypeDockerArtifact)
}
// either of v1.MimeTypeNativeReport OR v1.MimeTypeGenericVulnerabilityReport is required
found = false
for _, pm := range ca.ProducesMimeTypes {
if pm == MimeTypeNativeReport || pm == MimeTypeGenericVulnerabilityReport {
found = true
break
}
}
if !found {
return errors.Errorf("missing %s or %s in produces_mime_types", MimeTypeNativeReport, MimeTypeGenericVulnerabilityReport)
}
}
return nil
}
// HasCapability returns true when mine type of the artifact support by the scanner // HasCapability returns true when mine type of the artifact support by the scanner
func (md *ScannerAdapterMetadata) HasCapability(mimeType string) bool { func (md *ScannerAdapterMetadata) HasCapability(mimeType string) bool {
for _, capability := range md.Capabilities { for _, capability := range md.Capabilities {