diff --git a/src/controller/scanner/base_controller.go b/src/controller/scanner/base_controller.go index b402d7ba7..c265d1833 100644 --- a/src/controller/scanner/base_controller.go +++ b/src/controller/scanner/base_controller.go @@ -16,8 +16,13 @@ package scanner import ( "context" + "fmt" + "sync" + "time" "github.com/goharbor/harbor/src/jobservice/logger" + "github.com/goharbor/harbor/src/lib/cache" + _ "github.com/goharbor/harbor/src/lib/cache/memory" // memory cache "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/q" @@ -47,12 +52,24 @@ func New() Controller { // basicController is default implementation of api.Controller interface type basicController struct { + sync.Once + // Managers for managing the scanner registrations manager rscanner.Manager // For operating the project level configured scanner proMetaMgr metadata.Manager // Client pool for talking to adapters clientPool v1.ClientPool + // Cache of the scanner metadata + cache cache.Cache +} + +func (bc *basicController) Cache() cache.Cache { + bc.Do(func() { + bc.cache, _ = cache.New(cache.Memory, cache.Expiration(time.Second*30)) + }) + + return bc.cache } // ListRegistrations ... @@ -266,56 +283,28 @@ func (bc *basicController) Ping(ctx context.Context, registration *scanner.Regis return nil, errors.New("nil registration to ping") } - client, err := registration.Client(bc.clientPool) + var ( + err error + meta *v1.ScannerAdapterMetadata + ) + + if registration.ID > 0 { + meta, err = bc.getScannerAdapterMetadataWithCache(registration) + } else { + meta, err = bc.getScannerAdapterMetadata(registration) + } + if err != nil { + log.G(ctx).WithField("error", err).Error("failed to ping scanner") + return nil, errors.Wrap(err, "scanner controller: ping") } - meta, err := client.GetMetadata() - if err != nil { - return nil, errors.Wrap(err, "scanner controller: ping") + if err := meta.Validate(); err != nil { + return nil, err } - // Validate the required properties - if meta.Scanner == nil || - len(meta.Scanner.Name) == 0 || - len(meta.Scanner.Version) == 0 || - len(meta.Scanner.Vendor) == 0 { - return nil, errors.New("invalid scanner in metadata") - } - - if len(meta.Capabilities) == 0 { - return nil, errors.New("invalid capabilities in metadata") - } - - for _, ca := range meta.Capabilities { - // v1.MimeTypeDockerArtifact is required now - found := false - for _, cm := range ca.ConsumesMimeTypes { - if cm == v1.MimeTypeDockerArtifact { - found = true - break - } - } - if !found { - return nil, errors.Errorf("missing %s in consumes_mime_types", v1.MimeTypeDockerArtifact) - } - - // either of v1.MimeTypeNativeReport OR v1.MimeTypeGenericVulnerabilityReport is required - found = false - for _, pm := range ca.ProducesMimeTypes { - if pm == v1.MimeTypeNativeReport || pm == v1.MimeTypeGenericVulnerabilityReport { - found = true - break - } - } - - if !found { - return nil, errors.Errorf("missing %s or %s in produces_mime_types", v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport) - } - } - - return meta, err + return meta, nil } // GetMetadata ... @@ -336,6 +325,35 @@ func (bc *basicController) GetMetadata(ctx context.Context, registrationUUID str return bc.Ping(ctx, r) } +func (bc *basicController) getScannerAdapterMetadata(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) { + client, err := registration.Client(bc.clientPool) + if err != nil { + return nil, err + } + + return client.GetMetadata() +} + +func (bc *basicController) getScannerAdapterMetadataWithCache(registration *scanner.Registration) (*v1.ScannerAdapterMetadata, error) { + key := fmt.Sprintf("reg:%d:metadata", registration.ID) + + var result MetadataResult + err := cache.FetchOrSave(bc.Cache(), key, &result, func() (interface{}, error) { + meta, err := bc.getScannerAdapterMetadata(registration) + if err != nil { + return &MetadataResult{Error: err.Error()}, nil + } + + return &MetadataResult{Metadata: meta}, nil + }) + + if err != nil { + return nil, err + } + + return result.Unpack() +} + var ( reservedNames = []string{"Trivy"} ) @@ -349,3 +367,19 @@ func isReservedName(name string) bool { return false } + +// MetadataResult metadata or error saved in cache +type MetadataResult struct { + Metadata *v1.ScannerAdapterMetadata + Error string +} + +// Unpack get ScannerAdapterMetadata and error from the result +func (m *MetadataResult) Unpack() (*v1.ScannerAdapterMetadata, error) { + var err error + if m.Error != "" { + err = fmt.Errorf(m.Error) + } + + return m.Metadata, err +} diff --git a/src/lib/cache/cache.go b/src/lib/cache/cache.go index 9c4a715ee..efc91dfdf 100644 --- a/src/lib/cache/cache.go +++ b/src/lib/cache/cache.go @@ -25,6 +25,15 @@ import ( "github.com/goharbor/harbor/src/lib/log" ) +const ( + // Memory the cache name of memory + Memory = "memory" + // Redis the cache name of redis + Redis = "redis" + // RedisSentinel the cache name of redis sentinel + RedisSentinel = "redis+sentinel" +) + var ( // ErrNotFound error returns when the key value not found in the cache ErrNotFound = errors.New("key not found") diff --git a/src/lib/cache/memory/memory.go b/src/lib/cache/memory/memory.go index 540c612c0..1b1bfa3c5 100644 --- a/src/lib/cache/memory/memory.go +++ b/src/lib/cache/memory/memory.go @@ -116,5 +116,5 @@ func New(opts cache.Options) (cache.Cache, error) { } func init() { - cache.Register("memory", New) + cache.Register(cache.Memory, New) } diff --git a/src/lib/cache/redis/redis.go b/src/lib/cache/redis/redis.go index db4e2437e..2cee92a6e 100644 --- a/src/lib/cache/redis/redis.go +++ b/src/lib/cache/redis/redis.go @@ -127,6 +127,6 @@ func New(opts cache.Options) (cache.Cache, error) { } func init() { - cache.Register("redis", New) - cache.Register("redis+sentinel", New) + cache.Register(cache.Redis, New) + cache.Register(cache.RedisSentinel, New) } diff --git a/src/pkg/scan/rest/v1/client.go b/src/pkg/scan/rest/v1/client.go index 6ba8d0b37..d37d397dc 100644 --- a/src/pkg/scan/rest/v1/client.go +++ b/src/pkg/scan/rest/v1/client.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net" "net/http" "strconv" "time" @@ -82,15 +81,8 @@ type basicClient struct { // NewClient news a basic client func NewClient(url, authType, accessCredential string, skipCertVerify bool) (Client, error) { transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, + Proxy: http.ProxyFromEnvironment, + MaxIdleConns: 100, TLSClientConfig: &tls.Config{ InsecureSkipVerify: skipCertVerify, }, @@ -103,6 +95,7 @@ func NewClient(url, authType, accessCredential string, skipCertVerify bool) (Cli return &basicClient{ httpClient: &http.Client{ + Timeout: time.Second * 5, Transport: transport, CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse diff --git a/src/pkg/scan/rest/v1/models.go b/src/pkg/scan/rest/v1/models.go index 937964ebc..ba70c8bac 100644 --- a/src/pkg/scan/rest/v1/models.go +++ b/src/pkg/scan/rest/v1/models.go @@ -65,6 +65,50 @@ type ScannerAdapterMetadata struct { Properties ScannerProperties `json:"properties"` } +// Validate validate the metadata +func (md *ScannerAdapterMetadata) Validate() error { + // Validate the required properties + if md.Scanner == nil || + len(md.Scanner.Name) == 0 || + len(md.Scanner.Version) == 0 || + len(md.Scanner.Vendor) == 0 { + return errors.New("invalid scanner in metadata") + } + + if len(md.Capabilities) == 0 { + return errors.New("invalid capabilities in metadata") + } + + for _, ca := range md.Capabilities { + // v1.MimeTypeDockerArtifact is required now + found := false + for _, cm := range ca.ConsumesMimeTypes { + if cm == MimeTypeDockerArtifact { + found = true + break + } + } + if !found { + return errors.Errorf("missing %s in consumes_mime_types", MimeTypeDockerArtifact) + } + + // either of v1.MimeTypeNativeReport OR v1.MimeTypeGenericVulnerabilityReport is required + found = false + for _, pm := range ca.ProducesMimeTypes { + if pm == MimeTypeNativeReport || pm == MimeTypeGenericVulnerabilityReport { + found = true + break + } + } + + if !found { + return errors.Errorf("missing %s or %s in produces_mime_types", MimeTypeNativeReport, MimeTypeGenericVulnerabilityReport) + } + } + + return nil +} + // HasCapability returns true when mine type of the artifact support by the scanner func (md *ScannerAdapterMetadata) HasCapability(mimeType string) bool { for _, capability := range md.Capabilities {