2019-09-19 13:15:37 +02:00
|
|
|
// Copyright Project Harbor Authors
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package scan
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
|
|
|
"reflect"
|
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/goharbor/harbor/src/jobservice/job"
|
|
|
|
"github.com/goharbor/harbor/src/jobservice/logger"
|
|
|
|
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
|
|
|
|
"github.com/goharbor/harbor/src/pkg/scan/report"
|
|
|
|
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
// JobParamRegistration ...
|
|
|
|
JobParamRegistration = "registration"
|
|
|
|
// JobParameterRequest ...
|
|
|
|
JobParameterRequest = "scanRequest"
|
|
|
|
// JobParameterMimes ...
|
|
|
|
JobParameterMimes = "mimeTypes"
|
2019-10-21 14:07:00 +02:00
|
|
|
// JobParameterRobotID ...
|
|
|
|
JobParameterRobotID = "robotID"
|
2019-09-19 13:15:37 +02:00
|
|
|
|
|
|
|
checkTimeout = 30 * time.Minute
|
|
|
|
firstCheckInterval = 2 * time.Second
|
|
|
|
)
|
|
|
|
|
|
|
|
// CheckInReport defines model for checking in the scan report with specified mime.
|
|
|
|
type CheckInReport struct {
|
|
|
|
Digest string `json:"digest"`
|
|
|
|
RegistrationUUID string `json:"registration_uuid"`
|
|
|
|
MimeType string `json:"mime_type"`
|
|
|
|
RawReport string `json:"raw_report"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// FromJSON parse json to CheckInReport
|
|
|
|
func (cir *CheckInReport) FromJSON(jsonData string) error {
|
|
|
|
if len(jsonData) == 0 {
|
|
|
|
return errors.New("empty JSON data")
|
|
|
|
}
|
|
|
|
|
|
|
|
return json.Unmarshal([]byte(jsonData), cir)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ToJSON marshal CheckInReport to JSON
|
|
|
|
func (cir *CheckInReport) ToJSON() (string, error) {
|
|
|
|
jsonData, err := json.Marshal(cir)
|
|
|
|
if err != nil {
|
|
|
|
return "", errors.Wrap(err, "To JSON: CheckInReport")
|
|
|
|
}
|
|
|
|
|
|
|
|
return string(jsonData), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Job for running scan in the job service with async way
|
|
|
|
type Job struct{}
|
|
|
|
|
|
|
|
// MaxFails for defining the number of retries
|
|
|
|
func (j *Job) MaxFails() uint {
|
|
|
|
return 3
|
|
|
|
}
|
|
|
|
|
|
|
|
// ShouldRetry indicates if the job should be retried
|
|
|
|
func (j *Job) ShouldRetry() bool {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
// Validate the parameters of this job
|
|
|
|
func (j *Job) Validate(params job.Parameters) error {
|
|
|
|
if params == nil {
|
|
|
|
// Params are required
|
|
|
|
return errors.New("missing parameter of scan job")
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, err := extractRegistration(params); err != nil {
|
|
|
|
return errors.Wrap(err, "job validate")
|
|
|
|
}
|
|
|
|
|
2019-10-15 07:45:53 +02:00
|
|
|
if _, err := ExtractScanReq(params); err != nil {
|
2019-09-19 13:15:37 +02:00
|
|
|
return errors.Wrap(err, "job validate")
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, err := extractMimeTypes(params); err != nil {
|
|
|
|
return errors.Wrap(err, "job validate")
|
|
|
|
}
|
|
|
|
|
2019-10-21 14:07:00 +02:00
|
|
|
// No need to check param robotID which os treated as an optional one.
|
|
|
|
// It is used to clear the generated robot account to reduce dirty data.
|
|
|
|
// Failure of doing this will not influence the main flow.
|
|
|
|
|
2019-09-19 13:15:37 +02:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Run the job
|
|
|
|
func (j *Job) Run(ctx job.Context, params job.Parameters) error {
|
|
|
|
// Get logger
|
|
|
|
myLogger := ctx.GetLogger()
|
|
|
|
|
|
|
|
// Ignore errors as they have been validated already
|
|
|
|
r, _ := extractRegistration(params)
|
2019-10-15 07:45:53 +02:00
|
|
|
req, _ := ExtractScanReq(params)
|
2019-09-19 13:15:37 +02:00
|
|
|
mimes, _ := extractMimeTypes(params)
|
|
|
|
|
|
|
|
// Print related infos to log
|
|
|
|
printJSONParameter(JobParamRegistration, params[JobParamRegistration].(string), myLogger)
|
2019-09-24 09:17:40 +02:00
|
|
|
printJSONParameter(JobParameterRequest, removeAuthInfo(req), myLogger)
|
|
|
|
myLogger.Infof("Report mime types: %v\n", mimes)
|
2019-09-19 13:15:37 +02:00
|
|
|
|
|
|
|
// Submit scan request to the scanner adapter
|
2020-03-12 12:30:12 +01:00
|
|
|
client, err := r.Client(v1.DefaultClientPool)
|
2019-09-19 13:15:37 +02:00
|
|
|
if err != nil {
|
2019-09-24 09:17:40 +02:00
|
|
|
return logAndWrapError(myLogger, err, "scan job: get client")
|
2019-09-19 13:15:37 +02:00
|
|
|
}
|
|
|
|
|
2019-10-21 14:07:00 +02:00
|
|
|
// Ignore the namespace ID here
|
|
|
|
req.Artifact.NamespaceID = 0
|
2019-09-19 13:15:37 +02:00
|
|
|
resp, err := client.SubmitScan(req)
|
|
|
|
if err != nil {
|
2019-09-24 09:17:40 +02:00
|
|
|
return logAndWrapError(myLogger, err, "scan job: submit scan request")
|
2019-09-19 13:15:37 +02:00
|
|
|
}
|
|
|
|
|
2019-09-23 10:03:42 +02:00
|
|
|
// For collecting errors
|
|
|
|
errs := make([]error, len(mimes))
|
2019-09-19 13:15:37 +02:00
|
|
|
|
|
|
|
// Concurrently retrieving report by different mime types
|
|
|
|
wg := &sync.WaitGroup{}
|
|
|
|
wg.Add(len(mimes))
|
|
|
|
|
2019-09-23 10:03:42 +02:00
|
|
|
for i, mt := range mimes {
|
|
|
|
go func(i int, m string) {
|
2019-09-19 13:15:37 +02:00
|
|
|
defer wg.Done()
|
|
|
|
|
|
|
|
// Log info
|
|
|
|
myLogger.Infof("Get report for mime type: %s", m)
|
|
|
|
|
|
|
|
// Loop check if the report is ready
|
|
|
|
tm := time.NewTimer(firstCheckInterval)
|
|
|
|
defer tm.Stop()
|
|
|
|
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case t := <-tm.C:
|
|
|
|
myLogger.Debugf("check scan report for mime %s at %s", m, t.Format("2006/01/02 15:04:05"))
|
|
|
|
|
|
|
|
rawReport, err := client.GetScanReport(resp.ID, m)
|
|
|
|
if err != nil {
|
|
|
|
// Not ready yet
|
|
|
|
if notReadyErr, ok := err.(*v1.ReportNotReadyError); ok {
|
|
|
|
// Reset to the new check interval
|
|
|
|
tm.Reset(time.Duration(notReadyErr.RetryAfter) * time.Second)
|
|
|
|
myLogger.Infof("Report with mime type %s is not ready yet, retry after %d seconds", m, notReadyErr.RetryAfter)
|
|
|
|
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2019-09-23 10:03:42 +02:00
|
|
|
errs[i] = errors.Wrap(err, fmt.Sprintf("check scan report with mime type %s", m))
|
2019-09-19 13:15:37 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Make sure the data is aligned with the v1 spec.
|
|
|
|
if _, err = report.ResolveData(m, []byte(rawReport)); err != nil {
|
2019-09-23 10:03:42 +02:00
|
|
|
errs[i] = errors.Wrap(err, "scan job: resolve report data")
|
2019-09-19 13:15:37 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check in
|
|
|
|
cir := &CheckInReport{
|
|
|
|
Digest: req.Artifact.Digest,
|
|
|
|
RegistrationUUID: r.UUID,
|
|
|
|
MimeType: m,
|
|
|
|
RawReport: rawReport,
|
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
|
|
|
jsonData string
|
|
|
|
er error
|
|
|
|
)
|
|
|
|
if jsonData, er = cir.ToJSON(); er == nil {
|
|
|
|
if er = ctx.Checkin(jsonData); er == nil {
|
|
|
|
// Done!
|
|
|
|
myLogger.Infof("Report with mime type %s is checked in", m)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Send error and exit
|
2019-09-23 10:03:42 +02:00
|
|
|
errs[i] = errors.Wrap(er, fmt.Sprintf("check in scan report for mime type %s", m))
|
2019-09-19 13:15:37 +02:00
|
|
|
return
|
|
|
|
case <-ctx.SystemContext().Done():
|
|
|
|
// Terminated by system
|
|
|
|
return
|
|
|
|
case <-time.After(checkTimeout):
|
2019-09-23 10:03:42 +02:00
|
|
|
errs[i] = errors.New("check scan report timeout")
|
2019-09-19 13:15:37 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
2019-09-23 10:03:42 +02:00
|
|
|
}(i, mt)
|
2019-09-19 13:15:37 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Wait for all the retrieving routines are completed
|
|
|
|
wg.Wait()
|
2019-09-23 10:03:42 +02:00
|
|
|
|
|
|
|
// Merge errors
|
|
|
|
for _, e := range errs {
|
|
|
|
if e != nil {
|
|
|
|
if err != nil {
|
|
|
|
err = errors.Wrap(e, err.Error())
|
|
|
|
} else {
|
|
|
|
err = e
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-19 13:15:37 +02:00
|
|
|
// Log error to the job log
|
|
|
|
if err != nil {
|
|
|
|
myLogger.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-10-15 07:45:53 +02:00
|
|
|
// ExtractScanReq extracts the scan request from the job parameters.
|
|
|
|
func ExtractScanReq(params job.Parameters) (*v1.ScanRequest, error) {
|
|
|
|
v, ok := params[JobParameterRequest]
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.Errorf("missing job parameter '%s'", JobParameterRequest)
|
|
|
|
}
|
|
|
|
|
|
|
|
jsonData, ok := v.(string)
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.Errorf(
|
|
|
|
"malformed job parameter '%s', expecting string but got %s",
|
|
|
|
JobParameterRequest,
|
|
|
|
reflect.TypeOf(v).String(),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
req := &v1.ScanRequest{}
|
|
|
|
if err := req.FromJSON(jsonData); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if err := req.Validate(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return req, nil
|
|
|
|
}
|
|
|
|
|
2019-09-24 09:17:40 +02:00
|
|
|
func logAndWrapError(logger logger.Interface, err error, message string) error {
|
|
|
|
e := errors.Wrap(err, message)
|
|
|
|
logger.Error(e)
|
|
|
|
|
|
|
|
return e
|
|
|
|
}
|
|
|
|
|
2019-09-19 13:15:37 +02:00
|
|
|
func printJSONParameter(parameter string, v string, logger logger.Interface) {
|
|
|
|
logger.Infof("%s:\n", parameter)
|
|
|
|
printPrettyJSON([]byte(v), logger)
|
|
|
|
}
|
|
|
|
|
|
|
|
func printPrettyJSON(in []byte, logger logger.Interface) {
|
|
|
|
var out bytes.Buffer
|
|
|
|
if err := json.Indent(&out, in, "", " "); err != nil {
|
|
|
|
logger.Errorf("Print pretty JSON error: %s", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
logger.Infof("%s\n", out.String())
|
|
|
|
}
|
|
|
|
|
2019-09-24 09:17:40 +02:00
|
|
|
func removeAuthInfo(sr *v1.ScanRequest) string {
|
|
|
|
req := &v1.ScanRequest{
|
|
|
|
Artifact: sr.Artifact,
|
|
|
|
Registry: &v1.Registry{
|
|
|
|
URL: sr.Registry.URL,
|
|
|
|
Authorization: "[HIDDEN]",
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
str, err := req.ToJSON()
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(errors.Wrap(err, "scan job: remove auth"))
|
|
|
|
}
|
|
|
|
|
|
|
|
return str
|
|
|
|
}
|
|
|
|
|
2019-09-19 13:15:37 +02:00
|
|
|
func extractRegistration(params job.Parameters) (*scanner.Registration, error) {
|
|
|
|
v, ok := params[JobParamRegistration]
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.Errorf("missing job parameter '%s'", JobParamRegistration)
|
|
|
|
}
|
|
|
|
|
|
|
|
jsonData, ok := v.(string)
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.Errorf(
|
|
|
|
"malformed job parameter '%s', expecting string but got %s",
|
|
|
|
JobParamRegistration,
|
|
|
|
reflect.TypeOf(v).String(),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
r := &scanner.Registration{}
|
|
|
|
if err := r.FromJSON(jsonData); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := r.Validate(true); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return r, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func extractMimeTypes(params job.Parameters) ([]string, error) {
|
|
|
|
v, ok := params[JobParameterMimes]
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.Errorf("missing job parameter '%s'", JobParameterMimes)
|
|
|
|
}
|
|
|
|
|
2019-09-24 09:17:40 +02:00
|
|
|
l, ok := v.([]interface{})
|
2019-09-19 13:15:37 +02:00
|
|
|
if !ok {
|
|
|
|
return nil, errors.Errorf(
|
2019-09-24 09:17:40 +02:00
|
|
|
"malformed job parameter '%s', expecting []interface{} but got %s",
|
2019-09-19 13:15:37 +02:00
|
|
|
JobParameterMimes,
|
|
|
|
reflect.TypeOf(v).String(),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2019-09-24 09:17:40 +02:00
|
|
|
mimes := make([]string, 0)
|
|
|
|
for _, v := range l {
|
|
|
|
mime, ok := v.(string)
|
|
|
|
if !ok {
|
|
|
|
return nil, errors.Errorf("expect string but got %s", reflect.TypeOf(v).String())
|
|
|
|
}
|
|
|
|
|
|
|
|
mimes = append(mimes, mime)
|
|
|
|
}
|
|
|
|
|
|
|
|
return mimes, nil
|
2019-09-19 13:15:37 +02:00
|
|
|
}
|