feat(scan): merge reports for image index

1. Merge the scanning reports of referenced artifacts for image index.
2. Add artifact info for report.

Signed-off-by: He Weiwei <hweiwei@vmware.com>

src/controller/scan/base_controller.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
-	"strings"
 	"sync"
 
 	cj "github.com/goharbor/harbor/src/common/job"
@@ -435,12 +434,7 @@ func (bc *basicController) GetScanLog(uuid string) ([]byte, error) {
 		return nil, errors.New("empty uuid to get scan log")
 	}
 
-	data, err := base64.StdEncoding.DecodeString(uuid)
-	if err != nil {
-		data = []byte(uuid)
-	}
-
-	reportIDs := strings.Split(string(data), vuln.SummaryReportIDSeparator)
+	reportIDs := vuln.ParseReportIDs(uuid)
 
 	errs := map[string]error{}
 	logs := make(map[string][]byte, len(reportIDs))

src/pkg/scan/report/report.go

@@ -0,0 +1,54 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package report
+
+import (
+	v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
+	"github.com/goharbor/harbor/src/pkg/scan/vuln"
+	"github.com/pkg/errors"
+)
+
+// Merger is a helper function to merge report together
+type Merger func(r1, r2 interface{}) (interface{}, error)
+
+// SupportedMergers declares mappings between mime type and report merger func.
+var SupportedMergers = map[string]Merger{
+	v1.MimeTypeNativeReport: MergeNativeReport,
+}
+
+// Merge merge report r1 and r2
+func Merge(mimeType string, r1, r2 interface{}) (interface{}, error) {
+	m, ok := SupportedMergers[mimeType]
+	if !ok {
+		return nil, errors.Errorf("no report merger bound with mime type %s", mimeType)
+	}
+
+	return m(r1, r2)
+}
+
+// MergeNativeReport merge report r1 and r2
+func MergeNativeReport(r1, r2 interface{}) (interface{}, error) {
+	nr1, ok := r1.(*vuln.Report)
+	if !ok {
+		return nil, errors.New("native report required")
+	}
+
+	nr2, ok := r2.(*vuln.Report)
+	if !ok {
+		return nil, errors.New("native report required")
+	}
+
+	return nr1.Merge(nr2), nil
+}

src/pkg/scan/report/summary.go

@@ -36,6 +36,8 @@ func (cs CVESet) Contains(cve string) bool {
 
 // Options provides options for getting the report w/ summary.
 type Options struct {
+	// If it is set, the returned report will contains artifact digest for the vulnerabilities
+	ArtifactDigest string
 	// If it is set, the returned summary will not count the CVEs in the list in.
 	CVEWhitelist CVESet
 }
@@ -50,19 +52,26 @@ func WithCVEWhitelist(set *CVESet) Option {
 	}
 }
 
+// WithArtifactDigest is an option of setting artifact digest
+func WithArtifactDigest(artifactDigest string) Option {
+	return func(options *Options) {
+		options.ArtifactDigest = artifactDigest
+	}
+}
+
 // SummaryMerger is a helper function to merge summary together
 type SummaryMerger func(s1, s2 interface{}) (interface{}, error)
 
-// SupportedMergers declares mappings between mime type and summary merger func.
-var SupportedMergers = map[string]SummaryMerger{
+// SupportedSummaryMergers declares mappings between mime type and summary merger func.
+var SupportedSummaryMergers = map[string]SummaryMerger{
 	v1.MimeTypeNativeReport: MergeNativeSummary,
 }
 
 // MergeSummary merge summary s1 and s2
 func MergeSummary(mimeType string, s1, s2 interface{}) (interface{}, error) {
-	m, ok := SupportedMergers[mimeType]
+	m, ok := SupportedSummaryMergers[mimeType]
 	if !ok {
-		return nil, errors.Errorf("no mreger bound with mime type %s", mimeType)
+		return nil, errors.Errorf("no summary merger bound with mime type %s", mimeType)
 	}
 
 	return m(s1, s2)

src/pkg/scan/report/supported_mimes.go

@@ -30,7 +30,7 @@ var SupportedMimes = map[string]interface{}{
 }
 
 // ResolveData is a helper func to parse the JSON data with the given mime type.
-func ResolveData(mime string, jsonData []byte) (interface{}, error) {
+func ResolveData(mime string, jsonData []byte, options ...Option) (interface{}, error) {
 	// If no resolver defined for the given mime types, directly ignore it.
 	// The raw data will be used.
 	t, ok := SupportedMimes[mime]
@@ -54,5 +54,16 @@ func ResolveData(mime string, jsonData []byte) (interface{}, error) {
 		return nil, err
 	}
 
+	ops := &Options{}
+	for _, op := range options {
+		op(ops)
+	}
+
+	if ops.ArtifactDigest != "" {
+		if w, ok := rp.(interface{ WithArtifactDigest(string) }); ok {
+			w.WithArtifactDigest(ops.ArtifactDigest)
+		}
+	}
+
 	return rp, nil
 }

src/pkg/scan/vuln/report.go

@@ -30,6 +30,37 @@ type Report struct {
 	Vulnerabilities []*VulnerabilityItem `json:"vulnerabilities"`
 }
 
+// Merge ...
+func (report *Report) Merge(another *Report) *Report {
+	generatedAt := report.GeneratedAt
+	if another.GeneratedAt > generatedAt {
+		generatedAt = another.GeneratedAt
+	}
+
+	vulnerabilities := report.Vulnerabilities
+	if vulnerabilities == nil {
+		vulnerabilities = another.Vulnerabilities
+	} else {
+		vulnerabilities = append(vulnerabilities, another.Vulnerabilities...)
+	}
+
+	r := &Report{
+		GeneratedAt:     generatedAt,
+		Scanner:         report.Scanner,
+		Severity:        mergeSeverity(report.Severity, another.Severity),
+		Vulnerabilities: vulnerabilities,
+	}
+
+	return r
+}
+
+// WithArtifactDigest set artifact digest for the report
+func (report *Report) WithArtifactDigest(artifactDigest string) {
+	for _, vul := range report.Vulnerabilities {
+		vul.ArtifactDigest = artifactDigest
+	}
+}
+
 // VulnerabilityItem represents one found vulnerability
 type VulnerabilityItem struct {
 	// The unique identifier of the vulnerability.
@@ -55,4 +86,7 @@ type VulnerabilityItem struct {
 	// Format: URI
 	// e.g: List [ "https://security-tracker.debian.org/tracker/CVE-2017-8283" ]
 	Links []string `json:"links"`
+	// The artifact digest which the vulnerability belonged
+	// e.g: sha256@ee1d00c5250b5a886b09be2d5f9506add35dfb557f1ef37a7e4b8f0138f32956
+	ArtifactDigest string `json:"artifact_digest"`
 }

src/pkg/scan/vuln/report_test.go

@@ -0,0 +1,69 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vuln
+
+import (
+	"reflect"
+	"testing"
+
+	v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
+)
+
+func TestReport_Merge(t *testing.T) {
+	emptyVulnerabilities := []*VulnerabilityItem{}
+	a := []*VulnerabilityItem{
+		{ID: "CVE-2017-8283"},
+		{ID: "CVE-2017-8284"},
+	}
+	b := []*VulnerabilityItem{
+		{ID: "CVE-2017-8285"},
+	}
+	type fields struct {
+		GeneratedAt     string
+		Scanner         *v1.Scanner
+		Severity        Severity
+		Vulnerabilities []*VulnerabilityItem
+	}
+	type args struct {
+		another *Report
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   *Report
+	}{
+		{"GeneratedAt", fields{GeneratedAt: "2020-04-06T18:38:34.791086859Z"}, args{&Report{GeneratedAt: "2020-04-06T18:38:34.791086860Z"}}, &Report{GeneratedAt: "2020-04-06T18:38:34.791086860Z"}},
+		{"Vulnerabilities nil & nil", fields{Vulnerabilities: nil}, args{&Report{Vulnerabilities: nil}}, &Report{Vulnerabilities: nil}},
+		{"Vulnerabilities nil & not nil", fields{Vulnerabilities: nil}, args{&Report{Vulnerabilities: emptyVulnerabilities}}, &Report{Vulnerabilities: emptyVulnerabilities}},
+		{"Vulnerabilities not nil & nil", fields{Vulnerabilities: emptyVulnerabilities}, args{&Report{Vulnerabilities: nil}}, &Report{Vulnerabilities: emptyVulnerabilities}},
+		{"Vulnerabilities nil & a", fields{Vulnerabilities: nil}, args{&Report{Vulnerabilities: a}}, &Report{Vulnerabilities: a}},
+		{"Vulnerabilities a & nil", fields{Vulnerabilities: a}, args{&Report{Vulnerabilities: nil}}, &Report{Vulnerabilities: a}},
+		{"Vulnerabilities a & b", fields{Vulnerabilities: a}, args{&Report{Vulnerabilities: b}}, &Report{Vulnerabilities: append(a, b...)}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			report := &Report{
+				GeneratedAt:     tt.fields.GeneratedAt,
+				Scanner:         tt.fields.Scanner,
+				Severity:        tt.fields.Severity,
+				Vulnerabilities: tt.fields.Vulnerabilities,
+			}
+			if got := report.Merge(tt.args.another); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Report.Merge() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

src/pkg/scan/vuln/summary.go

@@ -15,18 +15,12 @@
 package vuln
 
 import (
-	"encoding/base64"
 	"time"
 
 	"github.com/goharbor/harbor/src/jobservice/job"
 	v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
 )
 
-const (
-	// SummaryReportIDSeparator the separator of the ReportID in the summary when its merged by multi summaries
-	SummaryReportIDSeparator = "|"
-)
-
 // NativeReportSummary is the default supported scan report summary model.
 // Generated based on the report with v1.MimeTypeNativeReport mime type.
 type NativeReportSummary struct {
@@ -109,61 +103,3 @@ func (v *VulnerabilitySummary) Merge(a *VulnerabilitySummary) *VulnerabilitySumm
 
 // SeveritySummary ...
 type SeveritySummary map[Severity]int
-
-func minTime(t1, t2 time.Time) time.Time {
-	if t1.Before(t2) {
-		return t1
-	}
-
-	return t2
-}
-
-func maxTime(t1, t2 time.Time) time.Time {
-	if t1.Before(t2) {
-		return t2
-	}
-
-	return t1
-}
-
-func mergeReportID(r1, r2 string) string {
-	src, err := base64.StdEncoding.DecodeString(r1)
-	if err != nil {
-		src = []byte(r1)
-	}
-	src = append(src, []byte(SummaryReportIDSeparator+r2)...)
-
-	return base64.StdEncoding.EncodeToString(src)
-}
-
-func mergeSeverity(s1, s2 Severity) Severity {
-	severityValue := func(s Severity) int {
-		if s.String() == "" {
-			return -1
-		}
-
-		return s.Code()
-	}
-
-	if severityValue(s1) > severityValue(s2) {
-		return s1
-	}
-
-	return s2
-}
-
-func mergeScanStatus(s1, s2 string) string {
-	j1, j2 := job.Status(s1), job.Status(s2)
-
-	if j1 == job.RunningStatus || j2 == job.RunningStatus {
-		return job.RunningStatus.String()
-	} else if j1 == job.SuccessStatus || j2 == job.SuccessStatus {
-		return job.SuccessStatus.String()
-	}
-
-	if j1.Compare(j2) > 0 {
-		return s1
-	}
-
-	return s2
-}

src/pkg/scan/vuln/summary_test.go

@@ -15,89 +15,12 @@
 package vuln
 
 import (
-	"encoding/base64"
 	"testing"
 
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/stretchr/testify/assert"
 )
 
-func Test_mergeReportID(t *testing.T) {
-	type args struct {
-		r1 string
-		r2 string
-	}
-	tests := []struct {
-		name string
-		args args
-		want string
-	}{
-		{"1|2", args{"1", "2"}, base64.StdEncoding.EncodeToString([]byte("1|2"))},
-		{"1|2|3", args{base64.StdEncoding.EncodeToString([]byte("1|2")), "3"}, base64.StdEncoding.EncodeToString([]byte("1|2|3"))},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := mergeReportID(tt.args.r1, tt.args.r2); got != tt.want {
-				t.Errorf("mergeReportID() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_mergeSeverity(t *testing.T) {
-	type args struct {
-		s1 Severity
-		s2 Severity
-	}
-	tests := []struct {
-		name string
-		args args
-		want Severity
-	}{
-		{"empty string and none", args{Severity(""), None}, None},
-		{"none and empty string", args{None, Severity("")}, None},
-		{"none and low", args{None, Low}, Low},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := mergeSeverity(tt.args.s1, tt.args.s2); got != tt.want {
-				t.Errorf("mergeSeverity() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_mergeScanStatus(t *testing.T) {
-	errorStatus := job.ErrorStatus.String()
-	runningStatus := job.RunningStatus.String()
-	successStatus := job.SuccessStatus.String()
-
-	type args struct {
-		s1 string
-		s2 string
-	}
-	tests := []struct {
-		name string
-		args args
-		want string
-	}{
-		{"running and error", args{runningStatus, errorStatus}, runningStatus},
-		{"running and success", args{runningStatus, successStatus}, runningStatus},
-		{"running and running", args{runningStatus, runningStatus}, runningStatus},
-		{"success and error", args{successStatus, errorStatus}, successStatus},
-		{"success and success", args{successStatus, successStatus}, successStatus},
-		{"error and error", args{errorStatus, errorStatus}, errorStatus},
-		{"error and empty string", args{errorStatus, ""}, errorStatus},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := mergeScanStatus(tt.args.s1, tt.args.s2); got != tt.want {
-				t.Errorf("mergeScanStatus() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 func TestMergeVulnerabilitySummary(t *testing.T) {
 	assert := assert.New(t)
 	v1 := VulnerabilitySummary{

src/pkg/scan/vuln/util.go

@@ -0,0 +1,96 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vuln
+
+import (
+	"encoding/base64"
+	"strings"
+	"time"
+
+	"github.com/goharbor/harbor/src/jobservice/job"
+)
+
+const (
+	// reportIDSeparator the separator of the ReportID in the summary when its merged by multi summaries
+	reportIDSeparator = "|"
+)
+
+// ParseReportIDs returns report ids from s
+func ParseReportIDs(s string) []string {
+	data, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		data = []byte(s)
+	}
+
+	return strings.Split(string(data), reportIDSeparator)
+}
+
+func minTime(t1, t2 time.Time) time.Time {
+	if t1.Before(t2) {
+		return t1
+	}
+
+	return t2
+}
+
+func maxTime(t1, t2 time.Time) time.Time {
+	if t1.Before(t2) {
+		return t2
+	}
+
+	return t1
+}
+
+func mergeReportID(r1, r2 string) string {
+	src, err := base64.StdEncoding.DecodeString(r1)
+	if err != nil {
+		src = []byte(r1)
+	}
+	src = append(src, []byte(reportIDSeparator+r2)...)
+
+	return base64.StdEncoding.EncodeToString(src)
+}
+
+func mergeSeverity(s1, s2 Severity) Severity {
+	severityValue := func(s Severity) int {
+		if s.String() == "" {
+			return -1
+		}
+
+		return s.Code()
+	}
+
+	if severityValue(s1) > severityValue(s2) {
+		return s1
+	}
+
+	return s2
+}
+
+func mergeScanStatus(s1, s2 string) string {
+	j1, j2 := job.Status(s1), job.Status(s2)
+
+	if j1 == job.RunningStatus || j2 == job.RunningStatus {
+		return job.RunningStatus.String()
+	} else if j1 == job.SuccessStatus || j2 == job.SuccessStatus {
+		return job.SuccessStatus.String()
+	}
+
+	if j1.Compare(j2) > 0 {
+		return s1
+	}
+
+	return s2
+}

src/pkg/scan/vuln/util_test.go

@@ -0,0 +1,121 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vuln
+
+import (
+	"encoding/base64"
+	"reflect"
+	"testing"
+
+	"github.com/goharbor/harbor/src/jobservice/job"
+)
+
+func Test_mergeReportID(t *testing.T) {
+	type args struct {
+		r1 string
+		r2 string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{"1|2", args{"1", "2"}, base64.StdEncoding.EncodeToString([]byte("1|2"))},
+		{"1|2|3", args{base64.StdEncoding.EncodeToString([]byte("1|2")), "3"}, base64.StdEncoding.EncodeToString([]byte("1|2|3"))},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := mergeReportID(tt.args.r1, tt.args.r2); got != tt.want {
+				t.Errorf("mergeReportID() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_mergeSeverity(t *testing.T) {
+	type args struct {
+		s1 Severity
+		s2 Severity
+	}
+	tests := []struct {
+		name string
+		args args
+		want Severity
+	}{
+		{"empty string and none", args{Severity(""), None}, None},
+		{"none and empty string", args{None, Severity("")}, None},
+		{"none and low", args{None, Low}, Low},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := mergeSeverity(tt.args.s1, tt.args.s2); got != tt.want {
+				t.Errorf("mergeSeverity() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_mergeScanStatus(t *testing.T) {
+	errorStatus := job.ErrorStatus.String()
+	runningStatus := job.RunningStatus.String()
+	successStatus := job.SuccessStatus.String()
+
+	type args struct {
+		s1 string
+		s2 string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{"running and error", args{runningStatus, errorStatus}, runningStatus},
+		{"running and success", args{runningStatus, successStatus}, runningStatus},
+		{"running and running", args{runningStatus, runningStatus}, runningStatus},
+		{"success and error", args{successStatus, errorStatus}, successStatus},
+		{"success and success", args{successStatus, successStatus}, successStatus},
+		{"error and error", args{errorStatus, errorStatus}, errorStatus},
+		{"error and empty string", args{errorStatus, ""}, errorStatus},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := mergeScanStatus(tt.args.s1, tt.args.s2); got != tt.want {
+				t.Errorf("mergeScanStatus() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestParseReportIDs(t *testing.T) {
+	type args struct {
+		s string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{"1", args{"1"}, []string{"1"}},
+		{"1|2", args{base64.StdEncoding.EncodeToString([]byte("1|2"))}, []string{"1", "2"}},
+		{"1|2|3", args{base64.StdEncoding.EncodeToString([]byte("1|2|3"))}, []string{"1", "2", "3"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ParseReportIDs(tt.args.s); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ParseReportIDs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

src/server/v2.0/handler/util.go

@@ -50,12 +50,20 @@ func resolveVulnerabilitiesAddition(ctx context.Context, artifact *artifact.Arti
 			continue
 		}
 
-		vrp, err := report.ResolveData(rp.MimeType, []byte(rp.Report))
+		vrp, err := report.ResolveData(rp.MimeType, []byte(rp.Report), report.WithArtifactDigest(rp.Digest))
 		if err != nil {
 			return nil, err
 		}
 
-		vulnerabilities[rp.MimeType] = vrp
+		if v, ok := vulnerabilities[rp.MimeType]; ok {
+			r, err := report.Merge(rp.MimeType, v, vrp)
+			if err != nil {
+				return nil, err
+			}
+			vulnerabilities[rp.MimeType] = r
+		} else {
+			vulnerabilities[rp.MimeType] = vrp
+		}
 	}
 
 	content, _ := json.Marshal(vulnerabilities)

src/server/v2.0/handler/util_test.go

@@ -40,7 +40,7 @@ func Test_unescapePathParams(t *testing.T) {
 		{"non struct", args{&str, []string{"RepositoryName"}}, true},
 		{"ptr of struct", args{&Params{}, []string{"RepositoryName"}}, false},
 		{"non string filed", args{&Params{}, []string{"ProjectID"}}, false},
-		{"filed not found", args{&Params{}, []string{"Name"}}, false},
+		{"field not found", args{&Params{}, []string{"Name"}}, false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {