fix(p2p-preheat):fix issues of triggering preheat

- fix invalid data type of vulnerability filter param
- add more debug logs
- add more logs in the preheat job
- fix issue of getting empty list when doing querying artifacts

Signed-off-by: Steven Zou <szou@vmware.com>

src/controller/p2p/preheat/enforcer.go

@@ -326,8 +326,8 @@ func (de *defaultEnforcer) getCandidates(ctx context.Context, ps *pol.Schema, p
 	// Only get the image type at this moment.
 	arts, err := de.artCtl.List(ctx, &q.Query{
 		Keywords: map[string]interface{}{
-			"project_id": ps.ProjectID,
-			"type":       pr.SupportedType,
+			"ProjectID": ps.ProjectID,
+			"Type":      strings.ToUpper(pr.SupportedType),
 		},
 	}, &artifact.Option{
 		WithLabel: true,
@@ -340,6 +340,8 @@ func (de *defaultEnforcer) getCandidates(ctx context.Context, ps *pol.Schema, p
 		return nil, err
 	}
 
+	log.Debugf("Default enforcer: get [%d] candidates for preheat policy %s", len(arts), ps.Name)
+
 	return de.toCandidates(ctx, p, arts)
 }
 

src/pkg/p2p/preheat/job.go

@@ -76,7 +76,7 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
 
 	// Print related info to log first
 	myLogger.Infof(
-		"Preheating image '%s:%s' to the target preheat provider: %s %s:%s",
+		"Preheating image '%s:%s' to the target preheat provider: %s %s:%s\n",
 		pi.ImageName,
 		pi.Tag,
 		p.Vendor,
@@ -163,6 +163,8 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
 				return preheatJobRunningError(err)
 			}
 
+			myLogger.Infof("Check preheating progress: %#v", s)
+
 			// Finished
 			if s.Status == provider.PreheatingStatusSuccess {
 				myLogger.Info("Preheating is completed")

src/pkg/p2p/preheat/policy/filter.go

@@ -15,9 +15,11 @@
 package policy
 
 import (
+	"reflect"
 	"sort"
 
 	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/log"
 	"github.com/goharbor/harbor/src/lib/selector"
 	"github.com/goharbor/harbor/src/lib/selector/selectors/doublestar"
 	"github.com/goharbor/harbor/src/lib/selector/selectors/label"
@@ -64,12 +66,16 @@ func (df *defaultFilter) Filter(candidates []*selector.Candidate) ([]*selector.C
 	)
 
 	// Do filters
-	for _, sl := range df.selectors {
+	for i, sl := range df.selectors {
+		log.Debugf("Preheat filter[%d] input: [%d] candidates", i, len(filtered))
+
 		filtered, err = sl.Select(filtered)
 		if err != nil {
 			return nil, errors.Wrap(err, "do filter error")
 		}
 
+		log.Debugf("Preheat filter[%d] output: [%d] candidates", i, len(filtered))
+
 		if len(filtered) == 0 {
 			// Return earlier
 			return filtered, nil
@@ -99,7 +105,9 @@ func (df *defaultFilter) BuildFrom(pl *policy.Schema) Filter {
 			df.selectors = make([]selector.Selector, 0)
 		}
 
-		for _, fl := range filters {
+		for i, fl := range filters {
+			log.Debugf("Build preheat filter[%d]: type=%s, value=%v", i, fl.Type, fl.Value)
+
 			sl, err := buildFilter(fl)
 			if err != nil {
 				df.error = errors.Wrap(err, "build filter error")
@@ -115,6 +123,7 @@ func (df *defaultFilter) BuildFrom(pl *policy.Schema) Filter {
 }
 
 // Assign the filter with different order weight and then do filters with fixed order.
+// Keep consistent with variable "orderedFilters".
 func filterOrder(t policy.FilterType) uint {
 	switch t {
 	case policy.FilterTypeRepository:
@@ -145,21 +154,24 @@ func buildFilter(f *policy.Filter) (selector.Selector, error) {
 		return nil, errors.Errorf("pattern value is missing for filter: %s", f.Type)
 	}
 
+	// Current value type
+	cvt := reflect.TypeOf(f.Value).Name()
+
 	// Check value type
 	switch f.Type {
 	case policy.FilterTypeRepository,
 		policy.FilterTypeTag,
 		policy.FilterTypeLabel:
 		if _, ok := f.Value.(string); !ok {
-			return nil, errors.Errorf("invalid string pattern format for filter: %s", f.Type)
+			return nil, errors.Errorf("invalid string pattern format(%s) for filter: %s", cvt, f.Type)
 		}
 	case policy.FilterTypeSignature:
 		if _, ok := f.Value.(bool); !ok {
-			return nil, errors.Errorf("invalid boolean pattern format for filter: %s", f.Type)
+			return nil, errors.Errorf("invalid boolean pattern format(%s) for filter: %s", cvt, f.Type)
 		}
 	case policy.FilterTypeVulnerability:
 		if _, ok := f.Value.(int); !ok {
-			return nil, errors.Errorf("invalid integer pattern format for filter: %s", f.Type)
+			return nil, errors.Errorf("invalid integer pattern format(%s) for filter: %s", cvt, f.Type)
 		}
 	}
 

src/pkg/p2p/preheat/policy/filter_test.go

@@ -187,3 +187,43 @@ func (suite *FilterTestSuite) TestFilters() {
 	require.Equal(suite.T(), 1, len(res), "number of matched candidates")
 	suite.Equal("sha256@fake", res[0].Digest, "digest of matched candidate")
 }
+
+// TestDefaultPatterns tests the case of using the default filter pattern.
+func (suite *FilterTestSuite) TestDefaultPatterns() {
+	p := &policy.Schema{
+		Filters: []*policy.Filter{
+			{
+				Type:  policy.FilterTypeRepository,
+				Value: "**",
+			},
+			{
+				Type:  policy.FilterTypeTag,
+				Value: "**",
+			},
+		},
+	}
+
+	res, err := NewFilter().BuildFrom(p).Filter(suite.candidates)
+	require.NoError(suite.T(), err, "do filters")
+	require.Equal(suite.T(), 7, len(res), "number of matched candidates")
+}
+
+// TestDefaultPatterns2 tests the case of using the default filter pattern.
+func (suite *FilterTestSuite) TestDefaultPatterns2() {
+	p := &policy.Schema{
+		Filters: []*policy.Filter{
+			{
+				Type:  policy.FilterTypeRepository,
+				Value: "**",
+			},
+			{
+				Type:  policy.FilterTypeTag,
+				Value: "*",
+			},
+		},
+	}
+
+	res, err := NewFilter().BuildFrom(p).Filter(suite.candidates)
+	require.NoError(suite.T(), err, "do filters")
+	require.Equal(suite.T(), 7, len(res), "number of matched candidates")
+}

src/pkg/p2p/preheat/policy/manager.go

@@ -17,6 +17,7 @@ package policy
 import (
 	"context"
 	"encoding/json"
+	"strconv"
 
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/q"
@@ -165,6 +166,23 @@ func parseFilters(filterStr string) ([]*policy.Filter, error) {
 		return nil, err
 	}
 
+	// Convert value type
+	// TODO: remove switch after UI bug #12579 fixed
+	for _, f := range filters {
+		if f.Type == policy.FilterTypeVulnerability {
+			switch f.Value.(type) {
+			case string:
+				sev, err := strconv.ParseInt(f.Value.(string), 10, 32)
+				if err != nil {
+					return nil, errors.Wrapf(err, "parse filters")
+				}
+				f.Value = (int)(sev)
+			case float64:
+				f.Value = (int)(f.Value.(float64))
+			}
+		}
+	}
+
 	return filters, nil
 }