fix(p2p-preheat):fix issues of triggering preheat

- fix invalid data type of vulnerability filter param
- add more debug logs
- add more logs in the preheat job
- fix issue of getting empty list when doing querying artifacts

Signed-off-by: Steven Zou <szou@vmware.com>
This commit is contained in:
Steven Zou 2020-07-26 13:47:58 +08:00
parent f73cd6fc0e
commit 716da7f3ff
5 changed files with 82 additions and 8 deletions

View File

@ -326,8 +326,8 @@ func (de *defaultEnforcer) getCandidates(ctx context.Context, ps *pol.Schema, p
// Only get the image type at this moment.
arts, err := de.artCtl.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"project_id": ps.ProjectID,
"type": pr.SupportedType,
"ProjectID": ps.ProjectID,
"Type": strings.ToUpper(pr.SupportedType),
},
}, &artifact.Option{
WithLabel: true,
@ -340,6 +340,8 @@ func (de *defaultEnforcer) getCandidates(ctx context.Context, ps *pol.Schema, p
return nil, err
}
log.Debugf("Default enforcer: get [%d] candidates for preheat policy %s", len(arts), ps.Name)
return de.toCandidates(ctx, p, arts)
}

View File

@ -76,7 +76,7 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
// Print related info to log first
myLogger.Infof(
"Preheating image '%s:%s' to the target preheat provider: %s %s:%s",
"Preheating image '%s:%s' to the target preheat provider: %s %s:%s\n",
pi.ImageName,
pi.Tag,
p.Vendor,
@ -163,6 +163,8 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
return preheatJobRunningError(err)
}
myLogger.Infof("Check preheating progress: %#v", s)
// Finished
if s.Status == provider.PreheatingStatusSuccess {
myLogger.Info("Preheating is completed")

View File

@ -15,9 +15,11 @@
package policy
import (
"reflect"
"sort"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/selector"
"github.com/goharbor/harbor/src/lib/selector/selectors/doublestar"
"github.com/goharbor/harbor/src/lib/selector/selectors/label"
@ -64,12 +66,16 @@ func (df *defaultFilter) Filter(candidates []*selector.Candidate) ([]*selector.C
)
// Do filters
for _, sl := range df.selectors {
for i, sl := range df.selectors {
log.Debugf("Preheat filter[%d] input: [%d] candidates", i, len(filtered))
filtered, err = sl.Select(filtered)
if err != nil {
return nil, errors.Wrap(err, "do filter error")
}
log.Debugf("Preheat filter[%d] output: [%d] candidates", i, len(filtered))
if len(filtered) == 0 {
// Return earlier
return filtered, nil
@ -99,7 +105,9 @@ func (df *defaultFilter) BuildFrom(pl *policy.Schema) Filter {
df.selectors = make([]selector.Selector, 0)
}
for _, fl := range filters {
for i, fl := range filters {
log.Debugf("Build preheat filter[%d]: type=%s, value=%v", i, fl.Type, fl.Value)
sl, err := buildFilter(fl)
if err != nil {
df.error = errors.Wrap(err, "build filter error")
@ -115,6 +123,7 @@ func (df *defaultFilter) BuildFrom(pl *policy.Schema) Filter {
}
// Assign the filter with different order weight and then do filters with fixed order.
// Keep consistent with variable "orderedFilters".
func filterOrder(t policy.FilterType) uint {
switch t {
case policy.FilterTypeRepository:
@ -145,21 +154,24 @@ func buildFilter(f *policy.Filter) (selector.Selector, error) {
return nil, errors.Errorf("pattern value is missing for filter: %s", f.Type)
}
// Current value type
cvt := reflect.TypeOf(f.Value).Name()
// Check value type
switch f.Type {
case policy.FilterTypeRepository,
policy.FilterTypeTag,
policy.FilterTypeLabel:
if _, ok := f.Value.(string); !ok {
return nil, errors.Errorf("invalid string pattern format for filter: %s", f.Type)
return nil, errors.Errorf("invalid string pattern format(%s) for filter: %s", cvt, f.Type)
}
case policy.FilterTypeSignature:
if _, ok := f.Value.(bool); !ok {
return nil, errors.Errorf("invalid boolean pattern format for filter: %s", f.Type)
return nil, errors.Errorf("invalid boolean pattern format(%s) for filter: %s", cvt, f.Type)
}
case policy.FilterTypeVulnerability:
if _, ok := f.Value.(int); !ok {
return nil, errors.Errorf("invalid integer pattern format for filter: %s", f.Type)
return nil, errors.Errorf("invalid integer pattern format(%s) for filter: %s", cvt, f.Type)
}
}

View File

@ -187,3 +187,43 @@ func (suite *FilterTestSuite) TestFilters() {
require.Equal(suite.T(), 1, len(res), "number of matched candidates")
suite.Equal("sha256@fake", res[0].Digest, "digest of matched candidate")
}
// TestDefaultPatterns tests the case of using the default filter pattern.
func (suite *FilterTestSuite) TestDefaultPatterns() {
p := &policy.Schema{
Filters: []*policy.Filter{
{
Type: policy.FilterTypeRepository,
Value: "**",
},
{
Type: policy.FilterTypeTag,
Value: "**",
},
},
}
res, err := NewFilter().BuildFrom(p).Filter(suite.candidates)
require.NoError(suite.T(), err, "do filters")
require.Equal(suite.T(), 7, len(res), "number of matched candidates")
}
// TestDefaultPatterns2 tests the case of using the default filter pattern.
func (suite *FilterTestSuite) TestDefaultPatterns2() {
p := &policy.Schema{
Filters: []*policy.Filter{
{
Type: policy.FilterTypeRepository,
Value: "**",
},
{
Type: policy.FilterTypeTag,
Value: "*",
},
},
}
res, err := NewFilter().BuildFrom(p).Filter(suite.candidates)
require.NoError(suite.T(), err, "do filters")
require.Equal(suite.T(), 7, len(res), "number of matched candidates")
}

View File

@ -17,6 +17,7 @@ package policy
import (
"context"
"encoding/json"
"strconv"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/q"
@ -165,6 +166,23 @@ func parseFilters(filterStr string) ([]*policy.Filter, error) {
return nil, err
}
// Convert value type
// TODO: remove switch after UI bug #12579 fixed
for _, f := range filters {
if f.Type == policy.FilterTypeVulnerability {
switch f.Value.(type) {
case string:
sev, err := strconv.ParseInt(f.Value.(string), 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "parse filters")
}
f.Value = (int)(sev)
case float64:
f.Value = (int)(f.Value.(float64))
}
}
}
return filters, nil
}