Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: add cache layer for artifact (#16739) 

Implement cache layer for resource artifact and define common
cache manager and workflow. Also add cache related options to
configuration yaml.

Signed-off-by: chlins <chenyuzh@vmware.com>
This commit is contained in:
chlins 2022-03-25 17:59:37 +08:00
parent fbff7ab5cb
commit 8c223135e7
30 changed files with 936 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
make/harbor.yml.tmpl
View File

@ -245,3 +245,8 @@ upload_purging:
# the interval of the purge operations
interval: 24h
dryrun: false
# Cache related config
cache:
enabled: false
expire_hours: 24

29
make/photon/prepare/models.py
View File

@ -7,6 +7,7 @@ from g import internal_tls_dir, DEFAULT_GID, DEFAULT_UID, PG_GID, PG_UID
from utils.misc import check_permission, owner_can_read, get_realpath, port_number_valid
from utils.cert import san_existed
class InternalTLS:
harbor_certs_filename = {
@ -137,8 +138,9 @@ class InternalTLS:
else:
os.chown(file, DEFAULT_UID, DEFAULT_GID)
class Metric:
def __init__(self, enabled: bool = False, port: int = 8080, path: str = "metrics" ):
def __init__(self, enabled: bool = False, port: int = 8080, path: str = "metrics"):
self.enabled = enabled
self.port = port
self.path = path
@ -166,6 +168,7 @@ class JaegerExporter:
if self.endpoint and self.agent_host:
raise Exception('Jaeger Colector Endpoint and Agent host both set, only can set one')
class OtelExporter:
def __init__(self, config: dict):
if not config:
@ -184,6 +187,7 @@ class OtelExporter:
if not self.url_path:
raise Exception('Trace url path not set')
class Trace:
def __init__(self, config: dict):
self.enabled = config.get('enabled') or False
@ -205,6 +209,7 @@ class Trace:
elif self.otel.enabled:
self.otel.validate()
class PurgeUpload:
def __init__(self, config: dict):
if not config:
@ -223,14 +228,30 @@ class PurgeUpload:
raise Exception('purge upload age should set with with nh, n is the number of hour')
# interval should larger than 2h
age = self.age[:-1]
if not age.isnumeric() or int(age) < 2 :
if not age.isnumeric() or int(age) < 2:
raise Exception('purge upload age should set with with nh, n is the number of hour and n should not be less than 2')
# interval should end with h
if not isinstance(self.interval, str) or not self.interval.endswith('h'):
raise Exception('purge upload interval should set with with nh, n is the number of hour')
# interval should larger than 2h
interval = self.interval[:-1]
if not interval.isnumeric() or int(interval) < 2 :
if not interval.isnumeric() or int(interval) < 2:
raise Exception('purge upload interval should set with with nh, n is the number of hour and n should not beless than 2')
return
class Cache:
def __init__(self, config: dict):
if not config:
self.enabled = False
self.enabled = config.get('enabled')
self.expire_hours = config.get('expire_hours')
def validate(self):
if not self.enabled:
return
if not self.expire_hours.isnumeric():
raise Exception('cache expire hours should be number')
return

5
make/photon/prepare/templates/core/env.jinja
View File

@ -84,3 +84,8 @@ TRACE_OTEL_TIMEOUT={{ trace.otel.timeout }}
TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
{% endif %}
{% endif %}
{% if cache.enabled %}
CACHE_ENABLED=true
CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
{% endif %}

28
make/photon/prepare/utils/configs.py
View File

@ -3,11 +3,13 @@ import os
import yaml
from urllib.parse import urlencode
from g import versions_file_path, host_root_dir, DEFAULT_UID, INTERNAL_NO_PROXY_DN
from models import InternalTLS, Metric, Trace, PurgeUpload
from models import InternalTLS, Metric, Trace, PurgeUpload, Cache
from utils.misc import generate_random_string, owner_can_read, other_can_read
default_db_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
default_db_max_open_conns = 0 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns
# NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
default_db_max_idle_conns = 2
# NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns
default_db_max_open_conns = 0
default_https_cert_path = '/your/certificate/path'
default_https_key_path = '/your/certificate/path'
@ -49,7 +51,8 @@ def validate(conf: dict, **kwargs):
raise Exception("Error: storage driver %s is not supported, only the following ones are supported: %s" % (
storage_provider_name, ",".join(valid_storage_drivers)))
storage_provider_config = conf.get("storage_provider_config") ## original is registry_storage_provider_config
# original is registry_storage_provider_config
storage_provider_config = conf.get("storage_provider_config")
if storage_provider_name != "filesystem":
if storage_provider_config == "":
raise Exception(
@ -78,10 +81,14 @@ def validate(conf: dict, **kwargs):
if conf.get('trace'):
conf['trace'].validate()
if conf.get('purge_upload'):
conf['purge_upload'].validate()
if conf.get('cache'):
conf['cache'].validate()
def parse_versions():
if not versions_file_path.is_file():
return {}
@ -168,7 +175,6 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
config_dict['notary_server_db_password'] = 'password'
config_dict['notary_server_db_sslmode'] = 'disable'
# Data path volume
config_dict['data_volume'] = configs['data_volume']
@ -214,9 +220,9 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
all_no_proxy |= set(no_proxy_config.split(','))
for proxy_component in proxy_components:
config_dict[proxy_component + '_http_proxy'] = proxy_config.get('http_proxy') or ''
config_dict[proxy_component + '_https_proxy'] = proxy_config.get('https_proxy') or ''
config_dict[proxy_component + '_no_proxy'] = ','.join(all_no_proxy)
config_dict[proxy_component + '_http_proxy'] = proxy_config.get('http_proxy') or ''
config_dict[proxy_component + '_https_proxy'] = proxy_config.get('https_proxy') or ''
config_dict[proxy_component + '_no_proxy'] = ','.join(all_no_proxy)
# Trivy configs, optional
trivy_configs = configs.get("trivy") or {}
@ -354,6 +360,10 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
purge_upload_config = configs.get('upload_purging')
config_dict['purge_upload'] = PurgeUpload(purge_upload_config or {})
# cache configs
cache_config = configs.get('cache')
config_dict['cache'] = Cache(cache_config or {})
return config_dict

9
src/common/const.go
View File

@ -196,4 +196,13 @@ const (
PullTimeUpdateDisable = "pull_time_update_disable"
// PullAuditLogDisable indicate if pull audit log is disable for pull request.
PullAuditLogDisable = "pull_audit_log_disable"
// Cache layer settings
// CacheEnabled indicate whether enable cache layer.
CacheEnabled = "cache_enabled"
// CacheExpireHours is the cache expiration time, unit is hour.
CacheExpireHours = "cache_expire_hours"
// DefaultCacheExpireHours is the default cache expire hours, default is
// 24h.
DefaultCacheExpireHours = 24
)

4
src/controller/artifact/abstractor.go
View File

@ -18,7 +18,9 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
@ -40,7 +42,7 @@ type Abstractor interface {
// NewAbstractor creates a new abstractor
func NewAbstractor() Abstractor {
return &abstractor{
artMgr: artifact.Mgr,
artMgr: pkg.ArtifactMgr,
blobMgr: blob.Mgr,
regCli: registry.Cli,
}

3
src/controller/artifact/controller.go
View File

@ -22,6 +22,7 @@ import (
"strings"
"time"
"github.com/goharbor/harbor/src/pkg"
accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model"
"github.com/goharbor/harbor/src/controller/artifact/processor/chart"
@ -116,7 +117,7 @@ func NewController() Controller {
return &controller{
tagCtl: tag.Ctl,
repoMgr: repository.Mgr,
artMgr: artifact.Mgr,
artMgr: pkg.ArtifactMgr,
artrashMgr: artifactrash.Mgr,
blobMgr: blob.Mgr,
sigMgr: signature.GetManager(),

11
src/controller/event/handler/internal/artifact_test.go
View File

@ -24,6 +24,7 @@ import (
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg"
"github.com/goharbor/harbor/src/pkg/artifact"
_ "github.com/goharbor/harbor/src/pkg/config/db"
repo "github.com/goharbor/harbor/src/pkg/repository"
@ -54,7 +55,7 @@ func (suite *ArtifactHandlerTestSuite) SetupSuite() {
suite.ctx = orm.NewContext(context.TODO(), beegoorm.NewOrm())
// mock artifact
_, err := artifact.Mgr.Create(suite.ctx, &artifact.Artifact{ID: 1, RepositoryID: 1})
_, err := pkg.ArtifactMgr.Create(suite.ctx, &artifact.Artifact{ID: 1, RepositoryID: 1})
suite.Nil(err)
// mock repository
_, err = repo.Mgr.Create(suite.ctx, &model.RepoRecord{RepositoryID: 1})
@ -70,7 +71,7 @@ func (suite *ArtifactHandlerTestSuite) TearDownSuite() {
err := tag.Mgr.Delete(suite.ctx, 1)
suite.Nil(err)
// delete artifact
err = artifact.Mgr.Delete(suite.ctx, 1)
err = pkg.ArtifactMgr.Delete(suite.ctx, 1)
suite.Nil(err)
// delete repository
err = repo.Mgr.Delete(suite.ctx, 1)
@ -107,7 +108,7 @@ func (suite *ArtifactHandlerTestSuite) TestOnPull() {
suite.Nil(err, "onPull should return nil")
// sync mode should update db immediately
// pull_time
art, err := artifact.Mgr.Get(suite.ctx, 1)
art, err := pkg.ArtifactMgr.Get(suite.ctx, 1)
suite.Nil(err)
suite.False(art.PullTime.IsZero(), "sync update pull_time")
lastPullTime := art.PullTime
@ -122,7 +123,7 @@ func (suite *ArtifactHandlerTestSuite) TestOnPull() {
suite.Nil(err, "onPull should return nil")
// async mode should not update db immediately
// pull_time
art, err = artifact.Mgr.Get(suite.ctx, 1)
art, err = pkg.ArtifactMgr.Get(suite.ctx, 1)
suite.Nil(err)
suite.Equal(lastPullTime, art.PullTime, "pull_time should not be updated immediately")
// pull_count
@ -131,7 +132,7 @@ func (suite *ArtifactHandlerTestSuite) TestOnPull() {
suite.Equal(int64(1), repository.PullCount, "pull_count should not be updated immediately")
// wait for db update
suite.Eventually(func() bool {
art, err = artifact.Mgr.Get(suite.ctx, 1)
art, err = pkg.ArtifactMgr.Get(suite.ctx, 1)
suite.Nil(err)
return art.PullTime.After(lastPullTime)
}, 3*asyncFlushDuration, asyncFlushDuration/2, "wait for pull_time async update")

4
src/controller/icon/controller.go
View File

@ -19,6 +19,7 @@ import (
"context"
"encoding/base64"
"image"
// import the gif format
_ "image/gif"
// import the jpeg format
@ -31,6 +32,7 @@ import (
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/icon"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/registry"
"github.com/nfnt/resize"
@ -83,7 +85,7 @@ type Controller interface {
// NewController creates a new instance of the icon controller
func NewController() Controller {
return &controller{
artMgr: artifact.Mgr,
artMgr: pkg.ArtifactMgr,
regCli: registry.Cli,
cache: sync.Map{},
}

3
src/controller/repository/controller.go
View File

@ -23,6 +23,7 @@ import (
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg"
art "github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/repository"
@ -61,7 +62,7 @@ func NewController() Controller {
return &controller{
proMgr: project.Mgr,
repoMgr: repository.Mgr,
artMgr: art.Mgr,
artMgr: pkg.ArtifactMgr,
artCtl: artifact.Ctl,
}
}

3
src/controller/tag/controller.go
View File

@ -24,6 +24,7 @@ import (
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/lib/selector"
"github.com/goharbor/harbor/src/pkg"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/immutable/match"
"github.com/goharbor/harbor/src/pkg/immutable/match/rule"
@ -61,7 +62,7 @@ type Controller interface {
func NewController() Controller {
return &controller{
tagMgr: tag.Mgr,
artMgr: artifact.Mgr,
artMgr: pkg.ArtifactMgr,
immutableMtr: rule.NewRuleMatcher(),
}
}

3
src/lib/cache/cache.go vendored
View File

@ -56,6 +56,9 @@ type Cache interface {
// Save cache the value by key
Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error
// Keys returns the key matched by prefixes
Keys(ctx context.Context, prefixes ...string) ([]string, error)
}
var (

24
src/lib/cache/memory/memory.go vendored
View File

@ -18,6 +18,7 @@ import (
"context"
"fmt"
"math"
"strings"
"sync"
"time"
@ -111,6 +112,29 @@ func (c *Cache) Save(ctx context.Context, key string, value interface{}, expirat
return nil
}
// Keys returns the key matched by prefixes.
func (c *Cache) Keys(ctx context.Context, prefixes ...string) ([]string, error) {
// if no prefix, means match all keys.
matchAll := len(prefixes) == 0
// range map to get all keys
keys := make([]string, 0)
c.storage.Range(func(k, v interface{}) bool {
ks := k.(string)
if matchAll {
keys = append(keys, ks)
} else {
for _, p := range prefixes {
if strings.HasPrefix(ks, c.opts.Key(p)) {
keys = append(keys, ks)
}
}
}
return true
})
return keys, nil
}
// New returns memory cache
func New(opts cache.Options) (cache.Cache, error) {
return &Cache{opts: &opts}, nil

24
src/lib/cache/memory/memory_test.go vendored
View File

@ -108,6 +108,30 @@ func (suite *CacheTestSuite) TestPing() {
suite.NoError(suite.cache.Ping(suite.ctx))
}
func (suite *CacheTestSuite) TestKeys() {
key1 := "p1"
key2 := "p2"
var err error
err = suite.cache.Save(suite.ctx, key1, "hello, p1")
suite.Nil(err)
err = suite.cache.Save(suite.ctx, key2, "hello, p2")
suite.Nil(err)
// should match all
keys, err := suite.cache.Keys(suite.ctx, "p")
suite.Nil(err)
suite.ElementsMatch([]string{"p1", "p2"}, keys)
// only get p1
keys, err = suite.cache.Keys(suite.ctx, key1)
suite.Nil(err)
suite.Equal([]string{"p1"}, keys)
// only get p2
keys, err = suite.cache.Keys(suite.ctx, key2)
suite.Nil(err)
suite.Equal([]string{"p2"}, keys)
}
func TestCacheTestSuite(t *testing.T) {
suite.Run(t, new(CacheTestSuite))
}

24
src/lib/cache/redis/redis.go vendored
View File

@ -87,6 +87,30 @@ func (c *Cache) Save(ctx context.Context, key string, value interface{}, expirat
return c.Client.Set(ctx, c.opts.Key(key), data, exp).Err()
}
// Keys returns the key matched by prefixes.
func (c *Cache) Keys(ctx context.Context, prefixes ...string) ([]string, error) {
patterns := make([]string, 0, len(prefixes))
if len(prefixes) == 0 {
patterns = append(patterns, "*")
} else {
for _, p := range prefixes {
patterns = append(patterns, c.opts.Key(p)+"*")
}
}
keys := make([]string, 0)
for _, pattern := range patterns {
cmd := c.Client.Keys(ctx, pattern)
if err := cmd.Err(); err != nil {
return nil, err
}
keys = append(keys, cmd.Val()...)
}
return keys, nil
}
// New returns redis cache
func New(opts cache.Options) (cache.Cache, error) {
if opts.Address == "" {

24
src/lib/cache/redis/redis_test.go vendored
View File

@ -109,6 +109,30 @@ func (suite *CacheTestSuite) TestPing() {
suite.NoError(suite.cache.Ping(suite.ctx))
}
func (suite *CacheTestSuite) TestKeys() {
key1 := "p1"
key2 := "p2"
var err error
err = suite.cache.Save(suite.ctx, key1, "hello, p1")
suite.Nil(err)
err = suite.cache.Save(suite.ctx, key2, "hello, p2")
suite.Nil(err)
// should match all
keys, err := suite.cache.Keys(suite.ctx, "p")
suite.Nil(err)
suite.ElementsMatch([]string{"p1", "p2"}, keys)
// only get p1
keys, err = suite.cache.Keys(suite.ctx, key1)
suite.Nil(err)
suite.Equal([]string{"p1"}, keys)
// only get p2
keys, err = suite.cache.Keys(suite.ctx, key2)
suite.Nil(err)
suite.Equal([]string{"p2"}, keys)
}
func TestCacheTestSuite(t *testing.T) {
suite.Run(t, new(CacheTestSuite))
}

3
src/lib/config/metadata/metadatalist.go
View File

@ -183,5 +183,8 @@ var (
{Name: common.PullTimeUpdateDisable, Scope: UserScope, Group: BasicGroup, EnvKey: "PULL_TIME_UPDATE_DISABLE", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The flag to indicate if pull time is disable for pull request.`},
{Name: common.PullCountUpdateDisable, Scope: UserScope, Group: BasicGroup, EnvKey: "PULL_COUNT_UPDATE_DISABLE", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The flag to indicate if pull count is disable for pull request.`},
{Name: common.PullAuditLogDisable, Scope: UserScope, Group: BasicGroup, EnvKey: "PULL_AUDIT_LOG_DISABLE", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `The flag to indicate if pull audit log is disable for pull request.`},
{Name: common.CacheEnabled, Scope: SystemScope, Group: BasicGroup, EnvKey: "CACHE_ENABLED", DefaultValue: "false", ItemType: &BoolType{}, Editable: false, Description: `Enable cache`},
{Name: common.CacheExpireHours, Scope: SystemScope, Group: BasicGroup, EnvKey: "CACHE_EXPIRE_HOURS", DefaultValue: "24", ItemType: &IntType{}, Editable: false, Description: `The expire hours for cache`},
}
)

31
src/lib/config/systemconfig.go
View File

@ -31,14 +31,15 @@ package config
import (
"context"
"errors"
"os"
"strconv"
"strings"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/secret"
"github.com/goharbor/harbor/src/lib/encrypt"
"github.com/goharbor/harbor/src/lib/log"
"os"
"strconv"
"strings"
)
var (
@ -244,6 +245,30 @@ func InitialAdminPassword() (string, error) {
return DefaultMgr().Get(backgroundCtx, common.AdminInitialPassword).GetString(), nil
}
// CacheEnabled returns whether enable cache layer.
func CacheEnabled() bool {
if DefaultMgr() != nil {
return DefaultMgr().Get(backgroundCtx, common.CacheEnabled).GetBool()
}
// backoff read from env.
return os.Getenv("CACHE_ENABLED") == "true"
}
// CacheExpireHours returns the cache expire hours for cache layer.
func CacheExpireHours() int {
if DefaultMgr() != nil {
return DefaultMgr().Get(backgroundCtx, common.CacheExpireHours).GetInt()
}
// backoff read from env.
hours, err := strconv.Atoi(os.Getenv("CACHE_EXPIRE_HOURS"))
if err != nil {
// use default if parse error.
hours = common.DefaultCacheExpireHours
}
return hours
}
// Database returns database settings
func Database() (*models.Database, error) {
database := &models.Database{}

7
src/migration/artifact.go
View File

@ -20,6 +20,7 @@ import (
art "github.com/goharbor/harbor/src/controller/artifact"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/repository"
@ -43,7 +44,7 @@ func abstractArtData(ctx context.Context) error {
}
for _, repo := range repos {
log.Infof("abstracting artifact metadata under repository %s ....", repo.Name)
arts, err := artifact.Mgr.List(ctx, &q.Query{
arts, err := pkg.ArtifactMgr.List(ctx, &q.Query{
Keywords: map[string]interface{}{
"RepositoryID": repo.RepositoryID,
},
@ -57,7 +58,7 @@ func abstractArtData(ctx context.Context) error {
log.Errorf("failed to abstract the artifact %s@%s: %v, skip", a.RepositoryName, a.Digest, err)
continue
}
if err = artifact.Mgr.Update(ctx, a); err != nil {
if err = pkg.ArtifactMgr.Update(ctx, a); err != nil {
log.Errorf("failed to update the artifact %s@%s: %v, skip", repo.Name, a.Digest, err)
continue
}
@ -73,7 +74,7 @@ func abstractArtData(ctx context.Context) error {
func abstract(ctx context.Context, abstractor art.Abstractor, art *artifact.Artifact) error {
// abstract the children
for _, reference := range art.References {
child, err := artifact.Mgr.Get(ctx, reference.ChildID)
child, err := pkg.ArtifactMgr.Get(ctx, reference.ChildID)
if err != nil {
log.Errorf("failed to get the artifact %d: %v, skip", reference.ChildID, err)
continue

5
src/pkg/artifact/manager.go
View File

@ -22,11 +22,6 @@ import (
"github.com/goharbor/harbor/src/pkg/artifact/dao"
)
var (
// Mgr is a global artifact manager instance
Mgr = NewManager()
)
// Manager is the only interface of artifact module to provide the management functions for artifacts
type Manager interface {
// Count returns the total count of artifacts according to the query.

257
src/pkg/cached/artifact/redis/manager.go Normal file
View File

@ -0,0 +1,257 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package redis
import (
"context"
"time"
libcache "github.com/goharbor/harbor/src/lib/cache"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/lib/retry"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/cached"
)
var _ CachedManager = &manager{}
// CachedManager is the interface combines raw resource manager and cached manager for better extension.
type CachedManager interface {
// Manager is the raw resource manager.
artifact.Manager
// Manager is the common interface for resource cache.
cached.Manager
}
// manager is the cached manager implemented by redis.
type manager struct {
// delegator delegates the raw crud to DAO.
delegator artifact.Manager
// client returns the redis cache client.
client func() libcache.Cache
// keyBuilder builds cache object key.
keyBuilder *cached.ObjectKey
// lifetime is the cache life time.
lifetime time.Duration
}
// NewManager returns the redis cache manager.
func NewManager(m artifact.Manager) *manager {
return &manager{
delegator: m,
client: func() libcache.Cache { return libcache.Default() },
keyBuilder: cached.NewObjectKey(cached.ResourceTypeArtifact),
lifetime: time.Duration(config.CacheExpireHours()) * time.Hour,
}
}
func (m *manager) Count(ctx context.Context, query *q.Query) (int64, error) {
return m.delegator.Count(ctx, query)
}
func (m *manager) List(ctx context.Context, query *q.Query) ([]*artifact.Artifact, error) {
return m.delegator.List(ctx, query)
}
func (m *manager) Create(ctx context.Context, artifact *artifact.Artifact) (int64, error) {
return m.delegator.Create(ctx, artifact)
}
func (m *manager) ListReferences(ctx context.Context, query *q.Query) ([]*artifact.Reference, error) {
return m.delegator.ListReferences(ctx, query)
}
func (m *manager) DeleteReference(ctx context.Context, id int64) error {
return m.delegator.DeleteReference(ctx, id)
}
func (m *manager) Get(ctx context.Context, id int64) (*artifact.Artifact, error) {
key, err := m.keyBuilder.Format("id", id)
if err != nil {
return nil, err
}
art := &artifact.Artifact{}
if err = m.client().Fetch(ctx, key, art); err == nil {
return art, nil
}
log.Debugf("get artifact %d from cache error: %v, will query from database.", id, err)
art, err = m.delegator.Get(ctx, id)
if err != nil {
return nil, err
}
if err = m.client().Save(ctx, key, art, m.lifetime); err != nil {
// log error if save to cache failed
log.Debugf("save artifact %s to cache error: %v", art.String(), err)
}
return art, nil
}
func (m *manager) GetByDigest(ctx context.Context, repository, digest string) (*artifact.Artifact, error) {
key, err := m.keyBuilder.Format("digest", digest)
if err != nil {
return nil, err
}
art := &artifact.Artifact{}
if err = m.client().Fetch(ctx, key, art); err == nil {
return art, nil
}
art, err = m.delegator.GetByDigest(ctx, repository, digest)
if err != nil {
return nil, err
}
if err = m.client().Save(ctx, key, art, m.lifetime); err != nil {
// log error if save to cache failed
log.Debugf("save artifact %s to cache error: %v", art.String(), err)
}
return art, nil
}
func (m *manager) Delete(ctx context.Context, id int64) error {
art, err := m.Get(ctx, id)
if err != nil {
return err
}
// pass on delete operation
if err := m.delegator.Delete(ctx, id); err != nil {
return err
}
// clean cache
m.cleanUp(ctx, art)
return nil
}
func (m *manager) Update(ctx context.Context, artifact *artifact.Artifact, props ...string) error {
// pass on update operation
if err := m.delegator.Update(ctx, artifact, props...); err != nil {
return err
}
// clean cache
m.cleanUp(ctx, artifact)
return nil
}
func (m *manager) UpdatePullTime(ctx context.Context, id int64, pullTime time.Time) error {
art, err := m.Get(ctx, id)
if err != nil {
return err
}
// pass on updatePullTime operation
if err = m.delegator.UpdatePullTime(ctx, id, pullTime); err != nil {
return err
}
// refresh cache
m.refreshCache(ctx, art)
return nil
}
// cleanUp cleans up data in cache.
func (m *manager) cleanUp(ctx context.Context, art *artifact.Artifact) {
// clean index by id
idIdx, err := m.keyBuilder.Format("id", art.ID)
if err != nil {
log.Errorf("format artifact id key error: %v", err)
} else {
// retry to avoid dirty data
if err = retry.Retry(func() error { return m.client().Delete(ctx, idIdx) }); err != nil {
log.Errorf("delete artifact cache key %s error: %v", idIdx, err)
}
}
// clean index by digest
digestIdx, err := m.keyBuilder.Format("digest", art.Digest)
if err != nil {
log.Errorf("format artifact digest key error: %v", err)
} else {
if err = retry.Retry(func() error { return m.client().Delete(ctx, digestIdx) }); err != nil {
log.Errorf("delete artifact cache key %s error: %v", digestIdx, err)
}
}
}
// refreshCache refreshes cache.
func (m *manager) refreshCache(ctx context.Context, art *artifact.Artifact) {
// refreshCache used for UpdatePullTime, because we have a background goroutine to
// update per artifact pull_time in period time, in that case, we don't want to lose
// cache every fixed interval, so prefer to use refreshCache instead of cleanUp.
// no need to consider lock because we only have one goroutine do this work one by one.
// refreshCache includes 2 steps:
// 1. cleanUp
// 2. re-get
m.cleanUp(ctx, art)
var err error
// re-get by id
_, err = m.Get(ctx, art.ID)
if err != nil {
log.Errorf("refresh cache by artifact id %d error: %v", art.ID, err)
}
// re-get by digest
_, err = m.GetByDigest(ctx, art.RepositoryName, art.Digest)
if err != nil {
log.Errorf("refresh cache by artifact digest %s error: %v", art.Digest, err)
}
}
func (m *manager) ResourceType(ctx context.Context) string {
return cached.ResourceTypeArtifact
}
func (m *manager) CountCache(ctx context.Context) (int64, error) {
// prefix is resource type
keys, err := m.client().Keys(ctx, m.ResourceType(ctx))
if err != nil {
return 0, err
}
return int64(len(keys)), nil
}
func (m *manager) DeleteCache(ctx context.Context, key string) error {
return m.client().Delete(ctx, key)
}
func (m *manager) FlushAll(ctx context.Context) error {
// prefix is resource type
keys, err := m.client().Keys(ctx, m.ResourceType(ctx))
if err != nil {
return err
}
var errs errors.Errors
for _, key := range keys {
if err = m.client().Delete(ctx, key); err != nil {
errs = append(errs, err)
}
}
if errs.Len() > 0 {
return errs
}
return nil
}

202
src/pkg/cached/artifact/redis/manager_test.go Normal file
View File

@ -0,0 +1,202 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package redis
import (
"context"
"errors"
"testing"
"time"
"github.com/goharbor/harbor/src/lib/cache"
"github.com/goharbor/harbor/src/pkg/artifact"
testcache "github.com/goharbor/harbor/src/testing/lib/cache"
"github.com/goharbor/harbor/src/testing/mock"
testArt "github.com/goharbor/harbor/src/testing/pkg/artifact"
"github.com/stretchr/testify/suite"
)
type managerTestSuite struct {
suite.Suite
cachedManager CachedManager
artMgr *testArt.Manager
cache *testcache.Cache
ctx context.Context
}
func (m *managerTestSuite) SetupTest() {
m.artMgr = &testArt.Manager{}
m.cache = &testcache.Cache{}
m.cachedManager = NewManager(
m.artMgr,
)
m.cachedManager.(*manager).client = func() cache.Cache { return m.cache }
m.ctx = context.TODO()
}
func (m *managerTestSuite) TestGet() {
// get from cache directly
m.cache.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
_, err := m.cachedManager.Get(m.ctx, 100)
m.NoError(err, "should get from cache")
m.artMgr.AssertNotCalled(m.T(), "Get", mock.Anything, mock.Anything)
// not found in cache, read from dao
m.cache.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(cache.ErrNotFound).Once()
m.cache.On("Save", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
m.artMgr.On("Get", mock.Anything, mock.Anything).Return(&artifact.Artifact{}, nil).Once()
_, err = m.cachedManager.Get(m.ctx, 100)
m.NoError(err, "should get from artMgr")
m.artMgr.AssertCalled(m.T(), "Get", mock.Anything, mock.Anything)
}
func (m *managerTestSuite) TestGetByDigest() {
// get from cache directly
m.cache.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
_, err := m.cachedManager.GetByDigest(m.ctx, "repo", "sha256:418fb88ec412e340cdbef913b8ca1bbe8f9e8dc705f9617414c1f2c8db980180")
m.NoError(err, "should get from cache")
m.artMgr.AssertNotCalled(m.T(), "Get", mock.Anything, mock.Anything)
// not found in cache, read from dao
m.cache.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(cache.ErrNotFound).Once()
m.cache.On("Save", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
m.artMgr.On("Get", mock.Anything, mock.Anything).Return(&artifact.Artifact{}, nil).Once()
_, err = m.cachedManager.Get(m.ctx, 100)
m.NoError(err, "should get from artMgr")
m.artMgr.AssertCalled(m.T(), "Get", mock.Anything, mock.Anything)
}
func (m *managerTestSuite) TestDelete() {
// delete from artMgr error
errDelete := errors.New("delete failed")
m.artMgr.On("Delete", mock.Anything, mock.Anything).Return(errDelete).Once()
m.artMgr.On("DeleteReferences", mock.Anything, mock.Anything).Return(nil).Once()
m.cache.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
err := m.cachedManager.Delete(m.ctx, 100)
m.ErrorIs(err, errDelete, "delete should error")
m.cache.AssertNotCalled(m.T(), "Delete", mock.Anything, mock.Anything)
// delete from artMgr success
m.artMgr.On("Delete", mock.Anything, mock.Anything).Return(nil).Once()
m.artMgr.On("DeleteReferences", mock.Anything, mock.Anything).Return(nil).Once()
m.cache.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
m.cache.On("Delete", mock.Anything, mock.Anything).Return(nil).Twice()
err = m.cachedManager.Delete(m.ctx, 100)
m.NoError(err, "delete should success")
m.cache.AssertCalled(m.T(), "Delete", mock.Anything, mock.Anything)
}
func (m *managerTestSuite) TestUpdate() {
// update from artMgr error
errUpdate := errors.New("update failed")
m.artMgr.On("Update", mock.Anything, mock.Anything).Return(errUpdate).Once()
err := m.cachedManager.Update(m.ctx, &artifact.Artifact{})
m.ErrorIs(err, errUpdate, "update should error")
m.cache.AssertNotCalled(m.T(), "Delete", mock.Anything, mock.Anything)
// update from artMgr success
m.artMgr.On("Update", mock.Anything, mock.Anything).Return(nil).Once()
m.cache.On("Delete", mock.Anything, mock.Anything).Return(nil).Twice()
err = m.cachedManager.Update(m.ctx, &artifact.Artifact{})
m.NoError(err, "update should success")
m.cache.AssertCalled(m.T(), "Delete", mock.Anything, mock.Anything)
}
func (m *managerTestSuite) TestUpdatePullTime() {
// update pull time from artMgr error
errUpdate := errors.New("update pull time failed")
m.artMgr.On("UpdatePullTime", mock.Anything, mock.Anything, mock.Anything).Return(errUpdate).Once()
m.cache.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(nil)
err := m.cachedManager.UpdatePullTime(m.ctx, 100, time.Now())
m.ErrorIs(err, errUpdate, "update pull time should error")
m.cache.AssertNotCalled(m.T(), "Delete", mock.Anything, mock.Anything)
// update pull time from artMgr success
m.artMgr.On("UpdatePullTime", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
m.cache.On("Delete", mock.Anything, mock.Anything).Return(nil).Twice()
err = m.cachedManager.UpdatePullTime(m.ctx, 100, time.Now())
m.NoError(err, "update pull time should success")
m.cache.AssertCalled(m.T(), "Delete", mock.Anything, mock.Anything)
}
func (m *managerTestSuite) TestCount() {
m.artMgr.On("Count", mock.Anything, mock.Anything).Return(int64(1), nil)
c, err := m.cachedManager.Count(m.ctx, nil)
m.NoError(err)
m.Equal(int64(1), c)
m.artMgr.AssertCalled(m.T(), "Count", mock.Anything, mock.Anything)
}
func (m *managerTestSuite) TestList() {
arts := []*artifact.Artifact{}
m.artMgr.On("List", mock.Anything, mock.Anything).Return(arts, nil)
as, err := m.cachedManager.List(m.ctx, nil)
m.NoError(err)
m.Equal(arts, as)
m.artMgr.AssertCalled(m.T(), "List", mock.Anything, mock.Anything)
}
func (m *managerTestSuite) TestCreate() {
m.artMgr.On("Create", mock.Anything, mock.Anything).Return(int64(1), nil)
id, err := m.cachedManager.Create(m.ctx, nil)
m.NoError(err)
m.Equal(int64(1), id)
m.artMgr.AssertCalled(m.T(), "Create", mock.Anything, mock.Anything)
}
func (m *managerTestSuite) TestListReferences() {
refs := []*artifact.Reference{}
m.artMgr.On("ListReferences", mock.Anything, mock.Anything).Return(refs, nil)
rs, err := m.cachedManager.ListReferences(m.ctx, nil)
m.NoError(err)
m.Equal(refs, rs)
m.artMgr.AssertCalled(m.T(), "ListReferences", mock.Anything, mock.Anything)
}
func (m *managerTestSuite) TestDeleteReference() {
m.artMgr.On("DeleteReference", mock.Anything, mock.Anything).Return(nil)
err := m.cachedManager.DeleteReference(m.ctx, 1)
m.NoError(err)
m.artMgr.AssertCalled(m.T(), "DeleteReference", mock.Anything, mock.Anything)
}
func (m *managerTestSuite) TestResourceType() {
t := m.cachedManager.ResourceType(m.ctx)
m.Equal("artifact", t)
}
func (m *managerTestSuite) TestCountCache() {
m.cache.On("Keys", mock.Anything, mock.Anything).Return([]string{"1"}, nil).Once()
c, err := m.cachedManager.CountCache(m.ctx)
m.NoError(err)
m.Equal(int64(1), c)
}
func (m *managerTestSuite) TestDeleteCache() {
m.cache.On("Delete", mock.Anything, mock.Anything).Return(nil).Once()
err := m.cachedManager.DeleteCache(m.ctx, "key")
m.NoError(err)
}
func (m *managerTestSuite) TestFlushAll() {
m.cache.On("Keys", mock.Anything, mock.Anything).Return([]string{"1"}, nil).Once()
m.cache.On("Delete", mock.Anything, mock.Anything).Return(nil).Once()
err := m.cachedManager.FlushAll(m.ctx)
m.NoError(err)
}
func TestManager(t *testing.T) {
suite.Run(t, &managerTestSuite{})
}

89
src/pkg/cached/manager.go Normal file
View File

@ -0,0 +1,89 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cached
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/lib/errors"
)
const (
// Resource type definitions
// ResourceTypeArtifact defines artifact type.
ResourceTypeArtifact = "artifact"
)
// Manager is the interface for resource cache manager.
// Provides common interfaces for admin to view and manage resource cache.
type Manager interface {
// ResourceType returns the resource type.
// eg. artifact、project、tag、repository
ResourceType(ctx context.Context) string
// CountCache returns current this resource occupied cache count.
CountCache(ctx context.Context) (int64, error)
// DeleteCache deletes specific cache by key.
DeleteCache(ctx context.Context, key string) error
// FlushAll flush this resource's all cache.
FlushAll(ctx context.Context) error
// TODO for more extensions like metrics.
}
// ObjectKey normalizes cache object key.
type ObjectKey struct {
// namespace as group or prefix, eg. artifact:id
namespace string
}
// NewObjectKey returns object key with namespace.
func NewObjectKey(ns string) *ObjectKey {
return &ObjectKey{namespace: ns}
}
// Format formats fields to string.
// eg. namespace: 'artifact'
// Format("id", 100, "digest", "aaa"): "artifact:id:100:digest:aaa"
func (ok *ObjectKey) Format(keysAndValues ...interface{}) (string, error) {
// keysAndValues must be paired.
if len(keysAndValues)%2 != 0 {
return "", errors.Errorf("invalid keysAndValues: %v", keysAndValues...)
}
s := ok.namespace
for i := 0; i < len(keysAndValues); i++ {
// even is key
if i%2 == 0 {
key, match := keysAndValues[i].(string)
if !match {
return "", errors.Errorf("key must be string, invalid key type: %#v", keysAndValues[i])
}
s += fmt.Sprintf(":%s", key)
} else {
switch keysAndValues[i].(type) {
case int, int16, int32, int64:
s += fmt.Sprintf(":%d", keysAndValues[i])
case string:
s += fmt.Sprintf(":%s", keysAndValues[i])
default:
return "", errors.Errorf("unsupported value type: %#v", keysAndValues[i])
}
}
}
return s, nil
}

36
src/pkg/cached/manager_test.go Normal file
View File

@ -0,0 +1,36 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cached
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestObjectKey(t *testing.T) {
ok := NewObjectKey("artifact")
// valid case
s, err := ok.Format("id", 100, "digest", "9834876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee8f0")
assert.NoError(t, err, "format should not error")
assert.Equal(t, "artifact:id:100:digest:9834876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee8f0", s)
// invalid case
_, err = ok.Format("id")
assert.Error(t, err, "invalid length should error")
_, err = ok.Format(1, 1)
assert.Error(t, err, "invalid key type should error")
_, err = ok.Format("id", struct{}{})
assert.Error(t, err, "invalid value type should error")
}

8
src/pkg/exporter/project_collector_test.go
View File

@ -1,11 +1,13 @@
package exporter
import (
proModels "github.com/goharbor/harbor/src/pkg/project/models"
"strconv"
"testing"
"time"
"github.com/goharbor/harbor/src/pkg"
proModels "github.com/goharbor/harbor/src/pkg/project/models"
"github.com/stretchr/testify/suite"
"github.com/goharbor/harbor/src/common"
@ -93,7 +95,7 @@ func setupTest(t *testing.T) {
art1.ProjectID = testPro1.ProjectID
art1.RepositoryID = repo1ID
art1.PushTime = time.Now()
_, err = artifact.Mgr.Create(ctx, &art1)
_, err = pkg.ArtifactMgr.Create(ctx, &art1)
if err != nil {
t.Errorf("add repo error %v", err)
}
@ -101,7 +103,7 @@ func setupTest(t *testing.T) {
art2.ProjectID = testPro2.ProjectID
art2.RepositoryID = repo2ID
art2.PushTime = time.Now()
_, err = artifact.Mgr.Create(ctx, &art2)
_, err = pkg.ArtifactMgr.Create(ctx, &art2)
if err != nil {
t.Errorf("add repo error %v", err)
}

49
src/pkg/factory.go Normal file
View File

@ -0,0 +1,49 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pkg
import (
"sync"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/cached/artifact/redis"
)
// Define global resource manager.
var (
// once for only init one time.
once sync.Once
// ArtifactMgr is the manager for artifact.
ArtifactMgr artifact.Manager
)
// init initialize mananger for resources
func init() {
once.Do(func() {
cacheEnabled := config.CacheEnabled()
initArtifactManager(cacheEnabled)
})
}
func initArtifactManager(cacheEnabled bool) {
artMgr := artifact.NewManager()
// check cache enable
if cacheEnabled {
ArtifactMgr = redis.NewManager(artMgr)
} else {
ArtifactMgr = artMgr
}
}

36
src/pkg/factory_test.go Normal file
View File

@ -0,0 +1,36 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pkg
import (
"testing"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/cached/artifact/redis"
"github.com/stretchr/testify/assert"
)
func TestInit(t *testing.T) {
// cache not enable
// artifact
assert.NotNil(t, ArtifactMgr)
assert.IsType(t, artifact.NewManager(), ArtifactMgr)
// cache enable
initArtifactManager(true)
// artifact
assert.NotNil(t, ArtifactMgr)
assert.IsType(t, redis.NewManager(artifact.NewManager()), ArtifactMgr)
}

18
src/server/middleware/cosign/cosign_test.go
View File

@ -2,9 +2,16 @@ package cosign
import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/goharbor/harbor/src/controller/repository"
"github.com/goharbor/harbor/src/lib"
"github.com/goharbor/harbor/src/lib/q"
"github.com/goharbor/harbor/src/pkg"
"github.com/goharbor/harbor/src/pkg/accessory"
"github.com/goharbor/harbor/src/pkg/accessory/model"
accessorymodel "github.com/goharbor/harbor/src/pkg/accessory/model"
@ -12,11 +19,6 @@ import (
"github.com/goharbor/harbor/src/pkg/distribution"
htesting "github.com/goharbor/harbor/src/testing"
"github.com/stretchr/testify/suite"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
type MiddlewareTestSuite struct {
@ -77,7 +79,7 @@ func (suite *MiddlewareTestSuite) addArt(pid, repositoryID int64, repositoryName
PushTime: time.Now(),
PullTime: time.Now(),
}
afid, err := artifact.Mgr.Create(suite.Context(), af)
afid, err := pkg.ArtifactMgr.Create(suite.Context(), af)
suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
return afid
}
@ -93,7 +95,7 @@ func (suite *MiddlewareTestSuite) addArtAcc(pid, repositoryID int64, repositoryN
PushTime: time.Now(),
PullTime: time.Now(),
}
subafid, err := artifact.Mgr.Create(suite.Context(), subaf)
subafid, err := pkg.ArtifactMgr.Create(suite.Context(), subaf)
suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
af := &artifact.Artifact{
@ -106,7 +108,7 @@ func (suite *MiddlewareTestSuite) addArtAcc(pid, repositoryID int64, repositoryN
PushTime: time.Now(),
PullTime: time.Now(),
}
afid, err := artifact.Mgr.Create(suite.Context(), af)
afid, err := pkg.ArtifactMgr.Create(suite.Context(), af)
suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
accid, err := accessory.Mgr.Create(suite.Context(), accessorymodel.AccessoryData{

12
src/server/middleware/immutable/pushmf_test.go
View File

@ -3,9 +3,6 @@ package immutable
import (
"context"
"fmt"
"github.com/goharbor/harbor/src/controller/immutable"
"github.com/goharbor/harbor/src/pkg/project"
proModels "github.com/goharbor/harbor/src/pkg/project/models"
"math/rand"
"net/http"
"net/http/httptest"
@ -13,6 +10,11 @@ import (
"testing"
"time"
"github.com/goharbor/harbor/src/controller/immutable"
"github.com/goharbor/harbor/src/pkg"
"github.com/goharbor/harbor/src/pkg/project"
proModels "github.com/goharbor/harbor/src/pkg/project/models"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/lib"
internal_orm "github.com/goharbor/harbor/src/lib/orm"
@ -92,7 +94,7 @@ func (suite *HandlerSuite) addArt(ctx context.Context, pid, repositoryID int64,
PushTime: time.Now(),
PullTime: time.Now(),
}
afid, err := artifact.Mgr.Create(ctx, af)
afid, err := pkg.ArtifactMgr.Create(ctx, af)
suite.Nil(err, fmt.Sprintf("Add artifact failed for %d", repositoryID))
return afid
}
@ -162,7 +164,7 @@ func (suite *HandlerSuite) TestPutDeleteManifestCreated() {
defer func() {
project.Mgr.Delete(ctx, projectID)
artifact.Mgr.Delete(ctx, afID)
pkg.ArtifactMgr.Delete(ctx, afID)
repository.Mgr.Delete(ctx, repoID)
tag.Mgr.Delete(ctx, tagID)
immutable.Ctr.DeleteImmutableRule(internal_orm.Context(), immuRuleID)

30
src/testing/lib/cache/cache.go vendored
View File

@ -56,6 +56,36 @@ func (_m *Cache) Fetch(ctx context.Context, key string, value interface{}) error
return r0
}
// Keys provides a mock function with given fields: ctx, prefixes
func (_m *Cache) Keys(ctx context.Context, prefixes ...string) ([]string, error) {
_va := make([]interface{}, len(prefixes))
for _i := range prefixes {
_va[_i] = prefixes[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
var r0 []string
if rf, ok := ret.Get(0).(func(context.Context, ...string) []string); ok {
r0 = rf(ctx, prefixes...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, ...string) error); ok {
r1 = rf(ctx, prefixes...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Ping provides a mock function with given fields: ctx
func (_m *Cache) Ping(ctx context.Context) error {
ret := _m.Called(ctx)