Rewrite registry manager with new interface

Signed-off-by: cd1989 <chende@caicloud.io>
This commit is contained in:
cd1989 2019-02-14 14:55:03 +08:00
parent 6bdf3053a7
commit 8732a20709
31 changed files with 789 additions and 848 deletions

View File

@ -0,0 +1,15 @@
CREATE TABLE registry (
id SERIAL PRIMARY KEY NOT NULL,
name varchar(256),
url varchar(256),
credential_type varchar(16),
access_key varchar(128),
access_secret varchar(1024),
type varchar(32),
insecure boolean,
description varchar(1024),
health varchar(16),
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP,
CONSTRAINT unique_registry_name UNIQUE (name)
);

View File

@ -670,15 +670,15 @@ func TestChangeUserProfile(t *testing.T) {
var targetID, policyID, policyID2, policyID3, jobID, jobID2, jobID3 int64
func TestAddRepTarget(t *testing.T) {
target := models.RepTarget{
Name: "test",
URL: "127.0.0.1:5000",
Username: "admin",
Password: "admin",
func TestAddRegistry(t *testing.T) {
registry := &models.Registry{
Name: "test",
URL: "127.0.0.1:5000",
AccessKey: "admin",
AccessSecret: "admin",
}
// _, err := AddRepTarget(target)
id, err := AddRepTarget(target)
id, err := AddRegistry(registry)
t.Logf("added target, id: %d", id)
if err != nil {
t.Errorf("Error occurred in AddRepTarget: %v", err)
@ -686,118 +686,125 @@ func TestAddRepTarget(t *testing.T) {
targetID = id
}
id2 := id + 99
tgt, err := GetRepTarget(id2)
r, err := GetRegistry(id2)
if err != nil {
t.Errorf("Error occurred in GetTarget: %v, id: %d", err, id2)
t.Errorf("Error occurred in GetRegistry: %v, id: %d", err, id2)
}
if tgt != nil {
if r != nil {
t.Errorf("There should not be a target with id: %d", id2)
}
tgt, err = GetRepTarget(id)
r, err = GetRegistry(id)
if err != nil {
t.Errorf("Error occurred in GetTarget: %v, id: %d", err, id)
}
if tgt == nil {
if r == nil {
t.Errorf("Unable to find a target with id: %d", id)
}
if tgt.URL != "127.0.0.1:5000" {
t.Errorf("Unexpected url in target: %s, expected 127.0.0.1:5000", tgt.URL)
if r.URL != "127.0.0.1:5000" {
t.Errorf("Unexpected url in target: %s, expected 127.0.0.1:5000", r.URL)
}
if tgt.Username != "admin" {
t.Errorf("Unexpected username in target: %s, expected admin", tgt.Username)
if r.AccessKey != "admin" {
t.Errorf("Unexpected username in target: %s, expected admin", r.AccessKey)
}
}
func TestGetRepTargetByName(t *testing.T) {
target, err := GetRepTarget(targetID)
func TestGetRegistryByName(t *testing.T) {
r, err := GetRegistry(targetID)
if err != nil {
t.Fatalf("failed to get target %d: %v", targetID, err)
t.Fatalf("failed to get registry %d: %v", targetID, err)
}
target2, err := GetRepTargetByName(target.Name)
r2, err := GetRegistryByName(r.Name)
if err != nil {
t.Fatalf("failed to get target %s: %v", target.Name, err)
t.Fatalf("failed to get registry %s: %v", r.Name, err)
}
if target.Name != target2.Name {
t.Errorf("unexpected target name: %s, expected: %s", target2.Name, target.Name)
if r.Name != r2.Name {
t.Errorf("unexpected registry name: %s, expected: %s", r2.Name, r.Name)
}
}
func TestGetRepTargetByEndpoint(t *testing.T) {
target, err := GetRepTarget(targetID)
func TestGetRegistryByURL(t *testing.T) {
r, err := GetRegistry(targetID)
if err != nil {
t.Fatalf("failed to get target %d: %v", targetID, err)
t.Fatalf("failed to get registry %d: %v", targetID, err)
}
target2, err := GetRepTargetByEndpoint(target.URL)
r2, err := GetRegistryByURL(r.URL)
if err != nil {
t.Fatalf("failed to get target %s: %v", target.URL, err)
t.Fatalf("failed to get registry %s: %v", r.URL, err)
}
if target.URL != target2.URL {
t.Errorf("unexpected target URL: %s, expected: %s", target2.URL, target.URL)
if r.URL != r2.URL {
t.Errorf("unexpected registry URL: %s, expected: %s", r2.URL, r.URL)
}
}
func TestUpdateRepTarget(t *testing.T) {
target := &models.RepTarget{
Name: "name",
URL: "http://url",
Username: "username",
Password: "password",
func TestUpdateRegistry(t *testing.T) {
registry := &models.Registry{
Name: "name",
URL: "http://url",
AccessKey: "username",
AccessSecret: "password",
}
id, err := AddRepTarget(*target)
id, err := AddRegistry(registry)
if err != nil {
t.Fatalf("failed to add target: %v", err)
t.Fatalf("failed to add registry: %v", err)
}
defer func() {
if err := DeleteRepTarget(id); err != nil {
t.Logf("failed to delete target %d: %v", id, err)
if err := DeleteRegistry(id); err != nil {
t.Logf("failed to delete registry %d: %v", id, err)
}
}()
target.ID = id
target.Name = "new_name"
target.URL = "http://new_url"
target.Username = "new_username"
target.Password = "new_password"
registry.ID = id
registry.Name = "new_name"
registry.URL = "http://new_url"
registry.AccessKey = "new_username"
registry.AccessSecret = "new_password"
if err = UpdateRepTarget(*target); err != nil {
t.Fatalf("failed to update target: %v", err)
if err = UpdateRegistry(registry); err != nil {
t.Fatalf("failed to update registry: %v", err)
}
target, err = GetRepTarget(id)
registry, err = GetRegistry(id)
if err != nil {
t.Fatalf("failed to get target %d: %v", id, err)
}
if target.Name != "new_name" {
t.Errorf("unexpected name: %s, expected: %s", target.Name, "new_name")
if registry.Name != "new_name" {
t.Errorf("unexpected name: %s, expected: %s", registry.Name, "new_name")
}
if target.URL != "http://new_url" {
t.Errorf("unexpected url: %s, expected: %s", target.URL, "http://new_url")
if registry.URL != "http://new_url" {
t.Errorf("unexpected url: %s, expected: %s", registry.URL, "http://new_url")
}
if target.Username != "new_username" {
t.Errorf("unexpected username: %s, expected: %s", target.Username, "new_username")
if registry.AccessKey != "new_username" {
t.Errorf("unexpected username: %s, expected: %s", registry.AccessKey, "new_username")
}
if target.Password != "new_password" {
t.Errorf("unexpected password: %s, expected: %s", target.Password, "new_password")
if registry.AccessSecret != "new_password" {
t.Errorf("unexpected password: %s, expected: %s", registry.AccessSecret, "new_password")
}
}
func TestFilterRepTargets(t *testing.T) {
targets, err := FilterRepTargets("test")
func TestListRegistries(t *testing.T) {
total, registries, err := ListRegistries(&ListRegistryQuery{
Query: "test",
Limit: -1,
})
if err != nil {
t.Fatalf("failed to get all targets: %v", err)
t.Fatalf("failed to get all registries: %v", err)
}
if len(targets) == 0 {
t.Errorf("unexpected num of targets: %d, expected: %d", len(targets), 1)
if total == 0 {
t.Errorf("unexpected num of registries: %d, expected: %d", total, 1)
}
if total != int64(len(registries)) {
t.Errorf("total (%d) should equals to registries count (%d) when pagination not set", total, len(registries))
}
}
@ -1053,14 +1060,14 @@ func TestDeleteRepJob(t *testing.T) {
}
}
func TestDeleteRepTarget(t *testing.T) {
err := DeleteRepTarget(targetID)
func TestDeleteRegistry(t *testing.T) {
err := DeleteRegistry(targetID)
if err != nil {
t.Errorf("Error occurred in DeleteRepTarget: %v, id: %d", err, targetID)
return
}
t.Logf("deleted target, id: %d", targetID)
tgt, err := GetRepTarget(targetID)
tgt, err := GetRegistry(targetID)
if err != nil {
t.Errorf("Error occurred in GetTarget: %v, id: %d", err, targetID)
}

107
src/common/dao/registry.go Normal file
View File

@ -0,0 +1,107 @@
package dao
import (
"time"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/models"
)
// ListRegistryQuery defines the query conditions to list registry.
type ListRegistryQuery struct {
// Query is name query
Query string
// Offset specifies the offset in the registry list to return
Offset int64
// Limit specifies the maximum registries to return
Limit int64
}
// AddRegistry add a new registry
func AddRegistry(registry *models.Registry) (int64, error) {
o := GetOrmer()
return o.Insert(registry)
}
// GetRegistry gets one registry from database by id.
func GetRegistry(id int64) (*models.Registry, error) {
o := GetOrmer()
r := models.Registry{ID: id}
err := o.Read(&r)
if err == orm.ErrNoRows {
return nil, nil
}
return &r, err
}
// GetRegistryByName gets one registry from database by its name.
func GetRegistryByName(name string) (*models.Registry, error) {
o := GetOrmer()
r := models.Registry{Name: name}
err := o.Read(&r)
if err == orm.ErrNoRows {
return nil, nil
}
return &r, err
}
// GetRegistryByURL gets one registry from database by its URL.
func GetRegistryByURL(url string) (*models.Registry, error) {
o := GetOrmer()
r := models.Registry{URL: url}
err := o.Read(&r)
if err == orm.ErrNoRows {
return nil, nil
}
return &r, err
}
// ListRegistries lists registries. Registries returned are sorted by creation time.
// - query: query to the registry name, name query and pagination are defined.
func ListRegistries(query ...*ListRegistryQuery) (int64, []*models.Registry, error) {
o := GetOrmer()
q := o.QueryTable(&models.Registry{})
if len(query) > 0 {
q = q.Filter("name__contains", query[0].Query)
}
total, err := q.Count()
if err != nil {
return -1, nil, err
}
// limit being -1 means no pagination specified.
if len(query) > 0 && query[0].Limit != -1 {
q = q.Offset(query[0].Offset).Limit(query[0].Limit)
}
var registries []*models.Registry
_, err = q.All(&registries)
if err != nil {
return total, nil, err
}
return total, registries, nil
}
// UpdateRegistry updates one registry
func UpdateRegistry(registry *models.Registry) error {
o := GetOrmer()
sql := `update registry
set url = ?, name = ?, credential_type = ?, access_key = ?, access_secret = ?, type = ?, insecure = ?, health = ?, description = ?, update_time = ?
where id = ?`
_, err := o.Raw(sql, registry.URL, registry.Name, registry.CredentialType, registry.AccessKey, registry.AccessSecret,
registry.Type, registry.Insecure, registry.Health, registry.Description, time.Now(), registry.ID).Exec()
return err
}
// DeleteRegistry deletes a registry
func DeleteRegistry(id int64) error {
o := GetOrmer()
_, err := o.Delete(&models.Registry{ID: id})
return err
}

View File

@ -24,97 +24,6 @@ import (
"github.com/goharbor/harbor/src/common/utils/log"
)
// AddRepTarget ...
func AddRepTarget(target models.RepTarget) (int64, error) {
o := GetOrmer()
sql := "insert into replication_target (name, url, username, password, insecure, target_type) values (?, ?, ?, ?, ?, ?) RETURNING id"
var targetID int64
err := o.Raw(sql, target.Name, target.URL, target.Username, target.Password, target.Insecure, target.Type).QueryRow(&targetID)
if err != nil {
return 0, err
}
return targetID, nil
}
// GetRepTarget ...
func GetRepTarget(id int64) (*models.RepTarget, error) {
o := GetOrmer()
t := models.RepTarget{ID: id}
err := o.Read(&t)
if err == orm.ErrNoRows {
return nil, nil
}
return &t, err
}
// GetRepTargetByName ...
func GetRepTargetByName(name string) (*models.RepTarget, error) {
o := GetOrmer()
t := models.RepTarget{Name: name}
err := o.Read(&t, "Name")
if err == orm.ErrNoRows {
return nil, nil
}
return &t, err
}
// GetRepTargetByEndpoint ...
func GetRepTargetByEndpoint(endpoint string) (*models.RepTarget, error) {
o := GetOrmer()
t := models.RepTarget{
URL: endpoint,
}
err := o.Read(&t, "URL")
if err == orm.ErrNoRows {
return nil, nil
}
return &t, err
}
// DeleteRepTarget ...
func DeleteRepTarget(id int64) error {
o := GetOrmer()
_, err := o.Delete(&models.RepTarget{ID: id})
return err
}
// UpdateRepTarget ...
func UpdateRepTarget(target models.RepTarget) error {
o := GetOrmer()
sql := `update replication_target
set url = ?, name = ?, username = ?, password = ?, insecure = ?, update_time = ?
where id = ?`
_, err := o.Raw(sql, target.URL, target.Name, target.Username, target.Password, target.Insecure, time.Now(), target.ID).Exec()
return err
}
// FilterRepTargets filters targets by name
func FilterRepTargets(name string) ([]*models.RepTarget, error) {
o := GetOrmer()
var args []interface{}
sql := `select * from replication_target `
if len(name) != 0 {
sql += `where name like ? `
args = append(args, "%"+Escape(name)+"%")
}
sql += `order by creation_time`
var targets []*models.RepTarget
if _, err := o.Raw(sql, args).QueryRows(&targets); err != nil {
return nil, err
}
return targets, nil
}
// AddRepPolicy ...
func AddRepPolicy(policy models.RepPolicy) (int64, error) {
o := GetOrmer()

View File

@ -23,12 +23,12 @@ import (
)
func TestMethodsOfWatchItem(t *testing.T) {
targetID, err := AddRepTarget(models.RepTarget{
registryID, err := AddRegistry(&models.Registry{
Name: "test_target_for_watch_item",
URL: "http://127.0.0.1",
})
require.Nil(t, err)
defer DeleteRepTarget(targetID)
defer DeleteRegistry(registryID)
policyID, err := AddRepPolicy(models.RepPolicy{
Name: "test_policy_for_watch_item",

View File

@ -19,7 +19,8 @@ import (
)
func init() {
orm.RegisterModel(new(RepTarget),
orm.RegisterModel(
new(Registry),
new(RepPolicy),
new(RepJob),
new(User),

View File

@ -0,0 +1,58 @@
package models
import (
"time"
"github.com/astaxie/beego/validation"
"github.com/goharbor/harbor/src/common/utils"
)
// Registry is the model for a registry, which wraps the endpoint URL and credential of a remote registry.
type Registry struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
URL string `orm:"column(url)" json:"endpoint"`
Name string `orm:"column(name)" json:"name"`
CredentialType string `orm:"column(credential_type);default(basic)" json:"credential_type"`
AccessKey string `orm:"column(access_key)" json:"access_key"`
AccessSecret string `orm:"column(access_secret)" json:"access_secret"`
Type string `orm:"column(type)" json:"type"`
Insecure bool `orm:"column(insecure)" json:"insecure"`
Description string `orm:"column(description)" json:"description"`
Health string `orm:"column(health)" json:"health"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// TableName is required by by beego orm to map Registry to table registry
func (r *Registry) TableName() string {
return RegistryTable
}
// Valid ...
func (r *Registry) Valid(v *validation.Validation) {
if len(r.Name) == 0 {
v.SetError("name", "can not be empty")
}
if len(r.Name) > 64 {
v.SetError("name", "max length is 64")
}
url, err := utils.ParseEndpoint(r.URL)
if err != nil {
v.SetError("endpoint", err.Error())
} else {
// Prevent SSRF security issue #3755
r.URL = url.Scheme + "://" + url.Host + url.Path
if len(r.URL) > 64 {
v.SetError("endpoint", "max length is 64")
}
}
// password is encoded using base64, the length of this field
// in DB is 64, so the max length in request is 48
if len(r.AccessSecret) > 48 {
v.SetError("password", "max length is 48")
}
}

View File

@ -22,58 +22,58 @@ import (
"github.com/stretchr/testify/require"
)
func TestValidOfTarget(t *testing.T) {
func TestValidOfRegistry(t *testing.T) {
cases := []struct {
target RepTarget
target Registry
err bool
expected RepTarget
expected Registry
}{
// name is null
{
RepTarget{
Registry{
Name: "",
},
true,
RepTarget{}},
Registry{}},
// url is null
{
RepTarget{
Registry{
Name: "endpoint01",
URL: "",
},
true,
RepTarget{},
Registry{},
},
// invalid url
{
RepTarget{
Registry{
Name: "endpoint01",
URL: "ftp://example.com",
},
true,
RepTarget{},
Registry{},
},
// invalid url
{
RepTarget{
Registry{
Name: "endpoint01",
URL: "ftp://example.com",
},
true,
RepTarget{},
Registry{},
},
// valid url
{
RepTarget{
Registry{
Name: "endpoint01",
URL: "example.com",
},
false,
RepTarget{
Registry{
Name: "endpoint01",
URL: "http://example.com",
},
@ -81,12 +81,12 @@ func TestValidOfTarget(t *testing.T) {
// valid url
{
RepTarget{
Registry{
Name: "endpoint01",
URL: "http://example.com",
},
false,
RepTarget{
Registry{
Name: "endpoint01",
URL: "http://example.com",
},
@ -94,12 +94,12 @@ func TestValidOfTarget(t *testing.T) {
// valid url
{
RepTarget{
Registry{
Name: "endpoint01",
URL: "https://example.com",
},
false,
RepTarget{
Registry{
Name: "endpoint01",
URL: "https://example.com",
},
@ -107,12 +107,12 @@ func TestValidOfTarget(t *testing.T) {
// valid url
{
RepTarget{
Registry{
Name: "endpoint01",
URL: "http://example.com/redirect?key=value",
},
false,
RepTarget{
Registry{
Name: "endpoint01",
URL: "http://example.com/redirect",
}},

View File

@ -16,9 +16,6 @@ package models
import (
"time"
"github.com/astaxie/beego/validation"
"github.com/goharbor/harbor/src/common/utils"
)
const (
@ -28,8 +25,8 @@ const (
RepOpDelete string = "delete"
// RepOpSchedule represents the operation of a job to schedule the real replication process
RepOpSchedule string = "schedule"
// RepTargetTable is the table name for replication targets
RepTargetTable = "replication_target"
// RegistryTable is the table name for registry
RegistryTable = "registry"
// RepJobTable is the table name for replication jobs
RepJobTable = "replication_job"
// RepPolicyTable is table name for replication policies
@ -67,53 +64,6 @@ type RepJob struct {
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// RepTarget is the model for a replication targe, i.e. destination, which wraps the endpoint URL and username/password of a remote registry.
type RepTarget struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
URL string `orm:"column(url)" json:"endpoint"`
Name string `orm:"column(name)" json:"name"`
Username string `orm:"column(username)" json:"username"`
Password string `orm:"column(password)" json:"password"`
Type int `orm:"column(target_type)" json:"type"`
Insecure bool `orm:"column(insecure)" json:"insecure"`
Health string `orm:"column(health)" json:"health"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// Valid ...
func (r *RepTarget) Valid(v *validation.Validation) {
if len(r.Name) == 0 {
v.SetError("name", "can not be empty")
}
if len(r.Name) > 64 {
v.SetError("name", "max length is 64")
}
url, err := utils.ParseEndpoint(r.URL)
if err != nil {
v.SetError("endpoint", err.Error())
} else {
// Prevent SSRF security issue #3755
r.URL = url.Scheme + "://" + url.Host + url.Path
if len(r.URL) > 64 {
v.SetError("endpoint", "max length is 64")
}
}
// password is encoded using base64, the length of this field
// in DB is 64, so the max length in request is 48
if len(r.Password) > 48 {
v.SetError("password", "max length is 48")
}
}
// TableName is required by by beego orm to map RepTarget to table replication_target
func (r *RepTarget) TableName() string {
return RepTargetTable
}
// TableName is required by by beego orm to map RepJob to table replication_job
func (r *RepJob) TableName() string {
return RepJobTable

View File

@ -20,3 +20,29 @@ import (
// ErrDupProject is the error returned when creating a duplicate project
var ErrDupProject = errors.New("duplicate project")
const (
// ReasonNotFound indicates resource not found
ReasonNotFound = "NotFound"
)
// KnownError represents known type errors
type KnownError struct {
// Reason is reason of the error, such as NotFound
Reason string
// Message is the message of the error
Message string
}
// Error returns the error message
func (e KnownError) Error() string {
return e.Message
}
// Is checks whether a error is a given type error
func Is(err error, reason string) bool {
if e, ok := err.(KnownError); ok && e.Reason == reason {
return true
}
return false
}

View File

@ -0,0 +1,40 @@
package error
import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
)
func TestIs(t *testing.T) {
cases := []struct {
err error
reason string
expected bool
}{
{
err: errors.New(""),
reason: ReasonNotFound,
expected: false,
},
{
err: KnownError{
Reason: ReasonNotFound,
},
reason: ReasonNotFound,
expected: true,
},
{
err: KnownError{
Reason: ReasonNotFound,
},
reason: "Other",
expected: false,
},
}
for _, c := range cases {
assert.Equal(t, c.expected, Is(c.err, c.reason))
}
}

View File

@ -14,13 +14,16 @@
package test
// FakeReplicatoinController ...
type FakeReplicatoinController struct {
FakePolicyManager
}
func (f *FakeReplicatoinController) Init() error {
// Init initialize replication controller
func (f *FakeReplicatoinController) Init(closing chan struct{}) error {
return nil
}
// Replicate ...
func (f *FakeReplicatoinController) Replicate(policyID int64, metadata ...map[string]interface{}) error {
return nil
}

View File

@ -23,14 +23,12 @@ import (
const (
// Prepare Test info
TestUserName = "testUser0001"
TestUserPwd = "testUser0001"
TestUserEmail = "testUser0001@mydomain.com"
TestProName = "testProject0001"
TestTargetName = "testTarget0001"
TestRepoName = "testRepo0001"
AdminName = "admin"
DefaultProjectName = "library"
TestUserName = "testUser0001"
TestUserPwd = "testUser0001"
TestUserEmail = "testUser0001@mydomain.com"
TestProName = "testProject0001"
TestRegistryName = "testRegistry0001"
TestRepoName = "testRepo0001"
)
func CommonAddUser() {
@ -83,25 +81,25 @@ func CommonDelProject() {
_ = dao.DeleteProject(commonProject.ProjectID)
}
func CommonAddTarget() {
func CommonAddRegistry() {
endPoint := os.Getenv("REGISTRY_URL")
commonTarget := &models.RepTarget{
URL: endPoint,
Name: TestTargetName,
Username: adminName,
Password: adminPwd,
commonRegistry := &models.Registry{
URL: endPoint,
Name: TestRegistryName,
AccessKey: adminName,
AccessSecret: adminPwd,
}
_, _ = dao.AddRepTarget(*commonTarget)
_, _ = dao.AddRegistry(commonRegistry)
}
func CommonGetTarget() int {
target, _ := dao.GetRepTargetByName(TestTargetName)
return int(target.ID)
func CommonGetRegistry() int {
registry, _ := dao.GetRegistryByName(TestRegistryName)
return int(registry.ID)
}
func CommonDelTarget() {
target, _ := dao.GetRepTargetByName(TestTargetName)
_ = dao.DeleteRepTarget(target.ID)
func CommonDelRegistry() {
registry, _ := dao.GetRegistryByName(TestRegistryName)
_ = dao.DeleteRegistry(registry.ID)
}
func CommonAddRepository() {

View File

@ -34,9 +34,6 @@ import (
"github.com/goharbor/harbor/src/core/filter"
"github.com/goharbor/harbor/tests/apitests/apilib"
// "strconv"
// "strings"
"github.com/astaxie/beego"
"github.com/dghubble/sling"
@ -126,11 +123,8 @@ func init() {
beego.Router("/api/repositories/*/tags/:tag/manifest", &RepositoryAPI{}, "get:GetManifests")
beego.Router("/api/repositories/*/signatures", &RepositoryAPI{}, "get:GetSignatures")
beego.Router("/api/repositories/top", &RepositoryAPI{}, "get:GetTopRepos")
beego.Router("/api/targets/", &TargetAPI{}, "get:List")
beego.Router("/api/targets/", &TargetAPI{}, "post:Post")
beego.Router("/api/targets/:id([0-9]+)", &TargetAPI{})
beego.Router("/api/targets/:id([0-9]+)/policies/", &TargetAPI{}, "get:ListPolicies")
beego.Router("/api/targets/ping", &TargetAPI{}, "post:Ping")
beego.Router("/api/registries", &RegistryAPI{}, "get:List;post:Post")
beego.Router("/api/registries/:id([0-9]+)", &RegistryAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/policies/replication/:id([0-9]+)", &RepPolicyAPI{})
beego.Router("/api/policies/replication", &RepPolicyAPI{}, "get:List")
beego.Router("/api/policies/replication", &RepPolicyAPI{}, "post:Post;delete:Delete")
@ -177,7 +171,7 @@ func init() {
beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels", chartLabelAPIType, "get:GetLabels;post:MarkLabel")
beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels/:id([0-9]+)", chartLabelAPIType, "delete:RemoveLabel")
if err := core.Init(); err != nil {
if err := core.Init(make(chan struct{})); err != nil {
log.Fatalf("failed to initialize GlobalController: %v", err)
}

View File

@ -455,18 +455,18 @@ func TestListResources(t *testing.T) {
require.Nil(t, err)
defer dao.DeleteLabel(projectLabelID)
targetID, err := dao.AddRepTarget(models.RepTarget{
registryID, err := dao.AddRegistry(&models.Registry{
Name: "target_for_testing_label_resource",
URL: "https://192.168.0.1",
})
require.Nil(t, err)
defer dao.DeleteRepTarget(targetID)
defer dao.DeleteRegistry(registryID)
// create a policy references both global and project labels
policyID, err := dao.AddRepPolicy(models.RepPolicy{
Name: "policy_for_testing_label_resource",
ProjectID: 1,
TargetID: targetID,
TargetID: registryID,
Trigger: fmt.Sprintf(`{"kind":"%s"}`, replication.TriggerKindManual),
Filters: fmt.Sprintf(`[{"kind":"%s","value":%d}, {"kind":"%s","value":%d}]`,
replication.FilterItemKindLabel, globalLabelID,

View File

@ -24,18 +24,18 @@ import (
// ReplicationPolicy defines the data model used in API level
type ReplicationPolicy struct {
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Filters []rep_models.Filter `json:"filters"`
ReplicateDeletion bool `json:"replicate_deletion"`
Trigger *rep_models.Trigger `json:"trigger"`
Projects []*common_models.Project `json:"projects"`
Targets []*common_models.RepTarget `json:"targets"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
ReplicateExistingImageNow bool `json:"replicate_existing_image_now"`
ErrorJobCount int64 `json:"error_job_count"`
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Filters []rep_models.Filter `json:"filters"`
ReplicateDeletion bool `json:"replicate_deletion"`
Trigger *rep_models.Trigger `json:"trigger"`
Projects []*common_models.Project `json:"projects"`
Registries []*common_models.Registry `json:"registries"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
ReplicateExistingImageNow bool `json:"replicate_existing_image_now"`
ErrorJobCount int64 `json:"error_job_count"`
}
// Valid ...
@ -52,7 +52,7 @@ func (r *ReplicationPolicy) Valid(v *validation.Validation) {
v.SetError("projects", "can not be empty")
}
if len(r.Targets) == 0 {
if len(r.Registries) == 0 {
v.SetError("targets", "can not be empty")
}

View File

@ -6,8 +6,10 @@ import (
"strconv"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
utilerr "github.com/goharbor/harbor/src/common/utils/error"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication/ng"
"github.com/goharbor/harbor/src/replication/ng/model"
"github.com/goharbor/harbor/src/replication/ng/registry"
)
@ -30,30 +32,25 @@ func (t *RegistryAPI) Prepare() {
return
}
t.manager = registry.NewDefaultManager()
if t.manager == nil {
log.Error("failed to create registry manager")
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
t.manager = ng.RegistryMgr
}
// Get gets a registry by id.
func (t *RegistryAPI) Get() {
id := t.GetIDFromURL()
registry, err := dao.GetRepTarget(id)
registry, err := t.manager.Get(id)
if err != nil {
if utilerr.Is(err, utilerr.ReasonNotFound) {
t.HandleNotFound(fmt.Sprintf("registry %d not found", id))
return
}
log.Errorf("failed to get registry %d: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if registry == nil {
t.HandleNotFound(fmt.Sprintf("registry %d not found", id))
return
}
// Hide password
registry.Password = ""
// Hide access secret
registry.Credential.AccessSecret = "*****"
t.Data["json"] = registry
t.ServeJSON()
@ -62,15 +59,18 @@ func (t *RegistryAPI) Get() {
// List lists all registries that match a given registry name.
func (t *RegistryAPI) List() {
name := t.GetString("name")
registries, err := dao.FilterRepTargets(name)
_, registries, err := t.manager.List(&model.RegistryQuery{
Name: name,
})
if err != nil {
log.Errorf("failed to filter registries %s: %v", name, err)
log.Errorf("failed to list registries %s: %v", name, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
// Hide passwords
for _, registry := range registries {
registry.Password = ""
registry.Credential.AccessSecret = "*****"
}
t.Data["json"] = registries
@ -80,32 +80,21 @@ func (t *RegistryAPI) List() {
// Post creates a registry
func (t *RegistryAPI) Post() {
registry := &models.RepTarget{}
registry := &model.Registry{}
t.DecodeJSONReqAndValidate(registry)
reg, err := dao.GetRepTargetByName(registry.Name)
reg, err := t.manager.GetByName(registry.Name)
if err != nil {
log.Errorf("failed to get registry %s: %v", registry.Name, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if reg != nil {
t.HandleConflict(fmt.Sprintf("name '%s' is already used"), registry.Name)
t.HandleConflict(fmt.Sprintf("name '%s' is already used", registry.Name))
return
}
reg, err = dao.GetRepTargetByEndpoint(registry.URL)
if err != nil {
log.Errorf("failed to get registry by URL [ %s ]: %v", registry.URL, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if reg != nil {
t.HandleConflict(fmt.Sprintf("registry with endpoint '%s' already exists", registry.URL))
return
}
id, err := t.manager.AddRegistry(registry)
id, err := t.manager.Add(registry)
if err != nil {
log.Errorf("Add registry '%s' error: %v", registry.URL, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
@ -118,7 +107,7 @@ func (t *RegistryAPI) Post() {
func (t *RegistryAPI) Put() {
id := t.GetIDFromURL()
registry, err := t.manager.GetRegistry(id)
registry, err := t.manager.Get(id)
if err != nil {
log.Errorf("Get registry by id %d error: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
@ -134,7 +123,6 @@ func (t *RegistryAPI) Put() {
t.DecodeJSONReq(&req)
originalName := registry.Name
originalURL := registry.URL
if req.Name != nil {
registry.Name = *req.Name
@ -143,10 +131,10 @@ func (t *RegistryAPI) Put() {
registry.URL = *req.Endpoint
}
if req.Username != nil {
registry.Username = *req.Username
registry.Credential.AccessKey = *req.Username
}
if req.Password != nil {
registry.Password = *req.Password
registry.Credential.AccessSecret = *req.Password
}
if req.Insecure != nil {
registry.Insecure = *req.Insecure
@ -155,7 +143,7 @@ func (t *RegistryAPI) Put() {
t.Validate(registry)
if registry.Name != originalName {
reg, err := dao.GetRepTargetByName(registry.Name)
reg, err := t.manager.GetByName(registry.Name)
if err != nil {
log.Errorf("Get registry by name '%s' error: %v", registry.Name, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
@ -167,20 +155,7 @@ func (t *RegistryAPI) Put() {
}
}
if registry.URL != originalURL {
reg, err := dao.GetRepTargetByEndpoint(registry.URL)
if err != nil {
log.Errorf("Get registry by URL '%s' error: %v", registry.URL, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if reg != nil {
t.HandleConflict(fmt.Sprintf("registry with endpoint '%s' already exists", registry.URL))
return
}
}
if err := t.manager.UpdateRegistry(registry); err != nil {
if err := t.manager.Update(registry); err != nil {
log.Errorf("Update registry %d error: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
@ -190,19 +165,37 @@ func (t *RegistryAPI) Put() {
func (t *RegistryAPI) Delete() {
id := t.GetIDFromURL()
registry, err := dao.GetRepTarget(id)
_, err := t.manager.Get(id)
if err != nil {
log.Errorf("Get registry %d error: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if utilerr.Is(err, utilerr.ReasonNotFound) {
t.HandleNotFound(fmt.Sprintf("registry %d not found", id))
return
}
if registry == nil {
t.HandleNotFound(fmt.Sprintf("target %d not found", id))
msg := fmt.Sprintf("Get registry %d error: %v", id, err)
log.Error(msg)
t.HandleInternalServerError(msg)
return
}
if err := t.manager.DeleteRegistry(id); err != nil {
log.Errorf("Delete registry %d error: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
policies, err := dao.GetRepPolicyByTarget(id)
if err != nil {
msg := fmt.Sprintf("Get policies related to registry %d error: %v", id, err)
log.Error(msg)
t.HandleInternalServerError(msg)
return
}
if len(policies) > 0 {
msg := fmt.Sprintf("Can't delete registry with replication policies, %d found", len(policies))
log.Error(msg)
t.HandleInternalServerError(msg)
return
}
if err := t.manager.Remove(id); err != nil {
msg := fmt.Sprintf("Delete registry %d error: %v", id, err)
log.Error(msg)
t.HandleInternalServerError(msg)
}
}

View File

@ -158,15 +158,15 @@ func (pa *RepPolicyAPI) Post() {
}
// check the existence of targets
for _, target := range policy.Targets {
t, err := dao.GetRepTarget(target.ID)
for _, r := range policy.Registries {
t, err := dao.GetRegistry(r.ID)
if err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", target.ID, err))
pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", r.ID, err))
return
}
if t == nil {
pa.HandleNotFound(fmt.Sprintf("target %d not found", target.ID))
pa.HandleNotFound(fmt.Sprintf("target %d not found", r.ID))
return
}
}
@ -271,15 +271,15 @@ func (pa *RepPolicyAPI) Put() {
}
// check the existence of targets
for _, target := range policy.Targets {
t, err := dao.GetRepTarget(target.ID)
for _, r := range policy.Registries {
t, err := dao.GetRegistry(r.ID)
if err != nil {
pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", target.ID, err))
pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", r.ID, err))
return
}
if t == nil {
pa.HandleNotFound(fmt.Sprintf("target %d not found", target.ID))
pa.HandleNotFound(fmt.Sprintf("target %d not found", r.ID))
return
}
}
@ -379,12 +379,12 @@ func convertFromRepPolicy(projectMgr promgr.ProjectManager, policy rep_models.Re
// populate targets
for _, targetID := range policy.TargetIDs {
target, err := dao.GetRepTarget(targetID)
r, err := dao.GetRegistry(targetID)
if err != nil {
return nil, err
}
target.Password = ""
ply.Targets = append(ply.Targets, target)
r.AccessSecret = ""
ply.Registries = append(ply.Registries, r)
}
// populate label used in label filter
@ -434,8 +434,8 @@ func convertToRepPolicy(policy *api_models.ReplicationPolicy) rep_models.Replica
ply.Namespaces = append(ply.Namespaces, project.Name)
}
for _, target := range policy.Targets {
ply.TargetIDs = append(ply.TargetIDs, target.ID)
for _, r := range policy.Registries {
ply.TargetIDs = append(ply.TargetIDs, r.ID)
}
return ply

View File

@ -50,8 +50,8 @@ func TestRepPolicyAPIPost(t *testing.T) {
return nil
}
CommonAddTarget()
targetID = int64(CommonGetTarget())
CommonAddRegistry()
targetID = int64(CommonGetRegistry())
var err error
labelID2, err = dao.AddLabel(&models.Label{
@ -131,7 +131,7 @@ func TestRepPolicyAPIPost(t *testing.T) {
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
Registries: []*models.Registry{
{
ID: targetID,
},
@ -159,7 +159,7 @@ func TestRepPolicyAPIPost(t *testing.T) {
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
Registries: []*models.Registry{
{
ID: targetID,
},
@ -190,7 +190,7 @@ func TestRepPolicyAPIPost(t *testing.T) {
ProjectID: 10000,
},
},
Targets: []*models.RepTarget{
Registries: []*models.Registry{
{
ID: targetID,
},
@ -221,7 +221,7 @@ func TestRepPolicyAPIPost(t *testing.T) {
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
Registries: []*models.Registry{
{
ID: 10000,
},
@ -252,7 +252,7 @@ func TestRepPolicyAPIPost(t *testing.T) {
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
Registries: []*models.Registry{
{
ID: targetID,
},
@ -287,7 +287,7 @@ func TestRepPolicyAPIPost(t *testing.T) {
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
Registries: []*models.Registry{
{
ID: targetID,
},
@ -323,7 +323,7 @@ func TestRepPolicyAPIPost(t *testing.T) {
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
Registries: []*models.Registry{
{
ID: targetID,
},
@ -567,7 +567,7 @@ func TestRepPolicyAPIPut(t *testing.T) {
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
Registries: []*models.Registry{
{
ID: targetID,
},
@ -598,7 +598,7 @@ func TestRepPolicyAPIPut(t *testing.T) {
ProjectID: projectID,
},
},
Targets: []*models.RepTarget{
Registries: []*models.Registry{
{
ID: targetID,
},
@ -677,7 +677,7 @@ func TestConvertToRepPolicy(t *testing.T) {
Name: "library",
},
},
Targets: []*models.RepTarget{
Registries: []*models.Registry{
{
ID: 1,
},

View File

@ -30,15 +30,15 @@ const (
)
func TestReplicationAPIPost(t *testing.T) {
targetID, err := dao.AddRepTarget(
models.RepTarget{
registryID, err := dao.AddRegistry(
&models.Registry{
Name: "test_replication_target",
URL: "127.0.0.1",
Username: "username",
Password: "password",
AccessKey: "username",
AccessSecret: "password",
})
require.Nil(t, err)
defer dao.DeleteRepTarget(targetID)
defer dao.DeleteRegistry(registryID)
policyID, err := dao.AddRepPolicy(
models.RepPolicy{

View File

@ -1,369 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"strconv"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
)
// TargetAPI handles request to /api/targets/ping /api/targets/{}
type TargetAPI struct {
BaseController
secretKey string
}
// Prepare validates the user
func (t *TargetAPI) Prepare() {
t.BaseController.Prepare()
if !t.SecurityCtx.IsAuthenticated() {
t.HandleUnauthorized()
return
}
if !t.SecurityCtx.IsSysAdmin() {
t.HandleForbidden(t.SecurityCtx.GetUsername())
return
}
var err error
t.secretKey, err = config.SecretKey()
if err != nil {
log.Errorf("failed to get secret key: %v", err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
}
func (t *TargetAPI) ping(endpoint, username, password string, insecure bool) {
registry, err := newRegistryClient(endpoint, insecure, username, password)
if err == nil {
err = registry.Ping()
}
if err != nil {
log.Errorf("failed to ping target: %v", err)
// do not return any detail information of the error, or may cause SSRF security issue #3755
t.RenderError(http.StatusBadRequest, "failed to ping target")
return
}
}
// Ping validates whether the target is reachable and whether the credential is valid
func (t *TargetAPI) Ping() {
req := struct {
ID *int64 `json:"id"`
Endpoint *string `json:"endpoint"`
Username *string `json:"username"`
Password *string `json:"password"`
Insecure *bool `json:"insecure"`
}{}
t.DecodeJSONReq(&req)
target := &models.RepTarget{}
if req.ID != nil {
var err error
target, err = dao.GetRepTarget(*req.ID)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", *req.ID, err))
return
}
if target == nil {
t.HandleNotFound(fmt.Sprintf("target %d not found", *req.ID))
return
}
if len(target.Password) != 0 {
target.Password, err = utils.ReversibleDecrypt(target.Password, t.secretKey)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("failed to decrypt password: %v", err))
return
}
}
}
if req.Endpoint != nil {
url, err := utils.ParseEndpoint(*req.Endpoint)
if err != nil {
t.HandleBadRequest(err.Error())
return
}
// Prevent SSRF security issue #3755
target.URL = url.Scheme + "://" + url.Host + url.Path
}
if req.Username != nil {
target.Username = *req.Username
}
if req.Password != nil {
target.Password = *req.Password
}
if req.Insecure != nil {
target.Insecure = *req.Insecure
}
t.ping(target.URL, target.Username, target.Password, target.Insecure)
}
// Get ...
func (t *TargetAPI) Get() {
id := t.GetIDFromURL()
target, err := dao.GetRepTarget(id)
if err != nil {
log.Errorf("failed to get target %d: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if target == nil {
t.HandleNotFound(fmt.Sprintf("target %d not found", id))
return
}
target.Password = ""
t.Data["json"] = target
t.ServeJSON()
}
// List ...
func (t *TargetAPI) List() {
name := t.GetString("name")
targets, err := dao.FilterRepTargets(name)
if err != nil {
log.Errorf("failed to filter targets %s: %v", name, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
for _, target := range targets {
target.Password = ""
}
t.Data["json"] = targets
t.ServeJSON()
return
}
// Post ...
func (t *TargetAPI) Post() {
target := &models.RepTarget{}
t.DecodeJSONReqAndValidate(target)
ta, err := dao.GetRepTargetByName(target.Name)
if err != nil {
log.Errorf("failed to get target %s: %v", target.Name, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if ta != nil {
t.HandleConflict("name is already used")
return
}
ta, err = dao.GetRepTargetByEndpoint(target.URL)
if err != nil {
log.Errorf("failed to get target [ %s ]: %v", target.URL, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if ta != nil {
t.HandleConflict(fmt.Sprintf("the target whose endpoint is %s already exists", target.URL))
return
}
if len(target.Password) != 0 {
target.Password, err = utils.ReversibleEncrypt(target.Password, t.secretKey)
if err != nil {
log.Errorf("failed to encrypt password: %v", err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
}
id, err := dao.AddRepTarget(*target)
if err != nil {
log.Errorf("failed to add target: %v", err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
t.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
}
// Put ...
func (t *TargetAPI) Put() {
id := t.GetIDFromURL()
target, err := dao.GetRepTarget(id)
if err != nil {
log.Errorf("failed to get target %d: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if target == nil {
t.HandleNotFound(fmt.Sprintf("target %d not found", id))
return
}
if len(target.Password) != 0 {
target.Password, err = utils.ReversibleDecrypt(target.Password, t.secretKey)
if err != nil {
log.Errorf("failed to decrypt password: %v", err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
}
req := struct {
Name *string `json:"name"`
Endpoint *string `json:"endpoint"`
Username *string `json:"username"`
Password *string `json:"password"`
Insecure *bool `json:"insecure"`
}{}
t.DecodeJSONReq(&req)
originalName := target.Name
originalURL := target.URL
if req.Name != nil {
target.Name = *req.Name
}
if req.Endpoint != nil {
target.URL = *req.Endpoint
}
if req.Username != nil {
target.Username = *req.Username
}
if req.Password != nil {
target.Password = *req.Password
}
if req.Insecure != nil {
target.Insecure = *req.Insecure
}
t.Validate(target)
if target.Name != originalName {
ta, err := dao.GetRepTargetByName(target.Name)
if err != nil {
log.Errorf("failed to get target %s: %v", target.Name, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if ta != nil {
t.HandleConflict("name is already used")
return
}
}
if target.URL != originalURL {
ta, err := dao.GetRepTargetByEndpoint(target.URL)
if err != nil {
log.Errorf("failed to get target [ %s ]: %v", target.URL, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if ta != nil {
t.HandleConflict(fmt.Sprintf("the target whose endpoint is %s already exists", target.URL))
return
}
}
if len(target.Password) != 0 {
target.Password, err = utils.ReversibleEncrypt(target.Password, t.secretKey)
if err != nil {
log.Errorf("failed to encrypt password: %v", err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
}
if err := dao.UpdateRepTarget(*target); err != nil {
log.Errorf("failed to update target %d: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
}
// Delete ...
func (t *TargetAPI) Delete() {
id := t.GetIDFromURL()
target, err := dao.GetRepTarget(id)
if err != nil {
log.Errorf("failed to get target %d: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if target == nil {
t.HandleNotFound(fmt.Sprintf("target %d not found", id))
return
}
policies, err := dao.GetRepPolicyByTarget(id)
if err != nil {
log.Errorf("failed to get policies according target %d: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if len(policies) > 0 {
log.Error("the target is used by policies, can not be deleted")
t.CustomAbort(http.StatusPreconditionFailed, "the target is used by policies, can not be deleted")
}
if err = dao.DeleteRepTarget(id); err != nil {
log.Errorf("failed to delete target %d: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
}
func newRegistryClient(endpoint string, insecure bool, username, password string) (*registry.Registry, error) {
transport := registry.GetHTTPTransport(insecure)
credential := auth.NewBasicAuthCredential(username, password)
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: transport,
}, credential)
return registry.NewRegistry(endpoint, &http.Client{
Transport: registry.NewTransport(transport, authorizer),
})
}
// ListPolicies ...
func (t *TargetAPI) ListPolicies() {
id := t.GetIDFromURL()
target, err := dao.GetRepTarget(id)
if err != nil {
log.Errorf("failed to get target %d: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if target == nil {
t.HandleNotFound(fmt.Sprintf("target %d not found", id))
return
}
policies, err := dao.GetRepPolicyByTarget(id)
if err != nil {
log.Errorf("failed to get policies according target %d: %v", id, err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
t.Data["json"] = policies
t.ServeJSON()
}

View File

@ -18,7 +18,9 @@ import (
"encoding/gob"
"fmt"
"os"
"os/signal"
"strconv"
"syscall"
"github.com/astaxie/beego"
_ "github.com/astaxie/beego/session/redis"
@ -39,8 +41,6 @@ import (
"github.com/goharbor/harbor/src/core/service/token"
"github.com/goharbor/harbor/src/replication/core"
_ "github.com/goharbor/harbor/src/replication/event"
"os/signal"
"syscall"
)
const (

View File

@ -97,11 +97,6 @@ func initRouters() {
beego.Router("/api/policies/replication/:id([0-9]+)", &api.RepPolicyAPI{})
beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "get:List")
beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "post:Post")
beego.Router("/api/targets/", &api.TargetAPI{}, "get:List")
beego.Router("/api/targets/", &api.TargetAPI{}, "post:Post")
beego.Router("/api/targets/:id([0-9]+)", &api.TargetAPI{})
beego.Router("/api/targets/:id([0-9]+)/policies/", &api.TargetAPI{}, "get:ListPolicies")
beego.Router("/api/targets/ping", &api.TargetAPI{}, "post:Ping")
beego.Router("/api/logs", &api.LogAPI{})
beego.Router("/api/internal/configurations", &api.ConfigAPI{}, "get:GetInternalConfig;put:Put")

View File

@ -25,13 +25,15 @@ import (
"github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/ng/model"
"github.com/goharbor/harbor/src/replication/ng/registry"
"github.com/goharbor/harbor/src/replication/policy"
"github.com/goharbor/harbor/src/replication/registry"
"github.com/goharbor/harbor/src/replication/replicator"
"github.com/goharbor/harbor/src/replication/source"
"github.com/goharbor/harbor/src/replication/trigger"
"github.com/docker/distribution/uuid"
"github.com/goharbor/harbor/src/replication/ng"
)
// Controller defines the methods that a replicatoin controllter should implement
@ -89,10 +91,15 @@ func NewDefaultController(cfg ControllerConfig) *DefaultController {
return ctl
}
// Init creates the GlobalController and inits it
// Init initializes GlobalController and replication related managers
func Init(closing chan struct{}) error {
GlobalController = NewDefaultController(ControllerConfig{}) // Use default data
return GlobalController.Init(closing)
GlobalController = NewDefaultController(ControllerConfig{})
err := GlobalController.Init(closing)
if err != nil {
return err
}
return ng.Init()
}
// Init will initialize the controller and the sub components
@ -219,13 +226,13 @@ func (ctl *DefaultController) Replicate(policyID int64, metadata ...map[string]i
log.Debugf("replication candidates are null, no further action needed")
}
targets := []*common_models.RepTarget{}
registries := []*model.Registry{}
for _, targetID := range policy.TargetIDs {
target, err := ctl.registryManager.GetRegistry(targetID)
r, err := ctl.registryManager.Get(targetID)
if err != nil {
return err
}
targets = append(targets, target)
registries = append(registries, r)
}
// Get operation uuid from metadata, if none provided, generate one.
@ -239,7 +246,7 @@ func (ctl *DefaultController) Replicate(policyID int64, metadata ...map[string]i
PolicyID: policyID,
OpUUID: opUUID,
Candidates: candidates,
Targets: targets,
Registries: registries,
})
}

View File

@ -21,24 +21,24 @@ import (
"github.com/goharbor/harbor/src/common/utils/test"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/ng/registry"
"github.com/goharbor/harbor/src/replication/source"
"github.com/goharbor/harbor/src/replication/target"
"github.com/goharbor/harbor/src/replication/trigger"
"github.com/stretchr/testify/assert"
)
func TestMain(m *testing.M) {
GlobalController = &DefaultController{
policyManager: &test.FakePolicyManager{},
targetManager: target.NewDefaultManager(),
sourcer: source.NewSourcer(),
triggerManager: trigger.NewManager(0),
policyManager: &test.FakePolicyManager{},
registryManager: registry.NewDefaultManager(),
sourcer: source.NewSourcer(),
triggerManager: trigger.NewManager(0),
}
os.Exit(m.Run())
}
func TestInit(t *testing.T) {
assert.Nil(t, GlobalController.Init())
assert.Nil(t, GlobalController.Init(make(chan struct{})))
}
func TestCreatePolicy(t *testing.T) {

View File

@ -68,12 +68,21 @@ func (f *fakedRegistryManager) Get(id int64) (*model.Registry, error) {
}
return nil, nil
}
func (f *fakedRegistryManager) GetByName(name string) (*model.Registry, error) {
return nil, nil
}
func (f *fakedRegistryManager) GetByURL(url string) (*model.Registry, error) {
return nil, nil
}
func (f *fakedRegistryManager) Update(*model.Registry, ...string) error {
return nil
}
func (f *fakedRegistryManager) Remove(int64) error {
return nil
}
func (f *fakedRegistryManager) HealthCheck() error {
return nil
}
type fakedExecutionManager struct{}

View File

@ -0,0 +1,31 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ng
import "github.com/goharbor/harbor/src/replication/ng/registry"
var (
// RegistryMgr is a global registry manager
RegistryMgr registry.Manager
)
// Init the global variables
func Init() error {
// Init registry manager
RegistryMgr = registry.NewDefaultManager()
return nil
}

View File

@ -15,12 +15,19 @@
package model
import (
"time"
"github.com/goharbor/harbor/src/common/models"
)
// RegistryType indicates the type of registry
type RegistryType string
const (
// RegistryTypeHarbor indicates registry type harbor
RegistryTypeHarbor = "harbor"
)
// Valid indicates whether the RegistryType is a valid value
func (r RegistryType) Valid() bool {
return len(r) > 0
@ -30,6 +37,13 @@ func (r RegistryType) Valid() bool {
// e.g: u/p, OAuth token
type CredentialType string
const (
// CredentialTypeBasic indicates credential by user name, password
CredentialTypeBasic = "basic"
// CredentialTypeOAuth indicates credential by OAuth token
CredentialTypeOAuth = "oauth"
)
// Credential keeps the access key and/or secret for the related registry
type Credential struct {
// Type of the credential
@ -44,18 +58,22 @@ type Credential struct {
// Data required for the secure access way is not contained here.
// DAO layer is not considered here
type Registry struct {
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Type RegistryType `json:"type"`
URL string `json:"url"`
Credential *Credential `json:"credential"`
Insecure bool `json:"insecure"`
Status string `json:"status"`
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Type RegistryType `json:"type"`
URL string `json:"url"`
Credential *Credential `json:"credential"`
Insecure bool `json:"insecure"`
Status string `json:"status"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
}
// RegistryQuery defines the query conditions for listing registries
type RegistryQuery struct {
// Name is name of the registry to query
Name string
models.Pagination
// Pagination specifies the pagination
Pagination *models.Pagination
}

View File

@ -49,6 +49,7 @@ func (c *HealthChecker) Run() {
interval = MinInterval
}
ticker := time.NewTicker(interval)
log.Infof("Start regular health check for registries with interval %v", interval)
for {
select {
case <-ticker.C:

View File

@ -15,17 +15,18 @@
package registry
import (
"errors"
"fmt"
"net/http"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
utilerr "github.com/goharbor/harbor/src/common/utils/error"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/replication/ng/model"
)
// HealthStatus describes whether a target is healthy or not
@ -42,10 +43,22 @@ const (
// Manager defines the methods that a target manager should implement
type Manager interface {
GetRegistry(int64) (*models.RepTarget, error)
AddRegistry(*models.RepTarget) (int64, error)
UpdateRegistry(*models.RepTarget) error
DeleteRegistry(int64) error
// Add new registry
Add(*model.Registry) (int64, error)
// List registries, returns total count, registry list and error
List(...*model.RegistryQuery) (int64, []*model.Registry, error)
// Get the specified registry
Get(int64) (*model.Registry, error)
// GetByName gets registry by name
GetByName(name string) (*model.Registry, error)
// GetByURL gets registry by its URL
GetByURL(url string) (*model.Registry, error)
// Update the registry, the "props" are the properties of registry
// that need to be updated
Update(registry *model.Registry, props ...string) error
// Remove the registry with the specified ID
Remove(int64) error
// HealthCheck checks health status of all registries and update result in database
HealthCheck() error
}
@ -57,87 +70,120 @@ func NewDefaultManager() *DefaultManager {
return &DefaultManager{}
}
// GetRegistry gets a registry by id
func (m *DefaultManager) GetRegistry(id int64) (*models.RepTarget, error) {
target, err := dao.GetRepTarget(id)
// Ensure *DefaultManager has implemented Manager interface.
var _ Manager = (*DefaultManager)(nil)
// Get gets a registry by id
func (m *DefaultManager) Get(id int64) (*model.Registry, error) {
registry, err := dao.GetRegistry(id)
if err != nil {
return nil, err
}
if target == nil {
return nil, fmt.Errorf("target '%d' does not exist", id)
if registry == nil {
return nil, utilerr.KnownError{
Reason: utilerr.ReasonNotFound,
Message: fmt.Sprintf("registry '%d' does not exist", id),
}
}
// decrypt the password
if len(target.Password) > 0 {
key, err := config.SecretKey()
if err != nil {
return nil, err
}
pwd, err := utils.ReversibleDecrypt(target.Password, key)
if err != nil {
return nil, err
}
target.Password = pwd
}
return target, nil
return fromDaoModel(registry)
}
// AddRegistry adds a new registry
func (m *DefaultManager) AddRegistry(registry *models.RepTarget) (int64, error) {
var err error
if len(registry.Password) != 0 {
key, err := config.SecretKey()
if err != nil {
return -1, err
}
registry.Password, err = utils.ReversibleEncrypt(registry.Password, key)
if err != nil {
log.Errorf("failed to encrypt password: %v", err)
return -1, err
}
// GetByName gets a registry by its name
func (m *DefaultManager) GetByName(name string) (*model.Registry, error) {
registry, err := dao.GetRegistryByName(name)
if err != nil {
return nil, err
}
id, err := dao.AddRepTarget(*registry)
if err != nil {
log.Errorf("failed to add registry: %v", err)
if registry == nil {
return nil, nil
}
return fromDaoModel(registry)
}
// GetByURL gets a registry by its URL
func (m *DefaultManager) GetByURL(url string) (*model.Registry, error) {
registry, err := dao.GetRegistryByURL(url)
if err != nil {
return nil, err
}
if registry == nil {
return nil, nil
}
return fromDaoModel(registry)
}
// List lists registries according to query provided.
func (m *DefaultManager) List(query ...*model.RegistryQuery) (int64, []*model.Registry, error) {
var registryQueries []*dao.ListRegistryQuery
for _, q := range query {
// limit being -1 indicates no pagination specified, result in all registries matching name returned.
listQuery := &dao.ListRegistryQuery{
Query: q.Name,
Limit: -1,
}
if q.Pagination != nil {
listQuery.Offset = q.Pagination.Page * q.Pagination.Size
listQuery.Limit = q.Pagination.Size
}
registryQueries = append(registryQueries, listQuery)
}
total, registries, err := dao.ListRegistries(registryQueries...)
if err != nil {
return -1, nil, err
}
var results []*model.Registry
for _, r := range registries {
registry, err := fromDaoModel(r)
if err != nil {
return -1, nil, err
}
results = append(results, registry)
}
return total, results, nil
}
// Add adds a new registry
func (m *DefaultManager) Add(registry *model.Registry) (int64, error) {
r, err := toDaoModel(registry)
if err != nil {
log.Errorf("Convert registry model to dao layer model error: %v", err)
return -1, err
}
id, err := dao.AddRegistry(r)
if err != nil {
log.Errorf("Add registry error: %v", err)
return -1, err
}
return id, nil
}
// UpdateRegistry updates a registry
func (m *DefaultManager) UpdateRegistry(registry *models.RepTarget) error {
// Encrypt the password if set
if len(registry.Password) > 0 {
key, err := config.SecretKey()
if err != nil {
return err
}
pwd, err := utils.ReversibleEncrypt(registry.Password, key)
if err != nil {
return err
}
registry.Password = pwd
}
// Update updates a registry
func (m *DefaultManager) Update(registry *model.Registry, props ...string) error {
// TODO(ChenDe): Only update the given props
return dao.UpdateRepTarget(*registry)
}
// DeleteRegistry deletes a registry
func (m *DefaultManager) DeleteRegistry(id int64) error {
policies, err := dao.GetRepPolicyByTarget(id)
r, err := toDaoModel(registry)
if err != nil {
log.Errorf("Get policies related to registry %d error: %v", id, err)
log.Errorf("Convert registry model to dao layer model error: %v", err)
return err
}
if len(policies) > 0 {
msg := fmt.Sprintf("Can't delete registry with replication policies, %d found", len(policies))
log.Error(msg)
return errors.New(msg)
}
return dao.UpdateRegistry(r)
}
if err = dao.DeleteRepTarget(id); err != nil {
// Remove deletes a registry
func (m *DefaultManager) Remove(id int64) error {
if err := dao.DeleteRegistry(id); err != nil {
log.Errorf("Delete registry %d error: %v", id, err)
return err
}
@ -148,16 +194,19 @@ func (m *DefaultManager) DeleteRegistry(id int64) error {
// HealthCheck checks health status of every registries and update their status. It will check whether a registry
// is reachable and the credential is valid
func (m *DefaultManager) HealthCheck() error {
registries, err := dao.FilterRepTargets("")
_, registries, err := m.List()
if err != nil {
return err
}
errCount := 0
for _, r := range registries {
status, _ := healthStatus(r)
r.Health = string(status)
err := m.UpdateRegistry(r)
status, err := healthStatus(r)
if err != nil {
log.Warningf("Check health status for %s error: %v", r.URL, err)
}
r.Status = string(status)
err = m.Update(r)
if err != nil {
log.Warningf("Update health status for '%s' error: %v", r.URL, err)
errCount++
@ -171,9 +220,19 @@ func (m *DefaultManager) HealthCheck() error {
return nil
}
func healthStatus(r *models.RepTarget) (HealthStatus, error) {
func healthStatus(r *model.Registry) (HealthStatus, error) {
// TODO(ChenDe): Support other credential type like OAuth, for the moment, only basic auth is supported.
if r.Credential.Type != model.CredentialTypeBasic {
return Unknown, fmt.Errorf("unknown credential type '%s', only '%s' supported yet", r.Credential.Type, model.CredentialTypeBasic)
}
// TODO(ChenDe): Support health check for other kinds of registry
if r.Type != model.RegistryTypeHarbor {
return Unknown, fmt.Errorf("unknown registry type '%s'", model.RegistryTypeHarbor)
}
transport := registry.GetHTTPTransport(r.Insecure)
credential := auth.NewBasicAuthCredential(r.Username, r.Password)
credential := auth.NewBasicAuthCredential(r.Credential.AccessKey, r.Credential.AccessSecret)
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: transport,
}, credential)
@ -191,3 +250,91 @@ func healthStatus(r *models.RepTarget) (HealthStatus, error) {
return Healthy, nil
}
// decrypt checks whether access secret is set in the registry, if so, decrypt it.
func decrypt(registry *model.Registry) error {
if len(registry.Credential.AccessSecret) == 0 {
return nil
}
key, err := config.SecretKey()
if err != nil {
return err
}
decrypted, err := utils.ReversibleDecrypt(registry.Credential.AccessSecret, key)
if err != nil {
return err
}
registry.Credential.AccessSecret = decrypted
return nil
}
// encrypt checks whether access secret is set in the registry, if so, encrypt it.
func encrypt(secret string) (string, error) {
if len(secret) == 0 {
return secret, nil
}
key, err := config.SecretKey()
if err != nil {
return "", err
}
encrypted, err := utils.ReversibleEncrypt(secret, key)
if err != nil {
return "", err
}
return encrypted, nil
}
// fromDaoModel converts DAO layer registry model to replication model.
// Also, if access secret is provided, decrypt it.
func fromDaoModel(registry *models.Registry) (*model.Registry, error) {
r := &model.Registry{
ID: registry.ID,
Name: registry.Name,
Description: registry.Description,
Type: model.RegistryType(registry.Type),
URL: registry.URL,
Credential: &model.Credential{
Type: model.CredentialType(registry.CredentialType),
AccessKey: registry.AccessKey,
AccessSecret: registry.AccessSecret,
},
Insecure: registry.Insecure,
Status: registry.Health,
CreationTime: registry.CreationTime,
UpdateTime: registry.UpdateTime,
}
if err := decrypt(r); err != nil {
return nil, err
}
return r, nil
}
// toDaoModel converts registry model from replication to DAO layer model.
// Also, if access secret is provided, encrypt it.
func toDaoModel(registry *model.Registry) (*models.Registry, error) {
encrypted, err := encrypt(registry.Credential.AccessSecret)
if err != nil {
return nil, err
}
return &models.Registry{
ID: registry.ID,
URL: registry.URL,
Name: registry.Name,
CredentialType: string(registry.Credential.Type),
AccessKey: registry.Credential.AccessKey,
AccessSecret: encrypted,
Type: string(registry.Type),
Insecure: registry.Insecure,
Description: registry.Description,
Health: registry.Status,
CreationTime: registry.CreationTime,
UpdateTime: registry.UpdateTime,
}, nil
}

View File

@ -25,6 +25,7 @@ import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/ng/model"
)
// Replication holds information for a replication
@ -32,7 +33,7 @@ type Replication struct {
PolicyID int64
OpUUID string
Candidates []models.FilterItem
Targets []*common_models.RepTarget
Registries []*model.Registry
Operation string
}
@ -68,7 +69,7 @@ func (d *DefaultReplicator) Replicate(replication *Replication) error {
operation = candidate.Operation
}
for _, target := range replication.Targets {
for _, registry := range replication.Registries {
for repository, tags := range repositories {
// create job in database
id, err := dao.AddRepJob(common_models.RepJob{
@ -84,7 +85,7 @@ func (d *DefaultReplicator) Replicate(replication *Replication) error {
// submit job to jobservice
log.Debugf("submiting replication job to jobservice, repository: %s, tags: %v, operation: %s, target: %s",
repository, tags, operation, target.URL)
repository, tags, operation, registry.URL)
job := &job_models.JobData{
Metadata: &job_models.JobMetadata{
JobKind: common_job.JobKindGeneric,
@ -101,20 +102,20 @@ func (d *DefaultReplicator) Replicate(replication *Replication) error {
"src_registry_url": config.InternalCoreURL(),
"src_registry_insecure": false,
"src_token_service_url": config.InternalTokenServiceEndpoint(),
"dst_registry_url": target.URL,
"dst_registry_insecure": target.Insecure,
"dst_registry_username": target.Username,
"dst_registry_password": target.Password,
"dst_registry_url": registry.URL,
"dst_registry_insecure": registry.Insecure,
"dst_registry_username": registry.Credential.AccessKey,
"dst_registry_password": registry.Credential.AccessSecret,
}
} else {
job.Name = common_job.ImageDelete
job.Parameters = map[string]interface{}{
"repository": repository,
"tags": tags,
"dst_registry_url": target.URL,
"dst_registry_insecure": target.Insecure,
"dst_registry_username": target.Username,
"dst_registry_password": target.Password,
"dst_registry_url": registry.URL,
"dst_registry_insecure": registry.Insecure,
"dst_registry_username": registry.Credential.AccessKey,
"dst_registry_password": registry.Credential.AccessSecret,
}
}