Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-29 20:18:05 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #7433 from goharbor/replication_ng

Browse Source 
Merge the replication ng branch to master
This commit is contained in:
Steven Zou 2019-04-18 16:35:45 +08:00 committed by GitHub
commit 16f97326ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
277 changed files with 17556 additions and 11119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,4 +1,3 @@
harbor
!/contrib/helm/harbor
make/docker-compose.yml

906
docs/swagger.yaml
View File

File diff suppressed because it is too large Load Diff

106
make/migrations/postgresql/0004_1.8.0_schema.up.sql
View File

@ -50,4 +50,108 @@ WHERE j.policy_id = p.id AND p.deleted = TRUE;
/*delete replication policy which has been marked as "deleted"*/
DELETE FROM replication_policy AS p
WHERE p.deleted = TRUE;
WHERE p.deleted = TRUE;
/*upgrade the replication_target to registry*/
DROP TRIGGER replication_target_update_time_at_modtime ON replication_target;
ALTER TABLE replication_target RENAME TO registry;
ALTER TABLE registry ALTER COLUMN url TYPE varchar(256);
ALTER TABLE registry ADD COLUMN credential_type varchar(16);
ALTER TABLE registry RENAME COLUMN username TO access_key;
ALTER TABLE registry RENAME COLUMN password TO access_secret;
ALTER TABLE registry ALTER COLUMN access_secret TYPE varchar(1024);
ALTER TABLE registry ADD COLUMN type varchar(32);
ALTER TABLE registry DROP COLUMN target_type;
ALTER TABLE registry ADD COLUMN description text;
ALTER TABLE registry ADD COLUMN health varchar(16);
UPDATE registry SET type='harbor';
UPDATE registry SET credential_type='basic';
/*upgrade the replication_policy*/
ALTER TABLE replication_policy ADD COLUMN creator varchar(256);
ALTER TABLE replication_policy ADD COLUMN src_registry_id int;
/*The predefined filters will be cleared and replaced by "project_name/"+double star.
if harbor is integrated with the external project service, we cannot get the project name by ID,
which means the repilcation policy will match all resources.*/
UPDATE replication_policy r SET filters=(SELECT CONCAT('[{"type":"name","value":"', p.name,'/**"}]') FROM project p WHERE p.project_id=r.project_id);
ALTER TABLE replication_policy RENAME COLUMN target_id TO dest_registry_id;
ALTER TABLE replication_policy ALTER COLUMN dest_registry_id DROP NOT NULL;
ALTER TABLE replication_policy ADD COLUMN dest_namespace varchar(256);
ALTER TABLE replication_policy ADD COLUMN override boolean;
ALTER TABLE replication_policy DROP COLUMN project_id;
ALTER TABLE replication_policy RENAME COLUMN cron_str TO trigger;
DROP TRIGGER replication_immediate_trigger_update_time_at_modtime ON replication_immediate_trigger;
DROP TABLE replication_immediate_trigger;
create table replication_execution (
id SERIAL NOT NULL,
policy_id int NOT NULL,
status varchar(32),
/*the status text may contain error message whose length is very long*/
status_text text,
total int NOT NULL DEFAULT 0,
failed int NOT NULL DEFAULT 0,
succeed int NOT NULL DEFAULT 0,
in_progress int NOT NULL DEFAULT 0,
stopped int NOT NULL DEFAULT 0,
trigger varchar(64),
start_time timestamp default CURRENT_TIMESTAMP,
end_time timestamp NULL,
PRIMARY KEY (id)
);
CREATE INDEX execution_policy ON replication_execution (policy_id);
create table replication_task (
id SERIAL NOT NULL,
execution_id int NOT NULL,
resource_type varchar(64),
src_resource varchar(256),
dst_resource varchar(256),
operation varchar(32),
job_id varchar(64),
status varchar(32),
start_time timestamp default CURRENT_TIMESTAMP,
end_time timestamp NULL,
PRIMARY KEY (id)
);
CREATE INDEX task_execution ON replication_task (execution_id);
/*migrate each replication_job record to one replication_execution and one replication_task record*/
DO $$
DECLARE
job RECORD;
execid integer;
BEGIN
FOR job IN SELECT * FROM replication_job WHERE operation != 'schedule'
LOOP
/*insert one execution record*/
INSERT INTO replication_execution (policy_id, start_time) VALUES (job.policy_id, job.creation_time) RETURNING id INTO execid;
/*insert one task record
doesn't record the tags info in "src_resource" and "dst_resource" as the length
of the tags may longer than the capability of the column*/
INSERT INTO replication_task (execution_id, resource_type, src_resource, dst_resource, operation, job_id, status, start_time, end_time)
VALUES (execid, 'image', job.repository, job.repository, job.operation, job.job_uuid, job.status, job.creation_time, job.update_time);
END LOOP;
END $$;
UPDATE replication_task SET status='Pending' WHERE status='pending';
UPDATE replication_task SET status='InProgress' WHERE status='scheduled';
UPDATE replication_task SET status='InProgress' WHERE status='running';
UPDATE replication_task SET status='Failed' WHERE status='error';
UPDATE replication_task SET status='Succeed' WHERE status='finished';
UPDATE replication_task SET operation='copy' WHERE operation='transfer';
UPDATE replication_task SET operation='deletion' WHERE operation='delete';
/*upgrade the replication_job to replication_schedule_job*/
DELETE FROM replication_job WHERE operation != 'schedule';
ALTER TABLE replication_job RENAME COLUMN job_uuid TO job_id;
ALTER TABLE replication_job DROP COLUMN repository;
ALTER TABLE replication_job DROP COLUMN operation;
ALTER TABLE replication_job DROP COLUMN tags;
ALTER TABLE replication_job DROP COLUMN op_uuid;
DROP INDEX policy;
DROP INDEX poid_uptime;
DROP INDEX poid_status;
DROP TRIGGER replication_job_update_time_at_modtime ON replication_job;
ALTER TABLE replication_job RENAME TO replication_schedule_job;

17
src/Gopkg.lock generated
View File

@ -81,6 +81,14 @@
pruneopts = "UT"
revision = "e87155e8f0c05bf323d0b13470e1b97af0cb5652"
[[projects]]
digest = "1:2aaf2cc045d0219bba79655e4df795b973168c310574669cb75786684f7287d3"
name = "github.com/bmatcuk/doublestar"
packages = ["."]
pruneopts = "UT"
revision = "85a78806aa1b4707d1dbace9be592cf1ece91ab3"
version = "v1.1.1"
[[projects]]
digest = "1:76ca0dfcbf951d1868c7449453981dba9e1f79034706d1500a5a785000f5f222"
name = "github.com/casbin/casbin"
@ -651,11 +659,10 @@
revision = "f534d624797b270e5e46104dc7e2c2d61edbb85d"
[[projects]]
digest = "1:b2a0bdcfc59bed6a64d3ade946f9bf807f8fcd105892d940a008b0b2816babe5"
digest = "1:131682c26796b64f0abb77ac3d85525712706fde0b085aaa7b6d10b4398167cc"
name = "k8s.io/client-go"
packages = [
"kubernetes/scheme",
"kubernetes/typed/authentication/v1beta1",
"pkg/apis/clientauthentication",
"pkg/apis/clientauthentication/v1alpha1",
"pkg/apis/clientauthentication/v1beta1",
@ -714,6 +721,7 @@
"github.com/astaxie/beego/session/redis",
"github.com/astaxie/beego/validation",
"github.com/beego/i18n",
"github.com/bmatcuk/doublestar",
"github.com/casbin/casbin",
"github.com/casbin/casbin/model",
"github.com/casbin/casbin/persist",
@ -729,7 +737,6 @@
"github.com/docker/distribution/reference",
"github.com/docker/distribution/registry/auth/token",
"github.com/docker/distribution/registry/client/auth/challenge",
"github.com/docker/distribution/uuid",
"github.com/docker/libtrust",
"github.com/docker/notary",
"github.com/docker/notary/client",
@ -760,7 +767,9 @@
"gopkg.in/yaml.v2",
"k8s.io/api/authentication/v1beta1",
"k8s.io/apimachinery/pkg/apis/meta/v1",
"k8s.io/client-go/kubernetes/typed/authentication/v1beta1",
"k8s.io/apimachinery/pkg/runtime/schema",
"k8s.io/apimachinery/pkg/runtime/serializer",
"k8s.io/client-go/kubernetes/scheme",
"k8s.io/client-go/rest",
"k8s.io/helm/cmd/helm/search",
"k8s.io/helm/pkg/chartutil",

4
src/Gopkg.toml
View File

@ -123,3 +123,7 @@ ignored = ["github.com/goharbor/harbor/tests*"]
[[constraint]]
name = "k8s.io/api"
version = "kubernetes-1.13.4"
[[constraint]]
name = "github.com/bmatcuk/doublestar"
version = "1.1.1"

37
src/chartserver/handler_manipulation.go
View File

@ -7,7 +7,14 @@ import (
"strings"
"github.com/ghodss/yaml"
"github.com/goharbor/harbor/src/replication"
rep_event "github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
helm_repo "k8s.io/helm/pkg/repo"
"os"
"github.com/goharbor/harbor/src/common/utils/log"
)
// ListCharts gets the chart list under the namespace
@ -63,7 +70,35 @@ func (c *Controller) DeleteChartVersion(namespace, chartName, version string) er
url := fmt.Sprintf("%s/%s/%s", c.APIPrefix(namespace), chartName, version)
return c.apiClient.DeleteContent(url)
err := c.apiClient.DeleteContent(url)
if err != nil {
return err
}
// send notification to replication handler
// Todo: it used as the replacement of webhook, will be removed when webhook to be introduced.
if os.Getenv("UTTEST") != "true" {
go func() {
e := &rep_event.Event{
Type: rep_event.EventTypeChartDelete,
Resource: &model.Resource{
Type: model.ResourceTypeChart,
Deleted: true,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: fmt.Sprintf("%s/%s", namespace, chartName),
},
Vtags: []string{version},
},
},
}
if err := replication.EventHandler.Handle(e); err != nil {
log.Errorf("failed to handle event: %v", err)
}
}()
}
return nil
}
// GetChartVersion returns the summary of the specified chart version.

36
src/chartserver/reverse_proxy.go
View File

@ -12,6 +12,12 @@ import (
"os"
"strconv"
"strings"
"github.com/goharbor/harbor/src/common"
hlog "github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication"
rep_event "github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
)
const (
@ -80,6 +86,36 @@ func director(target *url.URL, cred *Credential, req *http.Request) {
// Modify the http response
func modifyResponse(res *http.Response) error {
// Upload chart success, then to the notification to replication handler
if res.StatusCode == http.StatusCreated {
// 201 and has chart_upload(namespace-repository-version) context
// means this response is for uploading chart success.
chartUpload := res.Request.Context().Value(common.ChartUploadCtxKey).(string)
if chartUpload != "" {
chartUploadSplitted := strings.Split(chartUpload, ":")
if len(chartUploadSplitted) == 3 {
// Todo: it used as the replacement of webhook, will be removed when webhook to be introduced.
go func() {
e := &rep_event.Event{
Type: rep_event.EventTypeChartUpload,
Resource: &model.Resource{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: fmt.Sprintf("%s/%s", chartUploadSplitted[0], chartUploadSplitted[1]),
},
Vtags: []string{chartUploadSplitted[2]},
},
},
}
if err := replication.EventHandler.Handle(e); err != nil {
hlog.Errorf("failed to handle event: %v", err)
}
}()
}
}
}
// Accept cases
// Success or redirect
if res.StatusCode >= http.StatusOK && res.StatusCode <= http.StatusTemporaryRedirect {

4
src/common/const.go
View File

@ -14,6 +14,8 @@
package common
type contextKey string
// const variables
const (
DBAuth = "db_auth"
@ -136,4 +138,6 @@ const (
RobotTokenDuration = "robot_token_duration"
OIDCCallbackPath = "/c/oidc/callback"
ChartUploadCtxKey = contextKey("chart_upload")
)

445
src/common/dao/dao_test.go
View File

@ -26,7 +26,6 @@ import (
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func execUpdate(o orm.Ormer, sql string, params ...interface{}) error {
@ -104,16 +103,11 @@ func cleanByUser(username string) {
o.Rollback()
log.Error(err)
}
err = execUpdate(o, `delete from replication_job where id < 99`)
if err != nil {
log.Error(err)
}
err = execUpdate(o, `delete from replication_policy where id < 99`)
if err != nil {
log.Error(err)
}
err = execUpdate(o, `delete from replication_target where id < 99`)
err = execUpdate(o, `delete from registry where id < 99`)
if err != nil {
log.Error(err)
}
@ -165,8 +159,8 @@ func testForAll(m *testing.M) int {
func clearAll() {
tables := []string{"project_member",
"project_metadata", "access_log", "repository", "replication_policy",
"replication_target", "replication_job", "replication_immediate_trigger", "img_scan_job",
"img_scan_overview", "clair_vuln_timestamp", "project", "harbor_user"}
"registry", "replication_execution", "replication_task", "img_scan_job",
"replication_schedule_job", "img_scan_overview", "clair_vuln_timestamp", "project", "harbor_user"}
for _, t := range tables {
if err := ClearTable(t); err != nil {
log.Errorf("Failed to clear table: %s,error: %v", t, err)
@ -675,439 +669,6 @@ func TestChangeUserProfile(t *testing.T) {
var targetID, policyID, policyID2, policyID3, jobID, jobID2, jobID3 int64
func TestAddRepTarget(t *testing.T) {
target := models.RepTarget{
Name: "test",
URL: "127.0.0.1:5000",
Username: "admin",
Password: "admin",
}
// _, err := AddRepTarget(target)
id, err := AddRepTarget(target)
t.Logf("added target, id: %d", id)
if err != nil {
t.Errorf("Error occurred in AddRepTarget: %v", err)
} else {
targetID = id
}
id2 := id + 99
tgt, err := GetRepTarget(id2)
if err != nil {
t.Errorf("Error occurred in GetTarget: %v, id: %d", err, id2)
}
if tgt != nil {
t.Errorf("There should not be a target with id: %d", id2)
}
tgt, err = GetRepTarget(id)
if err != nil {
t.Errorf("Error occurred in GetTarget: %v, id: %d", err, id)
}
if tgt == nil {
t.Errorf("Unable to find a target with id: %d", id)
}
if tgt.URL != "127.0.0.1:5000" {
t.Errorf("Unexpected url in target: %s, expected 127.0.0.1:5000", tgt.URL)
}
if tgt.Username != "admin" {
t.Errorf("Unexpected username in target: %s, expected admin", tgt.Username)
}
}
func TestGetRepTargetByName(t *testing.T) {
target, err := GetRepTarget(targetID)
if err != nil {
t.Fatalf("failed to get target %d: %v", targetID, err)
}
target2, err := GetRepTargetByName(target.Name)
if err != nil {
t.Fatalf("failed to get target %s: %v", target.Name, err)
}
if target.Name != target2.Name {
t.Errorf("unexpected target name: %s, expected: %s", target2.Name, target.Name)
}
}
func TestGetRepTargetByEndpoint(t *testing.T) {
target, err := GetRepTarget(targetID)
if err != nil {
t.Fatalf("failed to get target %d: %v", targetID, err)
}
target2, err := GetRepTargetByEndpoint(target.URL)
if err != nil {
t.Fatalf("failed to get target %s: %v", target.URL, err)
}
if target.URL != target2.URL {
t.Errorf("unexpected target URL: %s, expected: %s", target2.URL, target.URL)
}
}
func TestUpdateRepTarget(t *testing.T) {
target := &models.RepTarget{
Name: "name",
URL: "http://url",
Username: "username",
Password: "password",
}
id, err := AddRepTarget(*target)
if err != nil {
t.Fatalf("failed to add target: %v", err)
}
defer func() {
if err := DeleteRepTarget(id); err != nil {
t.Logf("failed to delete target %d: %v", id, err)
}
}()
target.ID = id
target.Name = "new_name"
target.URL = "http://new_url"
target.Username = "new_username"
target.Password = "new_password"
if err = UpdateRepTarget(*target); err != nil {
t.Fatalf("failed to update target: %v", err)
}
target, err = GetRepTarget(id)
if err != nil {
t.Fatalf("failed to get target %d: %v", id, err)
}
if target.Name != "new_name" {
t.Errorf("unexpected name: %s, expected: %s", target.Name, "new_name")
}
if target.URL != "http://new_url" {
t.Errorf("unexpected url: %s, expected: %s", target.URL, "http://new_url")
}
if target.Username != "new_username" {
t.Errorf("unexpected username: %s, expected: %s", target.Username, "new_username")
}
if target.Password != "new_password" {
t.Errorf("unexpected password: %s, expected: %s", target.Password, "new_password")
}
}
func TestFilterRepTargets(t *testing.T) {
targets, err := FilterRepTargets("test")
if err != nil {
t.Fatalf("failed to get all targets: %v", err)
}
if len(targets) == 0 {
t.Errorf("unexpected num of targets: %d, expected: %d", len(targets), 1)
}
}
func TestAddRepPolicy(t *testing.T) {
policy := models.RepPolicy{
ProjectID: 1,
TargetID: targetID,
Description: "whatever",
Name: "mypolicy",
}
id, err := AddRepPolicy(policy)
t.Logf("added policy, id: %d", id)
if err != nil {
t.Errorf("Error occurred in AddRepPolicy: %v", err)
} else {
policyID = id
}
p, err := GetRepPolicy(id)
if err != nil {
t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, id)
}
if p == nil {
t.Errorf("Unable to find a policy with id: %d", id)
}
if p.Name != "mypolicy" || p.TargetID != targetID || p.Description != "whatever" {
t.Errorf("The data does not match, expected: Name: mypolicy, TargetID: %d, Description: whatever;\n result: Name: %s, TargetID: %d, Description: %s",
targetID, p.Name, p.TargetID, p.Description)
}
}
func TestGetRepPolicyByTarget(t *testing.T) {
policies, err := GetRepPolicyByTarget(targetID)
if err != nil {
t.Fatalf("failed to get policy according target %d: %v", targetID, err)
}
if len(policies) == 0 {
t.Fatal("unexpected length of policies 0, expected is >0")
}
if policies[0].ID != policyID {
t.Fatalf("unexpected policy: %d, expected: %d", policies[0].ID, policyID)
}
}
func TestGetRepPolicyByProjectAndTarget(t *testing.T) {
policies, err := GetRepPolicyByProjectAndTarget(1, targetID)
if err != nil {
t.Fatalf("failed to get policy according project %d and target %d: %v", 1, targetID, err)
}
if len(policies) == 0 {
t.Fatal("unexpected length of policies 0, expected is >0")
}
if policies[0].ID != policyID {
t.Fatalf("unexpected policy: %d, expected: %d", policies[0].ID, policyID)
}
}
func TestGetRepPolicyByName(t *testing.T) {
policy, err := GetRepPolicy(policyID)
if err != nil {
t.Fatalf("failed to get policy %d: %v", policyID, err)
}
policy2, err := GetRepPolicyByName(policy.Name)
if err != nil {
t.Fatalf("failed to get policy %s: %v", policy.Name, err)
}
if policy.Name != policy2.Name {
t.Errorf("unexpected name: %s, expected: %s", policy2.Name, policy.Name)
}
}
func TestAddRepJob(t *testing.T) {
job := models.RepJob{
Repository: "library/ubuntu",
PolicyID: policyID,
Operation: "transfer",
TagList: []string{"12.01", "14.04", "latest"},
}
id, err := AddRepJob(job)
if err != nil {
t.Errorf("Error occurred in AddRepJob: %v", err)
return
}
jobID = id
j, err := GetRepJob(id)
if err != nil {
t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, id)
return
}
if j == nil {
t.Errorf("Unable to find a job with id: %d", id)
return
}
if j.Status != models.JobPending || j.Repository != "library/ubuntu" || j.PolicyID != policyID || j.Operation != "transfer" || len(j.TagList) != 3 {
t.Errorf("Expected data of job, id: %d, Status: %s, Repository: library/ubuntu, PolicyID: %d, Operation: transfer, taglist length 3"+
"but in returned data:, Status: %s, Repository: %s, Operation: %s, PolicyID: %d, TagList: %v", id, models.JobPending, policyID, j.Status, j.Repository, j.Operation, j.PolicyID, j.TagList)
return
}
}
func TestSetRepJobUUID(t *testing.T) {
uuid := "u-rep-job-uuid"
assert := assert.New(t)
err := SetRepJobUUID(jobID, uuid)
assert.Nil(err)
j, err := GetRepJob(jobID)
assert.Nil(err)
assert.Equal(uuid, j.UUID)
}
func TestUpdateRepJobStatus(t *testing.T) {
err := UpdateRepJobStatus(jobID, models.JobFinished)
if err != nil {
t.Errorf("Error occurred in UpdateRepJobStatus, error: %v, id: %d", err, jobID)
return
}
j, err := GetRepJob(jobID)
if err != nil {
t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, jobID)
}
if j == nil {
t.Errorf("Unable to find a job with id: %d", jobID)
}
if j.Status != models.JobFinished {
t.Errorf("Job's status: %s, expected: %s, id: %d", j.Status, models.JobFinished, jobID)
}
err = UpdateRepJobStatus(jobID, models.JobPending)
if err != nil {
t.Errorf("Error occurred in UpdateRepJobStatus when update it back to status pending, error: %v, id: %d", err, jobID)
return
}
}
func TestGetRepPolicyByProject(t *testing.T) {
p1, err := GetRepPolicyByProject(99)
if err != nil {
t.Errorf("Error occurred in GetRepPolicyByProject:%v, project ID: %d", err, 99)
return
}
if len(p1) > 0 {
t.Errorf("Unexpected length of policy list, expected: 0, in fact: %d, project id: %d", len(p1), 99)
return
}
p2, err := GetRepPolicyByProject(1)
if err != nil {
t.Errorf("Error occuered in GetRepPolicyByProject:%v, project ID: %d", err, 2)
return
}
if len(p2) != 1 {
t.Errorf("Unexpected length of policy list, expected: 1, in fact: %d, project id: %d", len(p2), 1)
return
}
if p2[0].ID != policyID {
t.Errorf("Unexpecred policy id in result, expected: %d, in fact: %d", policyID, p2[0].ID)
return
}
}
func TestGetRepJobs(t *testing.T) {
var policyID int64 = 10000
repository := "repository_for_test_get_rep_jobs"
operation := "operation_for_test"
status := "status_for_test"
now := time.Now().Add(1 * time.Minute)
id, err := AddRepJob(models.RepJob{
PolicyID: policyID,
Repository: repository,
Operation: operation,
Status: status,
CreationTime: now,
UpdateTime: now,
})
require.Nil(t, err)
defer DeleteRepJob(id)
// no query
jobs, err := GetRepJobs()
require.Nil(t, err)
found := false
for _, job := range jobs {
if job.ID == id {
found = true
break
}
}
assert.True(t, found)
// query by policy ID
jobs, err = GetRepJobs(&models.RepJobQuery{
PolicyID: policyID,
})
require.Nil(t, err)
require.Equal(t, 1, len(jobs))
assert.Equal(t, id, jobs[0].ID)
// query by repository
jobs, err = GetRepJobs(&models.RepJobQuery{
Repository: repository,
})
require.Nil(t, err)
require.Equal(t, 1, len(jobs))
assert.Equal(t, id, jobs[0].ID)
// query by operation
jobs, err = GetRepJobs(&models.RepJobQuery{
Operations: []string{operation},
})
require.Nil(t, err)
require.Equal(t, 1, len(jobs))
assert.Equal(t, id, jobs[0].ID)
// query by status
jobs, err = GetRepJobs(&models.RepJobQuery{
Statuses: []string{status},
})
require.Nil(t, err)
require.Equal(t, 1, len(jobs))
assert.Equal(t, id, jobs[0].ID)
// query by creation time
jobs, err = GetRepJobs(&models.RepJobQuery{
StartTime: &now,
})
require.Nil(t, err)
require.Equal(t, 1, len(jobs))
assert.Equal(t, id, jobs[0].ID)
}
func TestDeleteRepJob(t *testing.T) {
err := DeleteRepJob(jobID)
if err != nil {
t.Errorf("Error occurred in DeleteRepJob: %v, id: %d", err, jobID)
return
}
t.Logf("deleted rep job, id: %d", jobID)
j, err := GetRepJob(jobID)
if err != nil {
t.Errorf("Error occurred in GetRepJob:%v", err)
return
}
if j != nil {
t.Errorf("Able to find rep job after deletion, id: %d", jobID)
return
}
}
func TestDeleteRepTarget(t *testing.T) {
err := DeleteRepTarget(targetID)
if err != nil {
t.Errorf("Error occurred in DeleteRepTarget: %v, id: %d", err, targetID)
return
}
t.Logf("deleted target, id: %d", targetID)
tgt, err := GetRepTarget(targetID)
if err != nil {
t.Errorf("Error occurred in GetTarget: %v, id: %d", err, targetID)
}
if tgt != nil {
t.Errorf("Able to find target after deletion, id: %d", targetID)
}
}
func TestGetTotalOfRepPolicies(t *testing.T) {
_, err := GetTotalOfRepPolicies("", 1)
require.Nil(t, err)
}
func TestFilterRepPolicies(t *testing.T) {
_, err := FilterRepPolicies("name", 0, 0, 0)
if err != nil {
t.Fatalf("failed to filter policy: %v", err)
}
}
func TestUpdateRepPolicy(t *testing.T) {
policy := &models.RepPolicy{
ID: policyID,
Name: "new_policy_name",
}
if err := UpdateRepPolicy(policy); err != nil {
t.Fatalf("failed to update policy")
}
}
func TestDeleteRepPolicy(t *testing.T) {
err := DeleteRepPolicy(policyID)
if err != nil {
t.Errorf("Error occurred in DeleteRepPolicy: %v, id: %d", err, policyID)
return
}
t.Logf("delete rep policy, id: %d", policyID)
p, err := GetRepPolicy(policyID)
require.Nil(t, err)
assert.Nil(t, p)
}
func TestGetOrmer(t *testing.T) {
o := GetOrmer()
if o == nil {

424
src/common/dao/replication_job.go
View File

@ -1,424 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"time"
"strings"
"github.com/astaxie/beego/orm"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
)
// AddRepTarget ...
func AddRepTarget(target models.RepTarget) (int64, error) {
o := GetOrmer()
sql := "insert into replication_target (name, url, username, password, insecure, target_type) values (?, ?, ?, ?, ?, ?) RETURNING id"
var targetID int64
err := o.Raw(sql, target.Name, target.URL, target.Username, target.Password, target.Insecure, target.Type).QueryRow(&targetID)
if err != nil {
return 0, err
}
return targetID, nil
}
// GetRepTarget ...
func GetRepTarget(id int64) (*models.RepTarget, error) {
o := GetOrmer()
t := models.RepTarget{ID: id}
err := o.Read(&t)
if err == orm.ErrNoRows {
return nil, nil
}
return &t, err
}
// GetRepTargetByName ...
func GetRepTargetByName(name string) (*models.RepTarget, error) {
o := GetOrmer()
t := models.RepTarget{Name: name}
err := o.Read(&t, "Name")
if err == orm.ErrNoRows {
return nil, nil
}
return &t, err
}
// GetRepTargetByEndpoint ...
func GetRepTargetByEndpoint(endpoint string) (*models.RepTarget, error) {
o := GetOrmer()
t := models.RepTarget{
URL: endpoint,
}
err := o.Read(&t, "URL")
if err == orm.ErrNoRows {
return nil, nil
}
return &t, err
}
// DeleteRepTarget ...
func DeleteRepTarget(id int64) error {
o := GetOrmer()
_, err := o.Delete(&models.RepTarget{ID: id})
return err
}
// UpdateRepTarget ...
func UpdateRepTarget(target models.RepTarget) error {
o := GetOrmer()
sql := `update replication_target
set url = ?, name = ?, username = ?, password = ?, insecure = ?, update_time = ?
where id = ?`
_, err := o.Raw(sql, target.URL, target.Name, target.Username, target.Password, target.Insecure, time.Now(), target.ID).Exec()
return err
}
// FilterRepTargets filters targets by name
func FilterRepTargets(name string) ([]*models.RepTarget, error) {
o := GetOrmer()
var args []interface{}
sql := `select * from replication_target `
if len(name) != 0 {
sql += `where name like ? `
args = append(args, "%"+Escape(name)+"%")
}
sql += `order by creation_time`
var targets []*models.RepTarget
if _, err := o.Raw(sql, args).QueryRows(&targets); err != nil {
return nil, err
}
return targets, nil
}
// AddRepPolicy ...
func AddRepPolicy(policy models.RepPolicy) (int64, error) {
o := GetOrmer()
sql := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, creation_time, update_time, filters, replicate_deletion)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id`
params := []interface{}{}
now := time.Now()
params = append(params, policy.Name, policy.ProjectID, policy.TargetID, true,
policy.Description, policy.Trigger, now, now, policy.Filters,
policy.ReplicateDeletion)
var policyID int64
err := o.Raw(sql, params...).QueryRow(&policyID)
if err != nil {
return 0, err
}
return policyID, nil
}
// GetRepPolicy ...
func GetRepPolicy(id int64) (*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where id = ? and deleted = false`
var policy models.RepPolicy
if err := o.Raw(sql, id).QueryRow(&policy); err != nil {
if err == orm.ErrNoRows {
return nil, nil
}
return nil, err
}
return &policy, nil
}
// GetTotalOfRepPolicies returns the total count of replication policies
func GetTotalOfRepPolicies(name string, projectID int64) (int64, error) {
qs := GetOrmer().QueryTable(&models.RepPolicy{}).Filter("deleted", false)
if len(name) != 0 {
qs = qs.Filter("name__icontains", name)
}
if projectID != 0 {
qs = qs.Filter("project_id", projectID)
}
return qs.Count()
}
// FilterRepPolicies filters policies by name and project ID
func FilterRepPolicies(name string, projectID, page, pageSize int64) ([]*models.RepPolicy, error) {
o := GetOrmer()
var args []interface{}
sql := `select rp.id, rp.project_id, rp.target_id,
rt.name as target_name, rp.name, rp.description,
rp.cron_str, rp.filters, rp.replicate_deletion,
rp.creation_time, rp.update_time,
count(rj.status) as error_job_count
from replication_policy rp
left join replication_target rt on rp.target_id=rt.id
left join replication_job rj on rp.id=rj.policy_id and (rj.status='error'
or rj.status='retrying')
where rp.deleted = false `
if len(name) != 0 && projectID != 0 {
sql += `and rp.name like ? and rp.project_id = ? `
args = append(args, "%"+Escape(name)+"%")
args = append(args, projectID)
} else if len(name) != 0 {
sql += `and rp.name like ? `
args = append(args, "%"+Escape(name)+"%")
} else if projectID != 0 {
sql += `and rp.project_id = ? `
args = append(args, projectID)
}
sql += `group by rt.name, rp.id order by rp.creation_time`
if page > 0 && pageSize > 0 {
sql += ` limit ? offset ?`
args = append(args, pageSize, (page-1)*pageSize)
}
var policies []*models.RepPolicy
if _, err := o.Raw(sql, args).QueryRows(&policies); err != nil {
return nil, err
}
return policies, nil
}
// GetRepPolicyByName ...
func GetRepPolicyByName(name string) (*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where deleted = false and name = ?`
var policy models.RepPolicy
if err := o.Raw(sql, name).QueryRow(&policy); err != nil {
if err == orm.ErrNoRows {
return nil, nil
}
return nil, err
}
return &policy, nil
}
// GetRepPolicyByProject ...
func GetRepPolicyByProject(projectID int64) ([]*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where deleted = false and project_id = ?`
var policies []*models.RepPolicy
if _, err := o.Raw(sql, projectID).QueryRows(&policies); err != nil {
return nil, err
}
return policies, nil
}
// GetRepPolicyByTarget ...
func GetRepPolicyByTarget(targetID int64) ([]*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where deleted = false and target_id = ?`
var policies []*models.RepPolicy
if _, err := o.Raw(sql, targetID).QueryRows(&policies); err != nil {
return nil, err
}
return policies, nil
}
// GetRepPolicyByProjectAndTarget ...
func GetRepPolicyByProjectAndTarget(projectID, targetID int64) ([]*models.RepPolicy, error) {
o := GetOrmer()
sql := `select * from replication_policy where deleted = false and project_id = ? and target_id = ?`
var policies []*models.RepPolicy
if _, err := o.Raw(sql, projectID, targetID).QueryRows(&policies); err != nil {
return nil, err
}
return policies, nil
}
// UpdateRepPolicy ...
func UpdateRepPolicy(policy *models.RepPolicy) error {
o := GetOrmer()
sql := `update replication_policy
set project_id = ?, target_id = ?, name = ?, description = ?, cron_str = ?, filters = ?, replicate_deletion = ?, update_time = ?
where id = ?`
_, err := o.Raw(sql, policy.ProjectID, policy.TargetID, policy.Name, policy.Description, policy.Trigger, policy.Filters, policy.ReplicateDeletion, time.Now(), policy.ID).Exec()
return err
}
// DeleteRepPolicy ...
func DeleteRepPolicy(id int64) error {
_, err := GetOrmer().Delete(&models.RepPolicy{
ID: id,
})
return err
}
// AddRepJob ...
func AddRepJob(job models.RepJob) (int64, error) {
o := GetOrmer()
if len(job.Status) == 0 {
job.Status = models.JobPending
}
if len(job.TagList) > 0 {
job.Tags = strings.Join(job.TagList, ",")
}
return o.Insert(&job)
}
// GetRepJob ...
func GetRepJob(id int64) (*models.RepJob, error) {
o := GetOrmer()
j := models.RepJob{ID: id}
err := o.Read(&j)
if err == orm.ErrNoRows {
return nil, nil
}
genTagListForJob(&j)
return &j, nil
}
// GetTotalCountOfRepJobs ...
func GetTotalCountOfRepJobs(query ...*models.RepJobQuery) (int64, error) {
qs := repJobQueryConditions(query...)
return qs.Count()
}
// GetRepJobs ...
func GetRepJobs(query ...*models.RepJobQuery) ([]*models.RepJob, error) {
jobs := []*models.RepJob{}
qs := repJobQueryConditions(query...)
if len(query) > 0 && query[0] != nil {
qs = paginateForQuerySetter(qs, query[0].Page, query[0].Size)
}
qs = qs.OrderBy("-UpdateTime")
if _, err := qs.All(&jobs); err != nil {
return jobs, err
}
genTagListForJob(jobs...)
return jobs, nil
}
func repJobQueryConditions(query ...*models.RepJobQuery) orm.QuerySeter {
qs := GetOrmer().QueryTable(new(models.RepJob))
if len(query) == 0 || query[0] == nil {
return qs
}
q := query[0]
if q.PolicyID != 0 {
qs = qs.Filter("PolicyID", q.PolicyID)
}
if len(q.OpUUID) > 0 {
qs = qs.Filter("OpUUID__exact", q.OpUUID)
}
if len(q.Repository) > 0 {
qs = qs.Filter("Repository__icontains", q.Repository)
}
if len(q.Statuses) > 0 {
qs = qs.Filter("Status__in", q.Statuses)
}
if len(q.Operations) > 0 {
qs = qs.Filter("Operation__in", q.Operations)
}
if q.StartTime != nil {
qs = qs.Filter("CreationTime__gte", q.StartTime)
}
if q.EndTime != nil {
qs = qs.Filter("CreationTime__lte", q.EndTime)
}
return qs
}
// DeleteRepJob ...
func DeleteRepJob(id int64) error {
o := GetOrmer()
_, err := o.Delete(&models.RepJob{ID: id})
return err
}
// DeleteRepJobs deletes replication jobs by policy ID
func DeleteRepJobs(policyID int64) error {
_, err := GetOrmer().QueryTable(&models.RepJob{}).Filter("PolicyID", policyID).Delete()
return err
}
// UpdateRepJobStatus ...
func UpdateRepJobStatus(id int64, status string) error {
o := GetOrmer()
j := models.RepJob{
ID: id,
Status: status,
UpdateTime: time.Now(),
}
n, err := o.Update(&j, "Status", "UpdateTime")
if n == 0 {
log.Warningf("no records are updated when updating replication job %d", id)
}
return err
}
// SetRepJobUUID ...
func SetRepJobUUID(id int64, uuid string) error {
o := GetOrmer()
j := models.RepJob{
ID: id,
UUID: uuid,
}
n, err := o.Update(&j, "UUID")
if n == 0 {
log.Warningf("no records are updated when updating replication job %d", id)
}
return err
}
func genTagListForJob(jobs ...*models.RepJob) {
for _, j := range jobs {
if len(j.Tags) > 0 {
j.TagList = strings.Split(j.Tags, ",")
}
}
}

56
src/common/dao/replication_job_test.go
View File

@ -1,56 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"testing"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDeleteRepJobs(t *testing.T) {
var policyID int64 = 999
_, err := AddRepJob(models.RepJob{
PolicyID: policyID,
Repository: "library/hello-world",
Operation: "delete",
Status: "success",
})
require.Nil(t, err)
_, err = AddRepJob(models.RepJob{
PolicyID: policyID,
Repository: "library/hello-world",
Operation: "delete",
Status: "success",
})
require.Nil(t, err)
jobs, err := GetRepJobs(&models.RepJobQuery{
PolicyID: policyID,
})
require.Nil(t, err)
require.Equal(t, 2, len(jobs))
err = DeleteRepJobs(policyID)
require.Nil(t, err)
jobs, err = GetRepJobs(&models.RepJobQuery{
PolicyID: policyID,
})
require.Nil(t, err)
assert.Equal(t, 0, len(jobs))
}

2
src/common/dao/sqlite.go
View File

@ -18,7 +18,7 @@ import (
"fmt"
"github.com/astaxie/beego/orm"
_ "github.com/mattn/go-sqlite3" // register sqlite driver
// _ "github.com/mattn/go-sqlite3" // register sqlite driver
)
type sqlite struct {

71
src/common/dao/watch_item.go
View File

@ -1,71 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"time"
"github.com/goharbor/harbor/src/common/models"
)
// DefaultDatabaseWatchItemDAO is an instance of DatabaseWatchItemDAO
var DefaultDatabaseWatchItemDAO WatchItemDAO = &DatabaseWatchItemDAO{}
// WatchItemDAO defines operations about WatchItem
type WatchItemDAO interface {
Add(*models.WatchItem) (int64, error)
DeleteByPolicyID(int64) error
Get(namespace, operation string) ([]models.WatchItem, error)
}
// DatabaseWatchItemDAO implements interface WatchItemDAO for database
type DatabaseWatchItemDAO struct{}
// Add a WatchItem
func (d *DatabaseWatchItemDAO) Add(item *models.WatchItem) (int64, error) {
o := GetOrmer()
var triggerID int64
now := time.Now()
sql := "insert into replication_immediate_trigger (policy_id, namespace, on_deletion, on_push, creation_time, update_time) values (?, ?, ?, ?, ?, ?) RETURNING id"
err := o.Raw(sql, item.PolicyID, item.Namespace, item.OnDeletion, item.OnPush, now, now).QueryRow(&triggerID)
if err != nil {
return 0, err
}
return triggerID, nil
}
// DeleteByPolicyID deletes the WatchItem specified by policy ID
func (d *DatabaseWatchItemDAO) DeleteByPolicyID(policyID int64) error {
_, err := GetOrmer().QueryTable(&models.WatchItem{}).Filter("PolicyID", policyID).Delete()
return err
}
// Get returns WatchItem list according to the namespace and operation
func (d *DatabaseWatchItemDAO) Get(namespace, operation string) ([]models.WatchItem, error) {
qs := GetOrmer().QueryTable(&models.WatchItem{}).Filter("Namespace", namespace)
if operation == "push" {
qs = qs.Filter("OnPush", true)
} else if operation == "delete" {
qs = qs.Filter("OnDeletion", true)
}
items := []models.WatchItem{}
_, err := qs.All(&items)
return items, err
}

71
src/common/dao/watch_item_test.go
View File

@ -1,71 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dao
import (
"testing"
"github.com/goharbor/harbor/src/common/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMethodsOfWatchItem(t *testing.T) {
targetID, err := AddRepTarget(models.RepTarget{
Name: "test_target_for_watch_item",
URL: "http://127.0.0.1",
})
require.Nil(t, err)
defer DeleteRepTarget(targetID)
policyID, err := AddRepPolicy(models.RepPolicy{
Name: "test_policy_for_watch_item",
ProjectID: 1,
TargetID: targetID,
})
require.Nil(t, err)
defer DeleteRepPolicy(policyID)
item := &models.WatchItem{
PolicyID: policyID,
Namespace: "library",
OnPush: false,
OnDeletion: true,
}
// test Add
id, err := DefaultDatabaseWatchItemDAO.Add(item)
require.Nil(t, err)
// test Get: operation-push
items, err := DefaultDatabaseWatchItemDAO.Get("library", "push")
require.Nil(t, err)
assert.Equal(t, 0, len(items))
// test Get: operation-delete
items, err = DefaultDatabaseWatchItemDAO.Get("library", "delete")
require.Nil(t, err)
assert.Equal(t, 1, len(items))
assert.Equal(t, id, items[0].ID)
assert.Equal(t, "library", items[0].Namespace)
assert.True(t, items[0].OnDeletion)
// test DeleteByPolicyID
err = DefaultDatabaseWatchItemDAO.DeleteByPolicyID(policyID)
require.Nil(t, err)
items, err = DefaultDatabaseWatchItemDAO.Get("library", "delete")
require.Nil(t, err)
assert.Equal(t, 0, len(items))
}

14
src/common/http/client.go
View File

@ -95,12 +95,16 @@ func (c *Client) Head(url string) error {
func (c *Client) Post(url string, v ...interface{}) error {
var reader io.Reader
if len(v) > 0 {
data, err := json.Marshal(v[0])
if err != nil {
return err
}
if r, ok := v[0].(io.Reader); ok {
reader = r
} else {
data, err := json.Marshal(v[0])
if err != nil {
return err
}
reader = bytes.NewReader(data)
reader = bytes.NewReader(data)
}
}
req, err := http.NewRequest(http.MethodPost, url, reader)

10
src/common/job/const.go
View File

@ -5,14 +5,12 @@ const (
ImageScanJob = "IMAGE_SCAN"
// ImageScanAllJob is the name of "scanall" job in job service
ImageScanAllJob = "IMAGE_SCAN_ALL"
// ImageTransfer : the name of image transfer job in job service
ImageTransfer = "IMAGE_TRANSFER"
// ImageDelete : the name of image delete job in job service
ImageDelete = "IMAGE_DELETE"
// ImageReplicate : the name of image replicate job in job service
ImageReplicate = "IMAGE_REPLICATE"
// ImageGC the name of image garbage collection job in job service
ImageGC = "IMAGE_GC"
// Replication : the name of the replication job in job service
Replication = "REPLICATION"
// ReplicationScheduler : the name of the replication scheduler job in job service
ReplicationScheduler = "IMAGE_REPLICATE"
// JobKindGeneric : Kind of generic job
JobKindGeneric = "Generic"

2
src/common/job/test/server.go
View File

@ -27,7 +27,7 @@ func currPath() string {
return path.Dir(f)
}
// NewJobServiceServer
// NewJobServiceServer ...
func NewJobServiceServer() *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc(fmt.Sprintf("%s/%s/log", jobsPrefix, jobUUID),

5
src/common/models/base.go
View File

@ -19,9 +19,7 @@ import (
)
func init() {
orm.RegisterModel(new(RepTarget),
new(RepPolicy),
new(RepJob),
orm.RegisterModel(
new(User),
new(Project),
new(Role),
@ -30,7 +28,6 @@ func init() {
new(RepoRecord),
new(ImgScanOverview),
new(ClairVulnTimestamp),
new(WatchItem),
new(ProjectMetadata),
new(ConfigEntry),
new(Label),

136
src/common/models/replication_job.go
View File

@ -1,136 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"time"
"github.com/astaxie/beego/validation"
"github.com/goharbor/harbor/src/common/utils"
)
const (
// RepOpTransfer represents the operation of a job to transfer repository to a remote registry/harbor instance.
RepOpTransfer string = "transfer"
// RepOpDelete represents the operation of a job to remove repository from a remote registry/harbor instance.
RepOpDelete string = "delete"
// RepOpSchedule represents the operation of a job to schedule the real replication process
RepOpSchedule string = "schedule"
// RepTargetTable is the table name for replication targets
RepTargetTable = "replication_target"
// RepJobTable is the table name for replication jobs
RepJobTable = "replication_job"
// RepPolicyTable is table name for replication policies
RepPolicyTable = "replication_policy"
)
// RepPolicy is the model for a replication policy, which associate to a project and a target (destination)
type RepPolicy struct {
ID int64 `orm:"pk;auto;column(id)"`
ProjectID int64 `orm:"column(project_id)" `
TargetID int64 `orm:"column(target_id)"`
Name string `orm:"column(name)"`
Description string `orm:"column(description)"`
Trigger string `orm:"column(cron_str)"`
Filters string `orm:"column(filters)"`
ReplicateDeletion bool `orm:"column(replicate_deletion)"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add"`
UpdateTime time.Time `orm:"column(update_time);auto_now"`
Deleted bool `orm:"column(deleted)"`
}
// RepJob is the model for a replication job, which is the execution unit on job service, currently it is used to transfer/remove
// a repository to/from a remote registry instance.
type RepJob struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
Status string `orm:"column(status)" json:"status"`
Repository string `orm:"column(repository)" json:"repository"`
PolicyID int64 `orm:"column(policy_id)" json:"policy_id"`
OpUUID string `orm:"column(op_uuid)" json:"op_uuid"`
Operation string `orm:"column(operation)" json:"operation"`
Tags string `orm:"column(tags)" json:"-"`
TagList []string `orm:"-" json:"tags"`
UUID string `orm:"column(job_uuid)" json:"-"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// RepTarget is the model for a replication targe, i.e. destination, which wraps the endpoint URL and username/password of a remote registry.
type RepTarget struct {
ID int64 `orm:"pk;auto;column(id)" json:"id"`
URL string `orm:"column(url)" json:"endpoint"`
Name string `orm:"column(name)" json:"name"`
Username string `orm:"column(username)" json:"username"`
Password string `orm:"column(password)" json:"password"`
Type int `orm:"column(target_type)" json:"type"`
Insecure bool `orm:"column(insecure)" json:"insecure"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
// Valid ...
func (r *RepTarget) Valid(v *validation.Validation) {
if len(r.Name) == 0 {
v.SetError("name", "can not be empty")
}
if len(r.Name) > 64 {
v.SetError("name", "max length is 64")
}
url, err := utils.ParseEndpoint(r.URL)
if err != nil {
v.SetError("endpoint", err.Error())
} else {
// Prevent SSRF security issue #3755
r.URL = url.Scheme + "://" + url.Host + url.Path
if len(r.URL) > 64 {
v.SetError("endpoint", "max length is 64")
}
}
// password is encoded using base64, the length of this field
// in DB is 64, so the max length in request is 48
if len(r.Password) > 48 {
v.SetError("password", "max length is 48")
}
}
// TableName is required by by beego orm to map RepTarget to table replication_target
func (r *RepTarget) TableName() string {
return RepTargetTable
}
// TableName is required by by beego orm to map RepJob to table replication_job
func (r *RepJob) TableName() string {
return RepJobTable
}
// TableName is required by by beego orm to map RepPolicy to table replication_policy
func (r *RepPolicy) TableName() string {
return RepPolicyTable
}
// RepJobQuery holds query conditions for replication job
type RepJobQuery struct {
PolicyID int64
OpUUID string
Repository string
Statuses []string
Operations []string
StartTime *time.Time
EndTime *time.Time
Pagination
}

6
src/common/rbac/const.go
View File

@ -42,8 +42,10 @@ const (
ResourceLog = Resource("log")
ResourceMember = Resource("member")
ResourceMetadata = Resource("metadata")
ResourceReplication = Resource("replication")
ResourceReplicationJob = Resource("replication-job")
ResourceReplication = Resource("replication") // TODO remove
ResourceReplicationJob = Resource("replication-job") // TODO remove
ResourceReplicationExecution = Resource("replication-execution")
ResourceReplicationTask = Resource("replication-task")
ResourceRepository = Resource("repository")
ResourceRepositoryLabel = Resource("repository-label")
ResourceRepositoryTag = Resource("repository-tag")

12
src/common/rbac/project/util.go
View File

@ -75,6 +75,18 @@ var (
{Resource: rbac.ResourceReplicationJob, Action: rbac.ActionRead},
{Resource: rbac.ResourceReplicationJob, Action: rbac.ActionList},
{Resource: rbac.ResourceReplicationExecution, Action: rbac.ActionRead},
{Resource: rbac.ResourceReplicationExecution, Action: rbac.ActionList},
{Resource: rbac.ResourceReplicationExecution, Action: rbac.ActionCreate},
{Resource: rbac.ResourceReplicationExecution, Action: rbac.ActionUpdate},
{Resource: rbac.ResourceReplicationExecution, Action: rbac.ActionDelete},
{Resource: rbac.ResourceReplicationTask, Action: rbac.ActionRead},
{Resource: rbac.ResourceReplicationTask, Action: rbac.ActionList},
{Resource: rbac.ResourceReplicationTask, Action: rbac.ActionCreate},
{Resource: rbac.ResourceReplicationTask, Action: rbac.ActionUpdate},
{Resource: rbac.ResourceReplicationTask, Action: rbac.ActionDelete},
{Resource: rbac.ResourceLabel, Action: rbac.ActionCreate},
{Resource: rbac.ResourceLabel, Action: rbac.ActionRead},
{Resource: rbac.ResourceLabel, Action: rbac.ActionUpdate},

7
src/common/utils/registry/auth/credential.go
View File

@ -38,7 +38,12 @@ func NewBasicAuthCredential(username, password string) Credential {
}
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
req.SetBasicAuth(b.username, b.password)
// only add the authentication info when the username isn't empty
// the logic is needed for requesting resources from docker hub's
// public repositories
if len(b.username) > 0 {
req.SetBasicAuth(b.username, b.password)
}
}
// implement github.com/goharbor/harbor/src/common/http/modifier.Modifier

2
src/common/utils/registry/auth/tokenauthorizer.go
View File

@ -278,7 +278,7 @@ func NewStandardTokenAuthorizer(client *http.Client, credential Credential,
// 1. performance issue
// 2. the realm field returned by registry is an IP which can not reachable
// inside Harbor
if len(customizedTokenService) > 0 {
if len(customizedTokenService) > 0 && len(customizedTokenService[0]) > 0 {
generator.realm = customizedTokenService[0]
}

18
src/common/utils/registry/registry.go
View File

@ -157,3 +157,21 @@ func (r *Registry) Ping() error {
Message: string(b),
}
}
// PingSimple checks whether the registry is available. It checks the connectivity and certificate (if TLS enabled)
// only, regardless of credential.
func (r *Registry) PingSimple() error {
err := r.Ping()
if err == nil {
return nil
}
httpErr, ok := err.(*commonhttp.Error)
if !ok {
return err
}
if httpErr.Code == http.StatusUnauthorized ||
httpErr.Code == http.StatusForbidden {
return nil
}
return httpErr
}

45
src/common/utils/test/policy_manager.go
View File

@ -1,45 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/models"
)
type FakePolicyManager struct {
}
func (f *FakePolicyManager) GetPolicies(query models.QueryParameter) (*models.ReplicationPolicyQueryResult, error) {
return &models.ReplicationPolicyQueryResult{}, nil
}
func (f *FakePolicyManager) GetPolicy(id int64) (models.ReplicationPolicy, error) {
return models.ReplicationPolicy{
ID: 1,
Trigger: &models.Trigger{
Kind: replication.TriggerKindManual,
},
}, nil
}
func (f *FakePolicyManager) CreatePolicy(policy models.ReplicationPolicy) (int64, error) {
return 1, nil
}
func (f *FakePolicyManager) UpdatePolicy(models.ReplicationPolicy) error {
return nil
}
func (f *FakePolicyManager) RemovePolicy(int64) error {
return nil
}

26
src/common/utils/test/replication_controllter.go
View File

@ -1,26 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
type FakeReplicatoinController struct {
FakePolicyManager
}
func (f *FakeReplicatoinController) Init() error {
return nil
}
func (f *FakeReplicatoinController) Replicate(policyID int64, metadata ...map[string]interface{}) error {
return nil
}

65
src/common/utils/test/watch_item.go
View File

@ -1,65 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package test
import (
"github.com/goharbor/harbor/src/common/models"
)
// FakeWatchItemDAO is the fake implement for the dao.WatchItemDAO
type FakeWatchItemDAO struct {
items []models.WatchItem
}
// Add ...
func (f *FakeWatchItemDAO) Add(item *models.WatchItem) (int64, error) {
f.items = append(f.items, *item)
return int64(len(f.items) + 1), nil
}
// DeleteByPolicyID : delete the WatchItem specified by policy ID
func (f *FakeWatchItemDAO) DeleteByPolicyID(policyID int64) error {
for i, item := range f.items {
if item.PolicyID == policyID {
f.items = append(f.items[:i], f.items[i+1:]...)
break
}
}
return nil
}
// Get returns WatchItem list according to the namespace and operation
func (f *FakeWatchItemDAO) Get(namespace, operation string) ([]models.WatchItem, error) {
items := []models.WatchItem{}
for _, item := range f.items {
if item.Namespace != namespace {
continue
}
if operation == "push" {
if item.OnPush {
items = append(items, item)
}
}
if operation == "delete" {
if item.OnDeletion {
items = append(items, item)
}
}
}
return items, nil
}

23
src/core/api/chart_repository.go
View File

@ -2,6 +2,7 @@ package api
import (
"bytes"
"context"
"errors"
"fmt"
"io"
@ -297,8 +298,28 @@ func (cra *ChartRepositoryAPI) UploadChartVersion() {
}
}
// set namespace/repository/version for replication event.
_, header, err := cra.GetFile(formFieldNameForChart)
if err != nil {
cra.SendInternalServerError(err)
return
}
req := cra.Ctx.Request
charFileName := header.Filename
if !strings.HasSuffix(charFileName, ".tgz") {
cra.SendInternalServerError(fmt.Errorf("chart file expected %s to end with .tgz", charFileName))
return
}
charFileName = strings.TrimSuffix(charFileName, ".tgz")
// colon cannot be used as namespace
charFileName = strings.Replace(charFileName, "-", ":", -1)
// value sample: library:redis:4.0.3 (namespace:repository:version)
ctx := context.WithValue(cra.Ctx.Request.Context(), common.ChartUploadCtxKey, cra.namespace+":"+charFileName)
req = req.WithContext(ctx)
// Directly proxy to the backend
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, cra.Ctx.Request)
chartController.ProxyTraffic(cra.Ctx.ResponseWriter, req)
}
// UploadChartProvFile handles POST /api/:repo/prov

42
src/core/api/dataprepare_test.go
View File

@ -19,18 +19,18 @@ import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
rep_dao "github.com/goharbor/harbor/src/replication/dao"
rep_models "github.com/goharbor/harbor/src/replication/dao/models"
)
const (
// Prepare Test info
TestUserName = "testUser0001"
TestUserPwd = "testUser0001"
TestUserEmail = "testUser0001@mydomain.com"
TestProName = "testProject0001"
TestTargetName = "testTarget0001"
TestRepoName = "testRepo0001"
AdminName = "admin"
DefaultProjectName = "library"
TestUserName = "testUser0001"
TestUserPwd = "testUser0001"
TestUserEmail = "testUser0001@mydomain.com"
TestProName = "testProject0001"
TestRegistryName = "testRegistry0001"
TestRepoName = "testRepo0001"
)
func CommonAddUser() {
@ -83,25 +83,25 @@ func CommonDelProject() {
_ = dao.DeleteProject(commonProject.ProjectID)
}
func CommonAddTarget() {
func CommonAddRegistry() {
endPoint := os.Getenv("REGISTRY_URL")
commonTarget := &models.RepTarget{
URL: endPoint,
Name: TestTargetName,
Username: adminName,
Password: adminPwd,
commonRegistry := &rep_models.Registry{
URL: endPoint,
Name: TestRegistryName,
AccessKey: adminName,
AccessSecret: adminPwd,
}
_, _ = dao.AddRepTarget(*commonTarget)
_, _ = rep_dao.AddRegistry(commonRegistry)
}
func CommonGetTarget() int {
target, _ := dao.GetRepTargetByName(TestTargetName)
return int(target.ID)
func CommonGetRegistry() int {
registry, _ := rep_dao.GetRegistryByName(TestRegistryName)
return int(registry.ID)
}
func CommonDelTarget() {
target, _ := dao.GetRepTargetByName(TestTargetName)
_ = dao.DeleteRepTarget(target.ID)
func CommonDelRegistry() {
registry, _ := rep_dao.GetRegistryByName(TestRegistryName)
_ = rep_dao.DeleteRegistry(registry.ID)
}
func CommonAddRepository() {

213
src/core/api/harborapi_test.go
View File

@ -27,25 +27,20 @@ import (
"runtime"
"strconv"
"github.com/astaxie/beego"
"github.com/dghubble/sling"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/job/test"
"github.com/goharbor/harbor/src/common/models"
testutils "github.com/goharbor/harbor/src/common/utils/test"
api_models "github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/filter"
"github.com/goharbor/harbor/tests/apitests/apilib"
// "strconv"
// "strings"
"github.com/astaxie/beego"
"github.com/dghubble/sling"
"github.com/goharbor/harbor/src/common/dao"
apimodels "github.com/goharbor/harbor/src/core/api/models"
_ "github.com/goharbor/harbor/src/core/auth/db"
_ "github.com/goharbor/harbor/src/core/auth/ldap"
"github.com/goharbor/harbor/src/replication/core"
_ "github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/filter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/tests/apitests/apilib"
)
const (
@ -128,14 +123,9 @@ func init() {
beego.Router("/api/repositories/*/tags/:tag/manifest", &RepositoryAPI{}, "get:GetManifests")
beego.Router("/api/repositories/*/signatures", &RepositoryAPI{}, "get:GetSignatures")
beego.Router("/api/repositories/top", &RepositoryAPI{}, "get:GetTopRepos")
beego.Router("/api/targets/", &TargetAPI{}, "get:List")
beego.Router("/api/targets/", &TargetAPI{}, "post:Post")
beego.Router("/api/targets/:id([0-9]+)", &TargetAPI{})
beego.Router("/api/targets/:id([0-9]+)/policies/", &TargetAPI{}, "get:ListPolicies")
beego.Router("/api/targets/ping", &TargetAPI{}, "post:Ping")
beego.Router("/api/policies/replication/:id([0-9]+)", &RepPolicyAPI{})
beego.Router("/api/policies/replication", &RepPolicyAPI{}, "get:List")
beego.Router("/api/policies/replication", &RepPolicyAPI{}, "post:Post;delete:Delete")
beego.Router("/api/registries", &RegistryAPI{}, "get:List;post:Post")
beego.Router("/api/registries/ping", &RegistryAPI{}, "post:Ping")
beego.Router("/api/registries/:id([0-9]+)", &RegistryAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/systeminfo", &SystemInfoAPI{}, "get:GetGeneralInfo")
beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo")
beego.Router("/api/systeminfo/getcert", &SystemInfoAPI{}, "get:GetCert")
@ -146,7 +136,6 @@ func init() {
beego.Router("/api/configurations", &ConfigAPI{})
beego.Router("/api/configs", &ConfigAPI{}, "get:GetInternalConfig")
beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")
beego.Router("/api/replications", &ReplicationAPI{})
beego.Router("/api/labels", &LabelAPI{}, "post:Post;get:List")
beego.Router("/api/labels/:id([0-9]+", &LabelAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/labels/:id([0-9]+)/resources", &LabelAPI{}, "get:ListResources")
@ -159,6 +148,15 @@ func init() {
beego.Router("/api/projects/:pid([0-9]+)/robots/", &RobotAPI{}, "post:Post;get:List")
beego.Router("/api/projects/:pid([0-9]+)/robots/:id([0-9]+)", &RobotAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/replication/adapters", &ReplicationAdapterAPI{}, "get:List")
beego.Router("/api/replication/executions", &ReplicationOperationAPI{}, "get:ListExecutions;post:CreateExecution")
beego.Router("/api/replication/executions/:id([0-9]+)", &ReplicationOperationAPI{}, "get:GetExecution;put:StopExecution")
beego.Router("/api/replication/executions/:id([0-9]+)/tasks", &ReplicationOperationAPI{}, "get:ListTasks")
beego.Router("/api/replication/executions/:id([0-9]+)/tasks/:tid([0-9]+)/log", &ReplicationOperationAPI{}, "get:GetTaskLog")
beego.Router("/api/replication/policies", &ReplicationPolicyAPI{}, "get:List;post:Create")
beego.Router("/api/replication/policies/:id([0-9]+)", &ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete")
// Charts are controlled under projects
chartRepositoryAPIType := &ChartRepositoryAPI{}
beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus")
@ -180,10 +178,6 @@ func init() {
beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels", chartLabelAPIType, "get:GetLabels;post:MarkLabel")
beego.Router("/api/chartrepo/:repo/charts/:name/:version/labels/:id([0-9]+)", chartLabelAPIType, "delete:RemoveLabel")
if err := core.Init(); err != nil {
log.Fatalf("failed to initialize GlobalController: %v", err)
}
// syncRegistry
if err := SyncRegistry(config.GlobalProjectMgr); err != nil {
log.Fatalf("failed to sync repositories from registry: %v", err)
@ -659,103 +653,6 @@ func (a testapi) GetReposTop(authInfo usrInfo, count string) (int, interface{},
return http.StatusOK, result, nil
}
// -------------------------Targets Test---------------------------------------//
// Create a new replication target
func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (int, string, error) {
_sling := sling.New().Post(a.basePath)
path := "/api/targets"
_sling = _sling.Path(path)
_sling = _sling.BodyJSON(repTarget)
httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
return httpStatusCode, string(body), err
}
// List filters targets by name
func (a testapi) ListTargets(authInfo usrInfo, targetName string) (int, []apilib.RepTarget, error) {
_sling := sling.New().Get(a.basePath)
path := "/api/targets?name=" + targetName
_sling = _sling.Path(path)
var successPayload []apilib.RepTarget
httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
if err == nil && httpStatusCode == 200 {
err = json.Unmarshal(body, &successPayload)
}
return httpStatusCode, successPayload, err
}
// Ping target
func (a testapi) PingTarget(authInfo usrInfo, body interface{}) (int, error) {
_sling := sling.New().Post(a.basePath)
path := "/api/targets/ping"
_sling = _sling.Path(path)
_sling = _sling.BodyJSON(body)
httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo)
return httpStatusCode, err
}
// Get target by targetID
func (a testapi) GetTargetByID(authInfo usrInfo, targetID string) (int, error) {
_sling := sling.New().Get(a.basePath)
path := "/api/targets/" + targetID
_sling = _sling.Path(path)
httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo)
return httpStatusCode, err
}
// Update target by targetID
func (a testapi) PutTargetByID(authInfo usrInfo, targetID string, repTarget apilib.RepTargetPost) (int, error) {
_sling := sling.New().Put(a.basePath)
path := "/api/targets/" + targetID
_sling = _sling.Path(path)
_sling = _sling.BodyJSON(repTarget)
httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo)
return httpStatusCode, err
}
// List the target relevant policies by targetID
func (a testapi) GetTargetPoliciesByID(authInfo usrInfo, targetID string) (int, error) {
_sling := sling.New().Get(a.basePath)
path := "/api/targets/" + targetID + "/policies/"
_sling = _sling.Path(path)
httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo)
return httpStatusCode, err
}
// Delete target by targetID
func (a testapi) DeleteTargetsByID(authInfo usrInfo, targetID string) (int, error) {
_sling := sling.New().Delete(a.basePath)
path := "/api/targets/" + targetID
_sling = _sling.Path(path)
httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo)
return httpStatusCode, err
}
// --------------------Replication_Policy Test--------------------------------//
// Create a new replication policy
@ -1244,3 +1141,73 @@ func (a testapi) ScanAllScheduleGet(authInfo usrInfo) (int, api_models.AdminJobS
return httpStatusCode, successPayLoad, err
}
func (a testapi) RegistryGet(authInfo usrInfo, registryID int64) (*model.Registry, int, error) {
_sling := sling.New().Base(a.basePath).Get(fmt.Sprintf("/api/registries/%d", registryID))
code, body, err := request(_sling, jsonAcceptHeader, authInfo)
if err == nil && code == http.StatusOK {
registry := model.Registry{}
if err := json.Unmarshal(body, &registry); err != nil {
return nil, code, err
}
return &registry, code, nil
}
return nil, code, err
}
func (a testapi) RegistryList(authInfo usrInfo) ([]*model.Registry, int, error) {
_sling := sling.New().Base(a.basePath).Get("/api/registries")
code, body, err := request(_sling, jsonAcceptHeader, authInfo)
if err != nil || code != http.StatusOK {
return nil, code, err
}
var registries []*model.Registry
if err := json.Unmarshal(body, &registries); err != nil {
return nil, code, err
}
return registries, code, nil
}
func (a testapi) RegistryCreate(authInfo usrInfo, registry *model.Registry) (int, error) {
_sling := sling.New().Base(a.basePath).Post("/api/registries").BodyJSON(registry)
code, _, err := request(_sling, jsonAcceptHeader, authInfo)
return code, err
}
type pingReq struct {
ID *int64 `json:"id"`
Type *string `json:"type"`
URL *string `json:"url"`
CredentialType *string `json:"credential_type"`
AccessKey *string `json:"access_key"`
AccessSecret *string `json:"access_secret"`
Insecure *bool `json:"insecure"`
}
func (a testapi) RegistryPing(authInfo usrInfo, registry *pingReq) (int, error) {
_sling := sling.New().Base(a.basePath).Post("/api/registries/ping").BodyJSON(registry)
code, _, err := request(_sling, jsonAcceptHeader, authInfo)
return code, err
}
func (a testapi) RegistryDelete(authInfo usrInfo, registryID int64) (int, error) {
_sling := sling.New().Base(a.basePath).Delete(fmt.Sprintf("/api/registries/%d", registryID))
code, _, err := request(_sling, jsonAcceptHeader, authInfo)
if err != nil || code != http.StatusOK {
return code, fmt.Errorf("delete registry error: %v", err)
}
return code, nil
}
func (a testapi) RegistryUpdate(authInfo usrInfo, registryID int64, req *apimodels.RegistryUpdateRequest) (int, error) {
_sling := sling.New().Base(a.basePath).Put(fmt.Sprintf("/api/registries/%d", registryID)).BodyJSON(req)
code, _, err := request(_sling, jsonAcceptHeader, authInfo)
if err != nil || code != http.StatusOK {
return code, fmt.Errorf("update registry error: %v", err)
}
return code, nil
}

37
src/core/api/label.go
View File

@ -24,9 +24,6 @@ import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/core"
rep_models "github.com/goharbor/harbor/src/replication/models"
)
// LabelAPI handles requests for label management
@ -332,26 +329,28 @@ func (l *LabelAPI) ListResources() {
return
}
result, err := core.GlobalController.GetPolicies(rep_models.QueryParameter{})
if err != nil {
l.SendInternalServerError(fmt.Errorf("failed to get policies: %v", err))
return
}
policies := []*rep_models.ReplicationPolicy{}
if result != nil {
for _, policy := range result.Policies {
for _, filter := range policy.Filters {
if filter.Kind != replication.FilterItemKindLabel {
continue
}
if filter.Value.(int64) == label.ID {
policies = append(policies, policy)
/*
result, err := core.GlobalController.GetPolicies(rep_models.QueryParameter{})
if err != nil {
l.HandleInternalServerError(fmt.Sprintf("failed to get policies: %v", err))
return
}
policies := []*rep_models.ReplicationPolicy{}
if result != nil {
for _, policy := range result.Policies {
for _, filter := range policy.Filters {
if filter.Kind != replication.FilterItemKindLabel {
continue
}
if filter.Value.(int64) == label.ID {
policies = append(policies, policy)
}
}
}
}
}
*/
resources := map[string]interface{}{}
resources["replication_policies"] = policies
resources["replication_policies"] = nil
l.Data["json"] = resources
l.ServeJSON()
}

105
src/core/api/label_test.go
View File

@ -21,10 +21,7 @@ import (
"testing"
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/replication"
rep_models "github.com/goharbor/harbor/src/replication/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -436,105 +433,3 @@ func TestLabelAPIDelete(t *testing.T) {
runCodeCheckingCases(t, cases...)
}
func TestListResources(t *testing.T) {
// global level label
globalLabelID, err := dao.AddLabel(&models.Label{
Name: "globel_level_label",
Scope: common.LabelScopeGlobal,
})
require.Nil(t, err)
defer dao.DeleteLabel(globalLabelID)
// project level label
projectLabelID, err := dao.AddLabel(&models.Label{
Name: "project_level_label",
Scope: common.LabelScopeProject,
ProjectID: 1,
})
require.Nil(t, err)
defer dao.DeleteLabel(projectLabelID)
targetID, err := dao.AddRepTarget(models.RepTarget{
Name: "target_for_testing_label_resource",
URL: "https://192.168.0.1",
})
require.Nil(t, err)
defer dao.DeleteRepTarget(targetID)
// create a policy references both global and project labels
policyID, err := dao.AddRepPolicy(models.RepPolicy{
Name: "policy_for_testing_label_resource",
ProjectID: 1,
TargetID: targetID,
Trigger: fmt.Sprintf(`{"kind":"%s"}`, replication.TriggerKindManual),
Filters: fmt.Sprintf(`[{"kind":"%s","value":%d}, {"kind":"%s","value":%d}]`,
replication.FilterItemKindLabel, globalLabelID,
replication.FilterItemKindLabel, projectLabelID),
})
require.Nil(t, err)
defer dao.DeleteRepPolicy(policyID)
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, globalLabelID),
},
code: http.StatusUnauthorized,
},
// 404
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, 10000),
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 403: global level label
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, globalLabelID),
credential: projAdmin,
},
code: http.StatusForbidden,
},
// 403: project level label
{
request: &testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, projectLabelID),
credential: projDeveloper,
},
code: http.StatusForbidden,
},
}
runCodeCheckingCases(t, cases...)
// 200: global level label
resources := map[string][]rep_models.ReplicationPolicy{}
err = handleAndParse(&testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, globalLabelID),
credential: sysAdmin,
}, &resources)
require.Nil(t, err)
policies := resources["replication_policies"]
require.Equal(t, 1, len(policies))
assert.Equal(t, policyID, policies[0].ID)
// 200: project level label
resources = map[string][]rep_models.ReplicationPolicy{}
err = handleAndParse(&testingRequest{
method: http.MethodGet,
url: fmt.Sprintf("%s/%d/resources", labelAPIBasePath, projectLabelID),
credential: projAdmin,
}, &resources)
require.Nil(t, err)
policies = resources["replication_policies"]
require.Equal(t, 1, len(policies))
assert.Equal(t, policyID, policies[0].ID)
}

15
src/core/api/models/execution.go Normal file
View File

@ -0,0 +1,15 @@
package models
import (
"time"
)
// Execution defines the data model used in API level
type Execution struct {
ID int64 `json:"id"`
Status string `json:"status"`
TriggerMode string `json:"trigger_mode"`
Duration int `json:"duration"`
SuccessRate string `json:"success_rate"`
StartTime time.Time `json:"start_time"`
}

12
src/core/api/models/registry.go Normal file
View File

@ -0,0 +1,12 @@
package models
// RegistryUpdateRequest is request used to update a registry.
type RegistryUpdateRequest struct {
Name *string `json:"name"`
Description *string `json:"description"`
URL *string `json:"url"`
CredentialType *string `json:"credential_type"`
AccessKey *string `json:"access_key"`
AccessSecret *string `json:"access_secret"`
Insecure *bool `json:"insecure"`
}

68
src/core/api/models/replication_policy.go
View File

@ -1,68 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models
import (
"time"
"github.com/astaxie/beego/validation"
common_models "github.com/goharbor/harbor/src/common/models"
rep_models "github.com/goharbor/harbor/src/replication/models"
)
// ReplicationPolicy defines the data model used in API level
type ReplicationPolicy struct {
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Filters []rep_models.Filter `json:"filters"`
ReplicateDeletion bool `json:"replicate_deletion"`
Trigger *rep_models.Trigger `json:"trigger"`
Projects []*common_models.Project `json:"projects"`
Targets []*common_models.RepTarget `json:"targets"`
CreationTime time.Time `json:"creation_time"`
UpdateTime time.Time `json:"update_time"`
ReplicateExistingImageNow bool `json:"replicate_existing_image_now"`
ErrorJobCount int64 `json:"error_job_count"`
}
// Valid ...
func (r *ReplicationPolicy) Valid(v *validation.Validation) {
if len(r.Name) == 0 {
v.SetError("name", "can not be empty")
}
if len(r.Name) > 256 {
v.SetError("name", "max length is 256")
}
if len(r.Projects) == 0 {
v.SetError("projects", "can not be empty")
}
if len(r.Targets) == 0 {
v.SetError("targets", "can not be empty")
}
for i := range r.Filters {
r.Filters[i].Valid(v)
}
if r.Trigger == nil {
v.SetError("trigger", "can not be empty")
} else {
r.Trigger.Valid(v)
}
}

34
src/core/api/project.go
View File

@ -118,7 +118,7 @@ func (p *ProjectAPI) Post() {
}
}
if onlyAdmin && !p.SecurityCtx.IsSysAdmin() {
if onlyAdmin && !(p.SecurityCtx.IsSysAdmin() || p.SecurityCtx.IsSolutionUser()) {
log.Errorf("Only sys admin can create project")
p.SendForbiddenError(errors.New("Only system admin can create project"))
return
@ -159,9 +159,23 @@ func (p *ProjectAPI) Post() {
pro.Metadata[models.ProMetaPublic] = strconv.FormatBool(false)
}
owner := p.SecurityCtx.GetUsername()
// set the owner as the system admin when the API being called by replication
// it's a solution to workaround the restriction of project creation API:
// only normal users can create projects
if p.SecurityCtx.IsSolutionUser() {
user, err := dao.GetUser(models.User{
UserID: 1,
})
if err != nil {
p.SendInternalServerError(fmt.Errorf("failed to get the user 1: %v", err))
return
}
owner = user.Username
}
projectID, err := p.ProjectMgr.Create(&models.Project{
Name: pro.Name,
OwnerName: p.SecurityCtx.GetUsername(),
OwnerName: owner,
Metadata: pro.Metadata,
})
if err != nil {
@ -291,18 +305,6 @@ func (p *ProjectAPI) deletable(projectID int64) (*deletableResp, error) {
}, nil
}
policies, err := dao.GetRepPolicyByProject(projectID)
if err != nil {
return nil, err
}
if len(policies) > 0 {
return &deletableResp{
Deletable: false,
Message: "the project contains replication rules, can not be deleted",
}, nil
}
// Check helm charts number
if config.WithChartMuseum() {
charts, err := chartController.ListCharts(p.project.Name)
@ -531,8 +533,8 @@ func (p *ProjectAPI) Logs() {
// TODO move this to pa ckage models
func validateProjectReq(req *models.ProjectRequest) error {
pn := req.Name
if utils.IsIllegalLength(req.Name, projectNameMinLen, projectNameMaxLen) {
return fmt.Errorf("Project name is illegal in length. (greater than %d or less than %d)", projectNameMaxLen, projectNameMinLen)
if utils.IsIllegalLength(pn, projectNameMinLen, projectNameMaxLen) {
return fmt.Errorf("Project name %s is illegal in length. (greater than %d or less than %d)", pn, projectNameMaxLen, projectNameMinLen)
}
validProjectName := regexp.MustCompile(`^` + restrictedNameChars + `$`)
legal := validProjectName.MatchString(pn)

496
src/core/api/registry.go Normal file
View File

@ -0,0 +1,496 @@
package api
import (
"errors"
"fmt"
"net/http"
"strconv"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/policy"
"github.com/goharbor/harbor/src/replication/registry"
)
// RegistryAPI handles requests to /api/registries/{}. It manages registries integrated to Harbor.
type RegistryAPI struct {
BaseController
manager registry.Manager
policyCtl policy.Controller
}
// Prepare validates the user
func (t *RegistryAPI) Prepare() {
t.BaseController.Prepare()
if !t.SecurityCtx.IsAuthenticated() {
t.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
if !t.SecurityCtx.IsSysAdmin() {
t.SendForbiddenError(errors.New(t.SecurityCtx.GetUsername()))
return
}
t.manager = replication.RegistryMgr
t.policyCtl = replication.PolicyCtl
}
// Ping checks health status of a registry
func (t *RegistryAPI) Ping() {
req := struct {
ID *int64 `json:"id"`
Type *string `json:"type"`
URL *string `json:"url"`
CredentialType *string `json:"credential_type"`
AccessKey *string `json:"access_key"`
AccessSecret *string `json:"access_secret"`
Insecure *bool `json:"insecure"`
}{}
t.DecodeJSONReq(&req)
reg := &model.Registry{}
var err error
if req.ID != nil {
reg, err = t.manager.Get(*req.ID)
if err != nil {
t.SendInternalServerError(fmt.Errorf("failed to get registry %d: %v", *req.ID, err))
return
}
if reg == nil {
t.SendNotFoundError(fmt.Errorf("registry %d not found", *req.ID))
return
}
}
if req.Type != nil {
reg.Type = model.RegistryType(*req.Type)
}
if req.URL != nil {
url, err := utils.ParseEndpoint(*req.URL)
if err != nil {
t.SendBadRequestError(err)
return
}
// Prevent SSRF security issue #3755
reg.URL = url.Scheme + "://" + url.Host + url.Path
}
if req.CredentialType != nil {
if reg.Credential == nil {
reg.Credential = &model.Credential{}
}
reg.Credential.Type = model.CredentialType(*req.CredentialType)
}
if req.AccessKey != nil {
if reg.Credential == nil {
reg.Credential = &model.Credential{}
}
reg.Credential.AccessKey = *req.AccessKey
}
if req.AccessSecret != nil {
if reg.Credential == nil {
reg.Credential = &model.Credential{}
}
reg.Credential.AccessSecret = *req.AccessSecret
}
if req.Insecure != nil {
reg.Insecure = *req.Insecure
}
if len(reg.Type) == 0 || len(reg.URL) == 0 {
t.SendBadRequestError(errors.New("type or url cannot be empty"))
return
}
status, err := registry.CheckHealthStatus(reg)
if err != nil {
e, ok := err.(*common_http.Error)
if ok && e.Code == http.StatusUnauthorized {
t.SendBadRequestError(errors.New("invalid credential"))
return
}
t.SendInternalServerError(fmt.Errorf("failed to check health of registry %s: %v", reg.URL, err))
return
}
if status != model.Healthy {
t.SendBadRequestError(errors.New(""))
return
}
return
}
// Get gets a registry by id.
func (t *RegistryAPI) Get() {
id, err := t.GetIDFromURL()
if err != nil {
t.SendBadRequestError(err)
return
}
r, err := t.manager.Get(id)
if err != nil {
log.Errorf("failed to get registry %d: %v", id, err)
t.SendInternalServerError(err)
return
}
if r == nil {
t.SendNotFoundError(fmt.Errorf("registry %d not found", id))
return
}
// Hide access secret
if r.Credential != nil && len(r.Credential.AccessSecret) != 0 {
r.Credential.AccessSecret = "*****"
}
t.Data["json"] = r
t.ServeJSON()
}
// List lists all registries that match a given registry name.
func (t *RegistryAPI) List() {
name := t.GetString("name")
_, registries, err := t.manager.List(&model.RegistryQuery{
Name: name,
})
if err != nil {
log.Errorf("failed to list registries %s: %v", name, err)
t.SendInternalServerError(err)
return
}
// Hide passwords
for _, r := range registries {
if r.Credential != nil && len(r.Credential.AccessSecret) != 0 {
r.Credential.AccessSecret = "*****"
}
}
t.Data["json"] = registries
t.ServeJSON()
return
}
// Post creates a registry
func (t *RegistryAPI) Post() {
r := &model.Registry{}
isValid, err := t.DecodeJSONReqAndValidate(r)
if !isValid {
t.SendBadRequestError(err)
return
}
reg, err := t.manager.GetByName(r.Name)
if err != nil {
log.Errorf("failed to get registry %s: %v", r.Name, err)
t.SendInternalServerError(err)
return
}
if reg != nil {
t.SendConflictError(fmt.Errorf("name '%s' is already used", r.Name))
return
}
status, err := registry.CheckHealthStatus(r)
if err != nil {
t.SendBadRequestError(fmt.Errorf("health check to registry %s failed: %v", r.URL, err))
return
}
if status != model.Healthy {
t.SendBadRequestError(fmt.Errorf("registry %s is unhealthy: %s", r.URL, status))
return
}
id, err := t.manager.Add(r)
if err != nil {
log.Errorf("Add registry '%s' error: %v", r.URL, err)
t.SendInternalServerError(err)
return
}
t.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
}
// Put updates a registry
func (t *RegistryAPI) Put() {
id, err := t.GetIDFromURL()
if err != nil {
t.SendBadRequestError(err)
return
}
r, err := t.manager.Get(id)
if err != nil {
log.Errorf("Get registry by id %d error: %v", id, err)
t.SendInternalServerError(err)
return
}
if r == nil {
t.SendNotFoundError(fmt.Errorf("Registry %d not found", id))
return
}
req := models.RegistryUpdateRequest{}
if err := t.DecodeJSONReq(&req); err != nil {
t.SendBadRequestError(err)
return
}
originalName := r.Name
if req.Name != nil {
r.Name = *req.Name
}
if req.Description != nil {
r.Description = *req.Description
}
if req.URL != nil {
r.URL = *req.URL
}
if req.CredentialType != nil {
r.Credential.Type = (model.CredentialType)(*req.CredentialType)
}
if req.AccessKey != nil {
r.Credential.AccessKey = *req.AccessKey
}
if req.AccessSecret != nil {
r.Credential.AccessSecret = *req.AccessSecret
}
if req.Insecure != nil {
r.Insecure = *req.Insecure
}
t.Validate(r)
if r.Name != originalName {
reg, err := t.manager.GetByName(r.Name)
if err != nil {
log.Errorf("Get registry by name '%s' error: %v", r.Name, err)
t.SendInternalServerError(err)
return
}
if reg != nil {
t.SendConflictError(errors.New("name is already used"))
return
}
}
status, err := registry.CheckHealthStatus(r)
if err != nil {
t.SendBadRequestError(fmt.Errorf("health check to registry %s failed: %v", r.URL, err))
return
}
if status != model.Healthy {
t.SendBadRequestError(fmt.Errorf("registry %s is unhealthy: %s", r.URL, status))
return
}
if err := t.manager.Update(r); err != nil {
log.Errorf("Update registry %d error: %v", id, err)
t.SendInternalServerError(err)
return
}
}
// Delete deletes a registry
func (t *RegistryAPI) Delete() {
id, err := t.GetIDFromURL()
if err != nil {
t.SendBadRequestError(err)
return
}
registry, err := t.manager.Get(id)
if err != nil {
msg := fmt.Sprintf("Get registry %d error: %v", id, err)
log.Error(msg)
t.SendInternalServerError(errors.New(msg))
return
}
if registry == nil {
t.SendNotFoundError(fmt.Errorf("Registry %d not found", id))
return
}
// Check whether there are replication policies that use this registry as source registry.
total, _, err := t.policyCtl.List([]*model.PolicyQuery{
{
SrcRegistry: id,
},
}...)
if err != nil {
t.SendInternalServerError(fmt.Errorf("List replication policies with source registry %d error: %v", id, err))
return
}
if total > 0 {
msg := fmt.Sprintf("Can't delete registry %d, %d replication policies use it as source registry", id, total)
log.Error(msg)
t.SendPreconditionFailedError(errors.New(msg))
return
}
// Check whether there are replication policies that use this registry as destination registry.
total, _, err = t.policyCtl.List([]*model.PolicyQuery{
{
DestRegistry: id,
},
}...)
if err != nil {
t.SendInternalServerError(fmt.Errorf("List replication policies with destination registry %d error: %v", id, err))
return
}
if total > 0 {
msg := fmt.Sprintf("Can't delete registry %d, %d replication policies use it as destination registry", id, total)
log.Error(msg)
t.SendPreconditionFailedError(errors.New(msg))
return
}
if err := t.manager.Remove(id); err != nil {
msg := fmt.Sprintf("Delete registry %d error: %v", id, err)
log.Error(msg)
t.SendPreconditionFailedError(errors.New(msg))
return
}
}
// GetInfo returns the base info and capability declarations of the registry
func (t *RegistryAPI) GetInfo() {
id, err := t.GetInt64FromPath(":id")
// "0" is used for the ID of the local Harbor registry
if err != nil || id < 0 {
t.SendBadRequestError(fmt.Errorf("invalid registry ID %s", t.GetString(":id")))
return
}
var registry *model.Registry
if id == 0 {
registry = event.GetLocalRegistry()
} else {
registry, err = t.manager.Get(id)
if err != nil {
t.SendInternalServerError(fmt.Errorf("failed to get registry %d: %v", id, err))
return
}
if registry == nil {
t.SendNotFoundError(fmt.Errorf("registry %d not found", id))
return
}
}
factory, err := adapter.GetFactory(registry.Type)
if err != nil {
t.SendInternalServerError(fmt.Errorf("failed to get the adapter factory for registry type %s: %v", registry.Type, err))
return
}
adp, err := factory(registry)
if err != nil {
t.SendInternalServerError(fmt.Errorf("failed to create the adapter for registry %d: %v", registry.ID, err))
return
}
info, err := adp.Info()
if err != nil {
t.SendInternalServerError(fmt.Errorf("failed to get registry info %d: %v", id, err))
return
}
t.WriteJSONData(process(info))
}
// GetNamespace get the namespace of a registry
// TODO remove
func (t *RegistryAPI) GetNamespace() {
/*
var registry *model.Registry
var err error
id, err := t.GetInt64FromPath(":id")
if err != nil || id < 0 {
t.HandleBadRequest(fmt.Sprintf("invalid registry ID %s", t.GetString(":id")))
return
}
if id > 0 {
registry, err = t.manager.Get(id)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("failed to get registry %d: %v", id, err))
return
}
} else if id == 0 {
registry = event.GetLocalRegistry()
}
if registry == nil {
t.HandleNotFound(fmt.Sprintf("registry %d not found", id))
return
}
if !adapter.HasFactory(registry.Type) {
t.HandleInternalServerError(fmt.Sprintf("no adapter factory found for %s", registry.Type))
return
}
regFactory, err := adapter.GetFactory(registry.Type)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("fail to get adapter factory %s", registry.Type))
return
}
regAdapter, err := regFactory(registry)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("fail to get adapter %s", registry.Type))
return
}
query := &model.NamespaceQuery{
Name: t.GetString("name"),
}
npResults, err := regAdapter.ListNamespaces(query)
if err != nil {
t.HandleInternalServerError(fmt.Sprintf("fail to list namespaces %s %v", registry.Type, err))
return
}
t.Data["json"] = npResults
t.ServeJSON()
*/
}
// merge "SupportedResourceTypes" into "SupportedResourceFilters" for UI to render easier
func process(info *model.RegistryInfo) *model.RegistryInfo {
if info == nil {
return nil
}
in := &model.RegistryInfo{
Type: info.Type,
Description: info.Description,
SupportedTriggers: info.SupportedTriggers,
}
filters := []*model.FilterStyle{}
for _, filter := range info.SupportedResourceFilters {
if filter.Type != model.FilterTypeResource {
filters = append(filters, filter)
}
}
values := []string{}
for _, resourceType := range info.SupportedResourceTypes {
values = append(values, string(resourceType))
}
filters = append(filters, &model.FilterStyle{
Type: model.FilterTypeResource,
Style: model.FilterStyleTypeRadio,
Values: values,
})
in.SupportedResourceFilters = filters
return in
}

184
src/core/api/registry_test.go Normal file
View File

@ -0,0 +1,184 @@
package api
import (
"net/http"
"testing"
"github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/dao"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
var (
testRegistry = &model.Registry{
Name: "test1",
URL: "https://registry-1.docker.io",
Type: "harbor",
Credential: nil,
}
testRegistry2 = &model.Registry{
Name: "test2",
URL: "https://registry-1.docker.io",
Type: "harbor",
Credential: nil,
}
)
type RegistrySuite struct {
suite.Suite
testAPI *testapi
defaultRegistry model.Registry
}
func (suite *RegistrySuite) SetupSuite() {
assert := assert.New(suite.T())
assert.Nil(replication.Init(make(chan struct{})))
suite.testAPI = newHarborAPI()
code, err := suite.testAPI.RegistryCreate(*admin, testRegistry)
assert.Nil(err)
assert.Equal(http.StatusCreated, code)
tmp, err := dao.GetRegistryByName(testRegistry.Name)
assert.Nil(err)
assert.NotNil(tmp)
suite.defaultRegistry = *testRegistry
suite.defaultRegistry.ID = tmp.ID
CommonAddUser()
}
func (suite *RegistrySuite) TearDownSuite() {
assert := assert.New(suite.T())
code, err := suite.testAPI.RegistryDelete(*admin, suite.defaultRegistry.ID)
assert.Nil(err)
assert.Equal(http.StatusOK, code)
CommonDelUser()
}
func (suite *RegistrySuite) TestGet() {
assert := assert.New(suite.T())
// Get a non-existed registry
_, code, _ := suite.testAPI.RegistryGet(*admin, 0)
assert.Equal(http.StatusBadRequest, code)
// Get as admin, should succeed
retrieved, code, err := suite.testAPI.RegistryGet(*admin, suite.defaultRegistry.ID)
assert.Nil(err)
assert.NotNil(retrieved)
assert.Equal(http.StatusOK, code)
assert.Equal("test1", retrieved.Name)
// Get as user, should fail
_, code, _ = suite.testAPI.RegistryGet(*testUser, suite.defaultRegistry.ID)
assert.Equal(http.StatusForbidden, code)
}
func (suite *RegistrySuite) TestList() {
assert := assert.New(suite.T())
// List as admin, should succeed
registries, code, err := suite.testAPI.RegistryList(*admin)
assert.Nil(err)
assert.Equal(http.StatusOK, code)
assert.Equal(1, len(registries))
// List as user, should fail
registries, code, err = suite.testAPI.RegistryList(*testUser)
assert.Equal(http.StatusForbidden, code)
assert.Equal(0, len(registries))
}
func (suite *RegistrySuite) TestPost() {
assert := assert.New(suite.T())
// Should conflict when create exited registry
code, err := suite.testAPI.RegistryCreate(*admin, testRegistry)
assert.Nil(err)
assert.Equal(http.StatusConflict, code)
// Create as user, should fail
code, err = suite.testAPI.RegistryCreate(*testUser, testRegistry2)
assert.Nil(err)
assert.Equal(http.StatusForbidden, code)
}
func (suite *RegistrySuite) TestPing() {
assert := assert.New(suite.T())
code, err := suite.testAPI.RegistryPing(*admin, &pingReq{
ID: &suite.defaultRegistry.ID,
})
assert.Nil(err)
assert.Equal(http.StatusOK, code)
var id int64 = -1
code, err = suite.testAPI.RegistryPing(*admin, &pingReq{
ID: &id,
})
assert.Nil(err)
assert.Equal(http.StatusNotFound, code)
code, err = suite.testAPI.RegistryPing(*admin, nil)
assert.Nil(err)
assert.Equal(http.StatusBadRequest, code)
code, err = suite.testAPI.RegistryPing(*testUser, &pingReq{
ID: &suite.defaultRegistry.ID,
})
assert.Nil(err)
assert.Equal(http.StatusForbidden, code)
}
func (suite *RegistrySuite) TestRegistryPut() {
assert := assert.New(suite.T())
// Update as admin, should succeed
description := "foobar"
updateReq := &models.RegistryUpdateRequest{
Description: &description,
}
code, err := suite.testAPI.RegistryUpdate(*admin, suite.defaultRegistry.ID, updateReq)
assert.Nil(err)
assert.Equal(http.StatusOK, code)
updated, code, err := suite.testAPI.RegistryGet(*admin, suite.defaultRegistry.ID)
assert.Nil(err)
assert.Equal(http.StatusOK, code)
assert.Equal("foobar", updated.Description)
// Update as user, should fail
code, err = suite.testAPI.RegistryUpdate(*testUser, suite.defaultRegistry.ID, updateReq)
assert.NotNil(err)
assert.Equal(http.StatusForbidden, code)
}
func (suite *RegistrySuite) TestDelete() {
assert := assert.New(suite.T())
code, err := suite.testAPI.RegistryCreate(*admin, testRegistry2)
assert.Nil(err)
assert.Equal(http.StatusCreated, code)
tmp, err := dao.GetRegistryByName(testRegistry2.Name)
assert.Nil(err)
assert.NotNil(tmp)
// Delete as user, should fail
code, err = suite.testAPI.RegistryDelete(*testUser, tmp.ID)
assert.NotNil(err)
assert.Equal(http.StatusForbidden, code)
// Delete as admin, should succeed
code, err = suite.testAPI.RegistryDelete(*admin, tmp.ID)
assert.Nil(err)
assert.Equal(http.StatusOK, code)
}
func TestRegistrySuite(t *testing.T) {
suite.Run(t, new(RegistrySuite))
}

111
src/core/api/replication.go
View File

@ -1,111 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"strings"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
api_models "github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/core/notifier"
"github.com/goharbor/harbor/src/replication/core"
"github.com/goharbor/harbor/src/replication/event/notification"
"github.com/goharbor/harbor/src/replication/event/topic"
"errors"
"github.com/docker/distribution/uuid"
)
// ReplicationAPI handles API calls for replication
type ReplicationAPI struct {
BaseController
}
// Prepare does authentication and authorization works
func (r *ReplicationAPI) Prepare() {
r.BaseController.Prepare()
if !r.SecurityCtx.IsAuthenticated() {
r.SendUnAuthorizedError(errors.New("Unauthorized"))
return
}
if !r.SecurityCtx.IsSysAdmin() && !r.SecurityCtx.IsSolutionUser() {
r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername()))
return
}
}
// Post trigger a replication according to the specified policy
func (r *ReplicationAPI) Post() {
replication := &api_models.Replication{}
isValid, err := r.DecodeJSONReqAndValidate(replication)
if !isValid {
r.SendBadRequestError(err)
return
}
policy, err := core.GlobalController.GetPolicy(replication.PolicyID)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to get replication policy %d: %v", replication.PolicyID, err))
return
}
if policy.ID == 0 {
r.SendNotFoundError(fmt.Errorf("replication policy %d not found", replication.PolicyID))
return
}
count, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{
PolicyID: replication.PolicyID,
Statuses: []string{models.JobPending, models.JobRunning},
Operations: []string{models.RepOpTransfer, models.RepOpDelete},
})
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to filter jobs of policy %d: %v",
replication.PolicyID, err))
return
}
if count > 0 {
r.SendPreconditionFailedError(errors.New("policy has running/pending jobs, new replication can not be triggered"))
return
}
opUUID, err := startReplication(replication.PolicyID)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to publish replication topic for policy %d: %v", replication.PolicyID, err))
return
}
log.Infof("replication signal for policy %d sent", replication.PolicyID)
r.Data["json"] = api_models.ReplicationResponse{
UUID: opUUID,
}
r.ServeJSON()
}
// startReplication triggers a replication and return the uuid of this replication.
func startReplication(policyID int64) (string, error) {
opUUID := strings.Replace(uuid.Generate().String(), "-", "", -1)
return opUUID, notifier.Publish(topic.StartReplicationTopic,
notification.StartReplicationNotification{
PolicyID: policyID,
Metadata: map[string]interface{}{
"op_uuid": opUUID,
},
})
}

46
src/core/api/replication_adapter.go Normal file
View File

@ -0,0 +1,46 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"errors"
"github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
)
// ReplicationAdapterAPI handles the replication adapter requests
type ReplicationAdapterAPI struct {
BaseController
}
// Prepare ...
func (r *ReplicationAdapterAPI) Prepare() {
r.BaseController.Prepare()
if !r.SecurityCtx.IsSysAdmin() {
if !r.SecurityCtx.IsAuthenticated() {
r.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername()))
return
}
}
// List the replication adapters
func (r *ReplicationAdapterAPI) List() {
types := []model.RegistryType{}
types = append(types, adapter.ListRegisteredAdapterTypes()...)
r.WriteJSONData(types)
}

63
src/core/api/replication_adapter_test.go Normal file
View File

@ -0,0 +1,63 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"net/http"
"testing"
"github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/model"
"github.com/stretchr/testify/require"
)
func fakedFactory(*model.Registry) (adapter.Adapter, error) {
return nil, nil
}
func TestReplicationAdapterAPIList(t *testing.T) {
err := adapter.RegisterFactory("test", fakedFactory)
require.Nil(t, err)
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/adapters",
},
code: http.StatusUnauthorized,
},
// 403
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/adapters",
credential: nonSysAdmin,
},
code: http.StatusForbidden,
},
// 200
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/adapters",
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}

277
src/core/api/replication_execution.go Normal file
View File

@ -0,0 +1,277 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"errors"
"fmt"
"net/http"
"strconv"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/dao/models"
"github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
)
// ReplicationOperationAPI handles the replication operation requests
type ReplicationOperationAPI struct {
BaseController
}
// Prepare ...
func (r *ReplicationOperationAPI) Prepare() {
r.BaseController.Prepare()
// As we delegate the jobservice to trigger the scheduled replication,
// we need to allow the jobservice to call the API
if !(r.SecurityCtx.IsSysAdmin() || r.SecurityCtx.IsSolutionUser()) {
if !r.SecurityCtx.IsAuthenticated() {
r.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername()))
return
}
}
// The API is open only for system admin currently, we can use
// the code commentted below to make the API available to the
// users who have permission for all projects that the policy
// refers
/*
func (r *ReplicationOperationAPI) authorized(policy *model.Policy, resource rbac.Resource, action rbac.Action) bool {
projects := []string{}
// pull mode
if policy.SrcRegistryID != 0 {
projects = append(projects, policy.DestNamespace)
} else {
// push mode
projects = append(projects, policy.SrcNamespaces...)
}
for _, project := range projects {
resource := rbac.NewProjectNamespace(project).Resource(resource)
if !r.SecurityCtx.Can(action, resource) {
r.HandleForbidden(r.SecurityCtx.GetUsername())
return false
}
}
return true
}
*/
// ListExecutions ...
func (r *ReplicationOperationAPI) ListExecutions() {
query := &models.ExecutionQuery{
Trigger: r.GetString("trigger"),
}
if len(r.GetString("status")) > 0 {
query.Statuses = []string{r.GetString("status")}
}
if len(r.GetString("policy_id")) > 0 {
policyID, err := r.GetInt64("policy_id")
if err != nil || policyID <= 0 {
r.SendBadRequestError(fmt.Errorf("invalid policy_id %s", r.GetString("policy_id")))
return
}
query.PolicyID = policyID
}
page, size, err := r.GetPaginationParams()
if err != nil {
r.SendBadRequestError(err)
return
}
query.Page = page
query.Size = size
total, executions, err := replication.OperationCtl.ListExecutions(query)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to list executions: %v", err))
return
}
r.SetPaginationHeader(total, query.Page, query.Size)
r.WriteJSONData(executions)
}
// CreateExecution starts a replication
func (r *ReplicationOperationAPI) CreateExecution() {
execution := &models.Execution{}
if err := r.DecodeJSONReq(execution); err != nil {
r.SendBadRequestError(err)
return
}
policy, err := replication.PolicyCtl.Get(execution.PolicyID)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", execution.PolicyID, err))
return
}
if policy == nil {
r.SendNotFoundError(fmt.Errorf("policy %d not found", execution.PolicyID))
return
}
if !policy.Enabled {
r.SendBadRequestError(fmt.Errorf("the policy %d is disabled", execution.PolicyID))
return
}
if err = event.PopulateRegistries(replication.RegistryMgr, policy); err != nil {
r.SendInternalServerError(fmt.Errorf("failed to populate registries for policy %d: %v", execution.PolicyID, err))
return
}
trigger := r.GetString("trigger", string(model.TriggerTypeManual))
executionID, err := replication.OperationCtl.StartReplication(policy, nil, model.TriggerType(trigger))
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to start replication for policy %d: %v", execution.PolicyID, err))
return
}
r.Redirect(http.StatusCreated, strconv.FormatInt(executionID, 10))
}
// GetExecution gets one execution of the replication
func (r *ReplicationOperationAPI) GetExecution() {
executionID, err := r.GetInt64FromPath(":id")
if err != nil || executionID <= 0 {
r.SendBadRequestError(errors.New("invalid execution ID"))
return
}
execution, err := replication.OperationCtl.GetExecution(executionID)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to get execution %d: %v", executionID, err))
return
}
if execution == nil {
r.SendNotFoundError(fmt.Errorf("execution %d not found", executionID))
return
}
r.WriteJSONData(execution)
}
// StopExecution stops one execution of the replication
func (r *ReplicationOperationAPI) StopExecution() {
executionID, err := r.GetInt64FromPath(":id")
if err != nil || executionID <= 0 {
r.SendBadRequestError(errors.New("invalid execution ID"))
return
}
execution, err := replication.OperationCtl.GetExecution(executionID)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to get execution %d: %v", executionID, err))
return
}
if execution == nil {
r.SendNotFoundError(fmt.Errorf("execution %d not found", executionID))
return
}
if err := replication.OperationCtl.StopReplication(executionID); err != nil {
r.SendInternalServerError(fmt.Errorf("failed to stop execution %d: %v", executionID, err))
return
}
}
// ListTasks ...
func (r *ReplicationOperationAPI) ListTasks() {
executionID, err := r.GetInt64FromPath(":id")
if err != nil || executionID <= 0 {
r.SendBadRequestError(errors.New("invalid execution ID"))
return
}
execution, err := replication.OperationCtl.GetExecution(executionID)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to get execution %d: %v", executionID, err))
return
}
if execution == nil {
r.SendNotFoundError(fmt.Errorf("execution %d not found", executionID))
return
}
query := &models.TaskQuery{
ExecutionID: executionID,
ResourceType: r.GetString("resource_type"),
}
status := r.GetString("status")
if len(status) > 0 {
query.Statuses = []string{status}
}
page, size, err := r.GetPaginationParams()
if err != nil {
r.SendBadRequestError(err)
return
}
query.Page = page
query.Size = size
total, tasks, err := replication.OperationCtl.ListTasks(query)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to list tasks: %v", err))
return
}
r.SetPaginationHeader(total, query.Page, query.Size)
r.WriteJSONData(tasks)
}
// GetTaskLog ...
func (r *ReplicationOperationAPI) GetTaskLog() {
executionID, err := r.GetInt64FromPath(":id")
if err != nil || executionID <= 0 {
r.SendBadRequestError(errors.New("invalid execution ID"))
return
}
execution, err := replication.OperationCtl.GetExecution(executionID)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to get execution %d: %v", executionID, err))
return
}
if execution == nil {
r.SendNotFoundError(fmt.Errorf("execution %d not found", executionID))
return
}
taskID, err := r.GetInt64FromPath(":tid")
if err != nil || taskID <= 0 {
r.SendBadRequestError(errors.New("invalid task ID"))
return
}
task, err := replication.OperationCtl.GetTask(taskID)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to get task %d: %v", taskID, err))
return
}
if task == nil {
r.SendNotFoundError(fmt.Errorf("task %d not found", taskID))
return
}
logBytes, err := replication.OperationCtl.GetTaskLog(taskID)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to get log of task %d: %v", taskID, err))
return
}
r.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes)))
r.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
_, err = r.Ctx.ResponseWriter.Write(logBytes)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to write log of task %d: %v", taskID, err))
return
}
}

429
src/core/api/replication_execution_test.go Normal file
View File

@ -0,0 +1,429 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"net/http"
"testing"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/dao/models"
"github.com/goharbor/harbor/src/replication/model"
)
type fakedOperationController struct{}
func (f *fakedOperationController) StartReplication(policy *model.Policy, resource *model.Resource, trigger model.TriggerType) (int64, error) {
return 1, nil
}
func (f *fakedOperationController) StopReplication(int64) error {
return nil
}
func (f *fakedOperationController) ListExecutions(...*models.ExecutionQuery) (int64, []*models.Execution, error) {
return 1, []*models.Execution{
{
ID: 1,
PolicyID: 1,
},
}, nil
}
func (f *fakedOperationController) GetExecution(id int64) (*models.Execution, error) {
if id == 1 {
return &models.Execution{
ID: 1,
PolicyID: 1,
}, nil
}
return nil, nil
}
func (f *fakedOperationController) ListTasks(...*models.TaskQuery) (int64, []*models.Task, error) {
return 1, []*models.Task{
{
ID: 1,
ExecutionID: 1,
},
}, nil
}
func (f *fakedOperationController) GetTask(id int64) (*models.Task, error) {
if id == 1 {
return &models.Task{
ID: 1,
ExecutionID: 1,
}, nil
}
return nil, nil
}
func (f *fakedOperationController) UpdateTaskStatus(id int64, status string, statusCondition ...string) error {
return nil
}
func (f *fakedOperationController) GetTaskLog(int64) ([]byte, error) {
return []byte("success"), nil
}
type fakedPolicyManager struct{}
func (f *fakedPolicyManager) Create(*model.Policy) (int64, error) {
return 0, nil
}
func (f *fakedPolicyManager) List(...*model.PolicyQuery) (int64, []*model.Policy, error) {
return 0, nil, nil
}
func (f *fakedPolicyManager) Get(id int64) (*model.Policy, error) {
if id == 1 {
return &model.Policy{
ID: 1,
Enabled: true,
SrcRegistry: &model.Registry{
ID: 1,
},
}, nil
}
if id == 2 {
return &model.Policy{
ID: 2,
Enabled: false,
SrcRegistry: &model.Registry{
ID: 1,
},
}, nil
}
return nil, nil
}
func (f *fakedPolicyManager) GetByName(name string) (*model.Policy, error) {
if name == "duplicate_name" {
return &model.Policy{
Name: "duplicate_name",
}, nil
}
return nil, nil
}
func (f *fakedPolicyManager) Update(*model.Policy) error {
return nil
}
func (f *fakedPolicyManager) Remove(int64) error {
return nil
}
func TestListExecutions(t *testing.T) {
operationCtl := replication.OperationCtl
defer func() {
replication.OperationCtl = operationCtl
}()
replication.OperationCtl = &fakedOperationController{}
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions",
},
code: http.StatusUnauthorized,
},
// 403
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions",
credential: nonSysAdmin,
},
code: http.StatusForbidden,
},
// 200
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions",
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestCreateExecution(t *testing.T) {
operationCtl := replication.OperationCtl
policyMgr := replication.PolicyCtl
registryMgr := replication.RegistryMgr
defer func() {
replication.OperationCtl = operationCtl
replication.PolicyCtl = policyMgr
replication.RegistryMgr = registryMgr
}()
replication.OperationCtl = &fakedOperationController{}
replication.PolicyCtl = &fakedPolicyManager{}
replication.RegistryMgr = &fakedRegistryManager{}
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/replication/executions",
},
code: http.StatusUnauthorized,
},
// 403
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/replication/executions",
credential: nonSysAdmin,
},
code: http.StatusForbidden,
},
// 404
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/replication/executions",
bodyJSON: &models.Execution{
PolicyID: 3,
},
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 400
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/replication/executions",
bodyJSON: &models.Execution{
PolicyID: 2,
},
credential: sysAdmin,
},
code: http.StatusBadRequest,
},
// 201
{
request: &testingRequest{
method: http.MethodPost,
url: "/api/replication/executions",
bodyJSON: &models.Execution{
PolicyID: 1,
},
credential: sysAdmin,
},
code: http.StatusCreated,
},
}
runCodeCheckingCases(t, cases...)
}
func TestGetExecution(t *testing.T) {
operationCtl := replication.OperationCtl
defer func() {
replication.OperationCtl = operationCtl
}()
replication.OperationCtl = &fakedOperationController{}
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/1",
},
code: http.StatusUnauthorized,
},
// 403
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/1",
credential: nonSysAdmin,
},
code: http.StatusForbidden,
},
// 404
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/2",
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 200
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/1",
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestStopExecution(t *testing.T) {
operationCtl := replication.OperationCtl
defer func() {
replication.OperationCtl = operationCtl
}()
replication.OperationCtl = &fakedOperationController{}
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodPut,
url: "/api/replication/executions/1",
},
code: http.StatusUnauthorized,
},
// 403
{
request: &testingRequest{
method: http.MethodPut,
url: "/api/replication/executions/1",
credential: nonSysAdmin,
},
code: http.StatusForbidden,
},
// 404
{
request: &testingRequest{
method: http.MethodPut,
url: "/api/replication/executions/2",
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 200
{
request: &testingRequest{
method: http.MethodPut,
url: "/api/replication/executions/1",
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestListTasks(t *testing.T) {
operationCtl := replication.OperationCtl
defer func() {
replication.OperationCtl = operationCtl
}()
replication.OperationCtl = &fakedOperationController{}
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/1/tasks",
},
code: http.StatusUnauthorized,
},
// 403
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/1/tasks",
credential: nonSysAdmin,
},
code: http.StatusForbidden,
},
// 404
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/2/tasks",
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 200
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/1/tasks",
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}
func TestGetTaskLog(t *testing.T) {
operationCtl := replication.OperationCtl
defer func() {
replication.OperationCtl = operationCtl
}()
replication.OperationCtl = &fakedOperationController{}
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/1/tasks/1/log",
},
code: http.StatusUnauthorized,
},
// 403
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/1/tasks/1/log",
credential: nonSysAdmin,
},
code: http.StatusForbidden,
},
// 404, execution not found
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/2/tasks/1/log",
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 404, task not found
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/1/tasks/2/log",
credential: sysAdmin,
},
code: http.StatusNotFound,
},
// 200
{
request: &testingRequest{
method: http.MethodGet,
url: "/api/replication/executions/1/tasks/1/log",
credential: sysAdmin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}

259
src/core/api/replication_job.go
View File

@ -1,259 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"errors"
"fmt"
"net/http"
"strconv"
"time"
"github.com/goharbor/harbor/src/common/dao"
common_job "github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils/log"
api_models "github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/replication/core"
)
// RepJobAPI handles request to /api/replicationJobs /api/replicationJobs/:id/log
type RepJobAPI struct {
BaseController
jobID int64
}
// Prepare validates that whether user has system admin role
func (ra *RepJobAPI) Prepare() {
ra.BaseController.Prepare()
if !ra.SecurityCtx.IsAuthenticated() {
ra.SendUnAuthorizedError(errors.New("Unauthorized"))
return
}
if !(ra.Ctx.Request.Method == http.MethodGet || ra.SecurityCtx.IsSysAdmin()) {
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
return
}
if len(ra.GetStringFromPath(":id")) != 0 {
id, err := ra.GetInt64FromPath(":id")
if err != nil {
ra.SendBadRequestError(fmt.Errorf("invalid ID: %s", ra.GetStringFromPath(":id")))
return
}
ra.jobID = id
}
}
// List filters jobs according to the parameters
func (ra *RepJobAPI) List() {
policyID, err := ra.GetInt64("policy_id")
if err != nil || policyID <= 0 {
ra.SendBadRequestError(fmt.Errorf("invalid policy_id: %s", ra.GetString("policy_id")))
return
}
policy, err := core.GlobalController.GetPolicy(policyID)
if err != nil {
log.Errorf("failed to get policy %d: %v", policyID, err)
ra.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", policyID, err))
return
}
if policy.ID == 0 {
ra.SendNotFoundError(fmt.Errorf("policy %d not found", policyID))
return
}
resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplicationJob)
if !ra.SecurityCtx.Can(rbac.ActionList, resource) {
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
return
}
query := &models.RepJobQuery{
PolicyID: policyID,
// hide the schedule job, the schedule job is used to trigger replication
// for scheduled policy
Operations: []string{models.RepOpTransfer, models.RepOpDelete},
}
query.Repository = ra.GetString("repository")
query.Statuses = ra.GetStrings("status")
query.OpUUID = ra.GetString("op_uuid")
startTimeStr := ra.GetString("start_time")
if len(startTimeStr) != 0 {
i, err := strconv.ParseInt(startTimeStr, 10, 64)
if err != nil {
ra.SendBadRequestError(fmt.Errorf("invalid start_time: %s", startTimeStr))
return
}
t := time.Unix(i, 0)
query.StartTime = &t
}
endTimeStr := ra.GetString("end_time")
if len(endTimeStr) != 0 {
i, err := strconv.ParseInt(endTimeStr, 10, 64)
if err != nil {
ra.SendBadRequestError(fmt.Errorf("invalid end_time: %s", endTimeStr))
return
}
t := time.Unix(i, 0)
query.EndTime = &t
}
query.Page, query.Size, err = ra.GetPaginationParams()
if err != nil {
ra.SendBadRequestError(err)
return
}
total, err := dao.GetTotalCountOfRepJobs(query)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to get total count of repository jobs of policy %d: %v", policyID, err))
return
}
jobs, err := dao.GetRepJobs(query)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to get repository jobs, query: %v :%v", query, err))
return
}
ra.SetPaginationHeader(total, query.Page, query.Size)
ra.Data["json"] = jobs
ra.ServeJSON()
}
// Delete ...
func (ra *RepJobAPI) Delete() {
if ra.jobID == 0 {
ra.SendBadRequestError(errors.New("ID is nil"))
return
}
job, err := dao.GetRepJob(ra.jobID)
if err != nil {
log.Errorf("failed to get job %d: %v", ra.jobID, err)
ra.SendInternalServerError(fmt.Errorf("failed to get job %d: %v", ra.jobID, err))
return
}
if job == nil {
ra.SendNotFoundError(fmt.Errorf("job %d not found", ra.jobID))
return
}
if job.Status == models.JobPending || job.Status == models.JobRunning {
ra.SendBadRequestError(fmt.Errorf("job is %s, can not be deleted", job.Status))
return
}
if err = dao.DeleteRepJob(ra.jobID); err != nil {
log.Errorf("failed to deleted job %d: %v", ra.jobID, err)
ra.SendInternalServerError(fmt.Errorf("failed to deleted job %d: %v", ra.jobID, err))
return
}
}
// GetLog ...
func (ra *RepJobAPI) GetLog() {
if ra.jobID == 0 {
ra.SendBadRequestError(errors.New("ID is nil"))
return
}
job, err := dao.GetRepJob(ra.jobID)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to get replication job %d: %v", ra.jobID, err))
return
}
if job == nil {
ra.SendNotFoundError(fmt.Errorf("replication job %d not found", ra.jobID))
return
}
policy, err := core.GlobalController.GetPolicy(job.PolicyID)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", job.PolicyID, err))
return
}
resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplicationJob)
if !ra.SecurityCtx.Can(rbac.ActionRead, resource) {
ra.SendForbiddenError(errors.New(ra.SecurityCtx.GetUsername()))
return
}
logBytes, err := utils.GetJobServiceClient().GetJobLog(job.UUID)
if err != nil {
ra.ParseAndHandleError(fmt.Sprintf("failed to get log of job %s",
job.UUID), err)
return
}
ra.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes)))
ra.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
_, err = ra.Ctx.ResponseWriter.Write(logBytes)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to write log of job %s: %v", job.UUID, err))
return
}
}
// StopJobs stop replication jobs for the policy
func (ra *RepJobAPI) StopJobs() {
req := &api_models.StopJobsReq{}
isValid, err := ra.DecodeJSONReqAndValidate(req)
if !isValid {
ra.SendBadRequestError(err)
return
}
policy, err := core.GlobalController.GetPolicy(req.PolicyID)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", req.PolicyID, err))
return
}
if policy.ID == 0 {
ra.SendNotFoundError(fmt.Errorf("policy %d not found", req.PolicyID))
return
}
jobs, err := dao.GetRepJobs(&models.RepJobQuery{
PolicyID: policy.ID,
Operations: []string{models.RepOpTransfer, models.RepOpDelete},
})
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to list jobs of policy %d: %v", policy.ID, err))
return
}
for _, job := range jobs {
if err = utils.GetJobServiceClient().PostAction(job.UUID, common_job.JobActionStop); err != nil {
log.Errorf("failed to stop job id-%d uuid-%s: %v", job.ID, job.UUID, err)
continue
}
}
}
// TODO:add Post handler to call job service API to submit jobs by policy

574
src/core/api/replication_policy.go
View File

@ -15,463 +15,243 @@
package api
import (
"fmt"
"errors"
"fmt"
"net/http"
"strconv"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/common/utils/log"
api_models "github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/core/promgr"
common_model "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/core"
rep_models "github.com/goharbor/harbor/src/replication/models"
"github.com/goharbor/harbor/src/replication/dao/models"
"github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/registry"
)
// RepPolicyAPI handles /api/replicationPolicies /api/replicationPolicies/:id/enablement
type RepPolicyAPI struct {
// TODO rename the file to "replication.go"
// ReplicationPolicyAPI handles the replication policy requests
type ReplicationPolicyAPI struct {
BaseController
}
// Prepare validates whether the user has system admin role
func (pa *RepPolicyAPI) Prepare() {
pa.BaseController.Prepare()
if !pa.SecurityCtx.IsAuthenticated() {
pa.SendUnAuthorizedError(errors.New("Unauthorized"))
return
}
if !(pa.Ctx.Request.Method == http.MethodGet || pa.SecurityCtx.IsSysAdmin()) {
pa.SendForbiddenError(errors.New(pa.SecurityCtx.GetUsername()))
return
}
}
// Get ...
func (pa *RepPolicyAPI) Get() {
id, err := pa.GetIDFromURL()
if err != nil {
pa.SendBadRequestError(err)
return
}
policy, err := core.GlobalController.GetPolicy(id)
if err != nil {
log.Errorf("failed to get policy %d: %v", id, err)
pa.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", id, err))
return
}
if policy.ID == 0 {
pa.SendNotFoundError(fmt.Errorf("policy %d not found", id))
return
}
resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplication)
if !pa.SecurityCtx.Can(rbac.ActionRead, resource) {
pa.SendForbiddenError(errors.New(pa.SecurityCtx.GetUsername()))
return
}
ply, err := convertFromRepPolicy(pa.ProjectMgr, policy)
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err)
return
}
pa.Data["json"] = ply
pa.ServeJSON()
}
// List ...
func (pa *RepPolicyAPI) List() {
queryParam := rep_models.QueryParameter{
Name: pa.GetString("name"),
}
projectIDStr := pa.GetString("project_id")
if len(projectIDStr) > 0 {
projectID, err := strconv.ParseInt(projectIDStr, 10, 64)
if err != nil || projectID <= 0 {
pa.SendBadRequestError(fmt.Errorf("invalid project ID: %s", projectIDStr))
// Prepare ...
func (r *ReplicationPolicyAPI) Prepare() {
r.BaseController.Prepare()
if !r.SecurityCtx.IsSysAdmin() {
if !r.SecurityCtx.IsAuthenticated() {
r.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
queryParam.ProjectID = projectID
}
var err error
queryParam.Page, queryParam.PageSize, err = pa.GetPaginationParams()
if err != nil {
pa.SendBadRequestError(err)
r.SendForbiddenError(errors.New(r.SecurityCtx.GetUsername()))
return
}
result, err := core.GlobalController.GetPolicies(queryParam)
if err != nil {
log.Errorf("failed to get policies: %v, query parameters: %v", err, queryParam)
pa.SendInternalServerError(fmt.Errorf("failed to get policies: %v, query parameters: %v", err, queryParam))
return
}
var total int64
policies := []*api_models.ReplicationPolicy{}
if result != nil {
total = result.Total
for _, policy := range result.Policies {
resource := rbac.NewProjectNamespace(policy.ProjectIDs[0]).Resource(rbac.ResourceReplication)
if !pa.SecurityCtx.Can(rbac.ActionRead, resource) {
continue
}
ply, err := convertFromRepPolicy(pa.ProjectMgr, *policy)
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err)
return
}
policies = append(policies, ply)
}
}
pa.SetPaginationHeader(total, queryParam.Page, queryParam.PageSize)
pa.Data["json"] = policies
pa.ServeJSON()
}
// Post creates a replicartion policy
func (pa *RepPolicyAPI) Post() {
policy := &api_models.ReplicationPolicy{}
isValid, err := pa.DecodeJSONReqAndValidate(policy)
// List the replication policies
func (r *ReplicationPolicyAPI) List() {
page, size, err := r.GetPaginationParams()
if err != nil {
r.SendInternalServerError(err)
return
}
// TODO: support more query
query := &model.PolicyQuery{
Name: r.GetString("name"),
Pagination: common_model.Pagination{
Page: page,
Size: size,
},
}
total, policies, err := replication.PolicyCtl.List(query)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to list policies: %v", err))
return
}
for _, policy := range policies {
if err = populateRegistries(replication.RegistryMgr, policy); err != nil {
r.SendInternalServerError(fmt.Errorf("failed to populate registries for policy %d: %v", policy.ID, err))
return
}
}
r.SetPaginationHeader(total, query.Page, query.Size)
r.WriteJSONData(policies)
}
// Create the replication policy
func (r *ReplicationPolicyAPI) Create() {
policy := &model.Policy{}
isValid, err := r.DecodeJSONReqAndValidate(policy)
if !isValid {
pa.SendBadRequestError(err)
r.SendBadRequestError(err)
return
}
// check the name
exist, err := exist(policy.Name)
if !r.validateName(policy) {
return
}
if !r.validateRegistry(policy) {
return
}
id, err := replication.PolicyCtl.Create(policy)
if err != nil {
pa.SendInternalServerError(fmt.Errorf("failed to check the existence of policy %s: %v", policy.Name, err))
r.SendInternalServerError(fmt.Errorf("failed to create the policy: %v", err))
return
}
if exist {
pa.SendConflictError(fmt.Errorf("name %s is already used", policy.Name))
return
}
// check the existence of projects
for _, project := range policy.Projects {
pro, err := pa.ProjectMgr.Get(project.ProjectID)
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err)
return
}
if pro == nil {
pa.SendNotFoundError(fmt.Errorf("project %d not found", project.ProjectID))
return
}
project.Name = pro.Name
}
// check the existence of targets
for _, target := range policy.Targets {
t, err := dao.GetRepTarget(target.ID)
if err != nil {
pa.SendInternalServerError(fmt.Errorf("failed to get target %d: %v", target.ID, err))
return
}
if t == nil {
pa.SendNotFoundError(fmt.Errorf("target %d not found", target.ID))
return
}
}
// check the existence of labels
for _, filter := range policy.Filters {
if filter.Kind == replication.FilterItemKindLabel {
labelID := filter.Value.(int64)
label, err := dao.GetLabel(labelID)
if err != nil {
pa.SendInternalServerError(fmt.Errorf("failed to get label %d: %v", labelID, err))
return
}
if label == nil || label.Deleted {
pa.SendNotFoundError(fmt.Errorf("label %d not found", labelID))
return
}
}
}
id, err := core.GlobalController.CreatePolicy(convertToRepPolicy(policy))
if err != nil {
pa.SendInternalServerError(fmt.Errorf("failed to create policy: %v", err))
return
}
if policy.ReplicateExistingImageNow {
go func() {
if _, err = startReplication(id); err != nil {
log.Errorf("failed to send replication signal for policy %d: %v", id, err)
return
}
log.Infof("replication signal for policy %d sent", id)
}()
}
pa.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
r.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
}
func exist(name string) (bool, error) {
result, err := core.GlobalController.GetPolicies(rep_models.QueryParameter{
Name: name,
})
// make sure the policy name doesn't exist
func (r *ReplicationPolicyAPI) validateName(policy *model.Policy) bool {
p, err := replication.PolicyCtl.GetByName(policy.Name)
if err != nil {
return false, err
r.SendInternalServerError(fmt.Errorf("failed to get policy %s: %v", policy.Name, err))
return false
}
for _, policy := range result.Policies {
if policy.Name == name {
return true, nil
}
if p != nil {
r.SendConflictError(fmt.Errorf("policy %s already exists", policy.Name))
return false
}
return false, nil
return true
}
// Put updates the replication policy
func (pa *RepPolicyAPI) Put() {
id, err := pa.GetIDFromURL()
// make sure the registry referred exists
func (r *ReplicationPolicyAPI) validateRegistry(policy *model.Policy) bool {
var registryID int64
if policy.SrcRegistry != nil && policy.SrcRegistry.ID > 0 {
registryID = policy.SrcRegistry.ID
} else {
registryID = policy.DestRegistry.ID
}
registry, err := replication.RegistryMgr.Get(registryID)
if err != nil {
pa.SendBadRequestError(err)
r.SendConflictError(fmt.Errorf("failed to get registry %d: %v", registryID, err))
return false
}
if registry == nil {
r.SendNotFoundError(fmt.Errorf("registry %d not found", registryID))
return false
}
return true
}
// Get the specified replication policy
func (r *ReplicationPolicyAPI) Get() {
id, err := r.GetInt64FromPath(":id")
if id <= 0 || err != nil {
r.SendBadRequestError(errors.New("invalid policy ID"))
return
}
originalPolicy, err := core.GlobalController.GetPolicy(id)
policy, err := replication.PolicyCtl.Get(id)
if err != nil {
log.Errorf("failed to get policy %d: %v", id, err)
pa.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", id, err))
r.SendInternalServerError(fmt.Errorf("failed to get the policy %d: %v", id, err))
return
}
if policy == nil {
r.SendNotFoundError(fmt.Errorf("policy %d not found", id))
return
}
if err = populateRegistries(replication.RegistryMgr, policy); err != nil {
r.SendInternalServerError(fmt.Errorf("failed to populate registries for policy %d: %v", policy.ID, err))
return
}
if originalPolicy.ID == 0 {
pa.SendNotFoundError(fmt.Errorf("policy %d not found", id))
r.WriteJSONData(policy)
}
// Update the replication policy
func (r *ReplicationPolicyAPI) Update() {
id, err := r.GetInt64FromPath(":id")
if id <= 0 || err != nil {
r.SendBadRequestError(errors.New("invalid policy ID"))
return
}
policy := &api_models.ReplicationPolicy{}
isValid, err := pa.DecodeJSONReqAndValidate(policy)
originalPolicy, err := replication.PolicyCtl.Get(id)
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to get the policy %d: %v", id, err))
return
}
if originalPolicy == nil {
r.SendNotFoundError(fmt.Errorf("policy %d not found", id))
return
}
policy := &model.Policy{}
isValid, err := r.DecodeJSONReqAndValidate(policy)
if !isValid {
pa.SendBadRequestError(err)
r.SendBadRequestError(err)
return
}
if policy.Name != originalPolicy.Name &&
!r.validateName(policy) {
return
}
if !r.validateRegistry(policy) {
return
}
policy.ID = id
// check the name
if policy.Name != originalPolicy.Name {
exist, err := exist(policy.Name)
if err != nil {
pa.SendInternalServerError(fmt.Errorf("failed to check the existence of policy %s: %v", policy.Name, err))
return
}
if exist {
pa.SendConflictError(fmt.Errorf("name %s is already used", policy.Name))
return
}
}
// check the existence of projects
for _, project := range policy.Projects {
pro, err := pa.ProjectMgr.Get(project.ProjectID)
if err != nil {
pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err)
return
}
if pro == nil {
pa.SendNotFoundError(fmt.Errorf("project %d not found", project.ProjectID))
return
}
project.Name = pro.Name
}
// check the existence of targets
for _, target := range policy.Targets {
t, err := dao.GetRepTarget(target.ID)
if err != nil {
pa.SendInternalServerError(fmt.Errorf("failed to get target %d: %v", target.ID, err))
return
}
if t == nil {
pa.SendNotFoundError(fmt.Errorf("target %d not found", target.ID))
return
}
}
// check the existence of labels
for _, filter := range policy.Filters {
if filter.Kind == replication.FilterItemKindLabel {
labelID := filter.Value.(int64)
label, err := dao.GetLabel(labelID)
if err != nil {
pa.SendInternalServerError(fmt.Errorf("failed to get label %d: %v", labelID, err))
return
}
if label == nil || label.Deleted {
pa.SendNotFoundError(fmt.Errorf("label %d not found", labelID))
return
}
}
}
if err = core.GlobalController.UpdatePolicy(convertToRepPolicy(policy)); err != nil {
pa.SendInternalServerError(fmt.Errorf("failed to update policy %d: %v", id, err))
if err := replication.PolicyCtl.Update(policy); err != nil {
r.SendInternalServerError(fmt.Errorf("failed to update the policy %d: %v", id, err))
return
}
if policy.ReplicateExistingImageNow {
go func() {
if _, err = startReplication(id); err != nil {
log.Errorf("failed to send replication signal for policy %d: %v", id, err)
return
}
log.Infof("replication signal for policy %d sent", id)
}()
}
}
// Delete the replication policy
func (pa *RepPolicyAPI) Delete() {
id, err := pa.GetIDFromURL()
func (r *ReplicationPolicyAPI) Delete() {
id, err := r.GetInt64FromPath(":id")
if id <= 0 || err != nil {
r.SendBadRequestError(errors.New("invalid policy ID"))
return
}
policy, err := replication.PolicyCtl.Get(id)
if err != nil {
pa.SendBadRequestError(err)
r.SendInternalServerError(fmt.Errorf("failed to get the policy %d: %v", id, err))
return
}
policy, err := core.GlobalController.GetPolicy(id)
if err != nil {
log.Errorf("failed to get policy %d: %v", id, err)
pa.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", id, err))
return
}
if policy.ID == 0 {
pa.SendNotFoundError(fmt.Errorf("policy %d not found", id))
return
}
count, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{
PolicyID: id,
Statuses: []string{models.JobRunning, models.JobRetrying, models.JobPending},
// only get the transfer and delete jobs, do not get schedule job
Operations: []string{models.RepOpTransfer, models.RepOpDelete},
})
if err != nil {
log.Errorf("failed to filter jobs of policy %d: %v", id, err)
pa.SendInternalServerError(fmt.Errorf("failed to filter jobs of policy %d: %v", id, err))
return
}
if count > 0 {
pa.SendPreconditionFailedError(errors.New("policy has running/retrying/pending jobs, can not be deleted"))
return
}
if err = core.GlobalController.RemovePolicy(id); err != nil {
log.Errorf("failed to delete policy %d: %v", id, err)
pa.SendInternalServerError(fmt.Errorf("failed to delete policy %d: %v", id, err))
return
}
}
func convertFromRepPolicy(projectMgr promgr.ProjectManager, policy rep_models.ReplicationPolicy) (*api_models.ReplicationPolicy, error) {
if policy.ID == 0 {
return nil, nil
}
// populate simple properties
ply := &api_models.ReplicationPolicy{
ID: policy.ID,
Name: policy.Name,
Description: policy.Description,
ReplicateDeletion: policy.ReplicateDeletion,
Trigger: policy.Trigger,
CreationTime: policy.CreationTime,
UpdateTime: policy.UpdateTime,
}
// populate projects
for _, projectID := range policy.ProjectIDs {
project, err := projectMgr.Get(projectID)
if err != nil {
return nil, err
}
ply.Projects = append(ply.Projects, project)
}
// populate targets
for _, targetID := range policy.TargetIDs {
target, err := dao.GetRepTarget(targetID)
if err != nil {
return nil, err
}
target.Password = ""
ply.Targets = append(ply.Targets, target)
}
// populate label used in label filter
for _, filter := range policy.Filters {
if filter.Kind == replication.FilterItemKindLabel {
labelID := filter.Value.(int64)
label, err := dao.GetLabel(labelID)
if err != nil {
return nil, err
}
filter.Value = label
}
ply.Filters = append(ply.Filters, filter)
}
// TODO call the method from replication controller
errJobCount, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{
PolicyID: policy.ID,
Statuses: []string{models.JobError},
})
if err != nil {
return nil, err
}
ply.ErrorJobCount = errJobCount
return ply, nil
}
func convertToRepPolicy(policy *api_models.ReplicationPolicy) rep_models.ReplicationPolicy {
if policy == nil {
return rep_models.ReplicationPolicy{}
r.SendNotFoundError(fmt.Errorf("policy %d not found", id))
return
}
ply := rep_models.ReplicationPolicy{
ID: policy.ID,
Name: policy.Name,
Description: policy.Description,
Filters: policy.Filters,
ReplicateDeletion: policy.ReplicateDeletion,
Trigger: policy.Trigger,
CreationTime: policy.CreationTime,
UpdateTime: policy.UpdateTime,
_, executions, err := replication.OperationCtl.ListExecutions(&models.ExecutionQuery{
PolicyID: id,
})
if err != nil {
r.SendInternalServerError(fmt.Errorf("failed to get the executions of policy %d: %v", id, err))
return
}
for _, project := range policy.Projects {
ply.ProjectIDs = append(ply.ProjectIDs, project.ProjectID)
ply.Namespaces = append(ply.Namespaces, project.Name)
for _, execution := range executions {
if execution.Status == models.ExecutionStatusInProgress {
r.SendInternalServerError(fmt.Errorf("the policy %d has running executions, can not be deleted", id))
return
}
}
for _, target := range policy.Targets {
ply.TargetIDs = append(ply.TargetIDs, target.ID)
if err := replication.PolicyCtl.Remove(id); err != nil {
r.SendInternalServerError(fmt.Errorf("failed to delete the policy %d: %v", id, err))
return
}
return ply
}
// ignore the credential for the registries
func populateRegistries(registryMgr registry.Manager, policy *model.Policy) error {
if err := event.PopulateRegistries(registryMgr, policy); err != nil {
return err
}
if policy.SrcRegistry != nil {
policy.SrcRegistry.Credential = nil
}
if policy.DestRegistry != nil {
policy.DestRegistry.Credential = nil
}
return nil
}

943
src/core/api/replication_policy_test.go
View File

File diff suppressed because it is too large Load Diff

92
src/core/api/replication_test.go
View File

@ -1,92 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"testing"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
api_models "github.com/goharbor/harbor/src/core/api/models"
"github.com/goharbor/harbor/src/replication"
"github.com/stretchr/testify/require"
)
const (
replicationAPIBaseURL = "/api/replications"
)
func TestReplicationAPIPost(t *testing.T) {
targetID, err := dao.AddRepTarget(
models.RepTarget{
Name: "test_replication_target",
URL: "127.0.0.1",
Username: "username",
Password: "password",
})
require.Nil(t, err)
defer dao.DeleteRepTarget(targetID)
policyID, err := dao.AddRepPolicy(
models.RepPolicy{
Name: "test_replication_policy",
ProjectID: 1,
TargetID: targetID,
Trigger: fmt.Sprintf("{\"kind\":\"%s\"}", replication.TriggerKindManual),
})
require.Nil(t, err)
defer dao.DeleteRepPolicy(policyID)
cases := []*codeCheckingCase{
// 401
{
request: &testingRequest{
method: http.MethodPost,
url: replicationAPIBaseURL,
bodyJSON: &api_models.Replication{
PolicyID: policyID,
},
},
code: http.StatusUnauthorized,
},
// 404
{
request: &testingRequest{
method: http.MethodPost,
url: replicationAPIBaseURL,
bodyJSON: &api_models.Replication{
PolicyID: 10000,
},
credential: admin,
},
code: http.StatusNotFound,
},
// 200
{
request: &testingRequest{
method: http.MethodPost,
url: replicationAPIBaseURL,
bodyJSON: &api_models.Replication{
PolicyID: policyID,
},
credential: admin,
},
code: http.StatusOK,
},
}
runCodeCheckingCases(t, cases...)
}

29
src/core/api/repository.go
View File

@ -38,10 +38,10 @@ import (
"github.com/goharbor/harbor/src/common/utils/notary"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/notifier"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/replication/event/notification"
"github.com/goharbor/harbor/src/replication/event/topic"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
)
// RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put
@ -330,15 +330,22 @@ func (ra *RepositoryAPI) Delete() {
log.Infof("delete tag: %s:%s", repoName, t)
go func(tag string) {
image := repoName + ":" + tag
err := notifier.Publish(topic.ReplicationEventTopicOnDeletion, notification.OnDeletionNotification{
Image: image,
})
if err != nil {
log.Errorf("failed to publish on deletion topic for resource %s: %v", image, err)
return
e := &event.Event{
Type: event.EventTypeImagePush,
Resource: &model.Resource{
Type: model.ResourceTypeImage,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: repoName,
},
Vtags: []string{tag},
},
Deleted: true,
},
}
if err := replication.EventHandler.Handle(e); err != nil {
log.Errorf("failed to handle event: %v", err)
}
log.Debugf("the on deletion topic for resource %s published", image)
}(t)
go func(tag string) {

415
src/core/api/target.go
View File

@ -1,415 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"errors"
"fmt"
"net/http"
"strconv"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/core/config"
)
// TargetAPI handles request to /api/targets/ping /api/targets/{}
type TargetAPI struct {
BaseController
secretKey string
}
// Prepare validates the user
func (t *TargetAPI) Prepare() {
t.BaseController.Prepare()
if !t.SecurityCtx.IsAuthenticated() {
t.SendUnAuthorizedError(errors.New("UnAuthorized"))
return
}
if !t.SecurityCtx.IsSysAdmin() {
t.SendForbiddenError(errors.New(t.SecurityCtx.GetUsername()))
return
}
var err error
t.secretKey, err = config.SecretKey()
if err != nil {
log.Errorf("failed to get secret key: %v", err)
t.SendInternalServerError(errors.New(""))
return
}
}
func (t *TargetAPI) ping(endpoint, username, password string, insecure bool) {
registry, err := newRegistryClient(endpoint, insecure, username, password)
if err == nil {
err = registry.Ping()
}
if err != nil {
log.Errorf("failed to ping target: %v", err)
// do not return any detail information of the error, or may cause SSRF security issue #3755
t.SendConflictError(errors.New("failed to ping target"))
return
}
}
// Ping validates whether the target is reachable and whether the credential is valid
func (t *TargetAPI) Ping() {
req := struct {
ID *int64 `json:"id"`
Endpoint *string `json:"endpoint"`
Username *string `json:"username"`
Password *string `json:"password"`
Insecure *bool `json:"insecure"`
}{}
if err := t.DecodeJSONReq(&req); err != nil {
t.SendBadRequestError(err)
return
}
target := &models.RepTarget{}
if req.ID != nil {
var err error
target, err = dao.GetRepTarget(*req.ID)
if err != nil {
t.SendInternalServerError(fmt.Errorf("failed to get target %d: %v", *req.ID, err))
return
}
if target == nil {
t.SendNotFoundError(fmt.Errorf("target %d not found", *req.ID))
return
}
if len(target.Password) != 0 {
target.Password, err = utils.ReversibleDecrypt(target.Password, t.secretKey)
if err != nil {
t.SendInternalServerError(fmt.Errorf("failed to decrypt password: %v", err))
return
}
}
}
if req.Endpoint != nil {
url, err := utils.ParseEndpoint(*req.Endpoint)
if err != nil {
t.SendBadRequestError(err)
return
}
// Prevent SSRF security issue #3755
target.URL = url.Scheme + "://" + url.Host + url.Path
}
if req.Username != nil {
target.Username = *req.Username
}
if req.Password != nil {
target.Password = *req.Password
}
if req.Insecure != nil {
target.Insecure = *req.Insecure
}
t.ping(target.URL, target.Username, target.Password, target.Insecure)
}
// Get ...
func (t *TargetAPI) Get() {
id, err := t.GetIDFromURL()
if err != nil {
t.SendBadRequestError(err)
return
}
target, err := dao.GetRepTarget(id)
if err != nil {
log.Errorf("failed to get target %d: %v", id, err)
t.SendInternalServerError(errors.New(""))
return
}
if target == nil {
t.SendNotFoundError(fmt.Errorf("target %d not found", id))
return
}
target.Password = ""
t.Data["json"] = target
t.ServeJSON()
}
// List ...
func (t *TargetAPI) List() {
name := t.GetString("name")
targets, err := dao.FilterRepTargets(name)
if err != nil {
log.Errorf("failed to filter targets %s: %v", name, err)
t.SendInternalServerError(errors.New(""))
return
}
for _, target := range targets {
target.Password = ""
}
t.Data["json"] = targets
t.ServeJSON()
return
}
// Post ...
func (t *TargetAPI) Post() {
target := &models.RepTarget{}
isValid, err := t.DecodeJSONReqAndValidate(target)
if !isValid {
t.SendBadRequestError(err)
return
}
ta, err := dao.GetRepTargetByName(target.Name)
if err != nil {
log.Errorf("failed to get target %s: %v", target.Name, err)
t.SendInternalServerError(errors.New(""))
return
}
if ta != nil {
t.SendConflictError(errors.New("name is already used"))
return
}
ta, err = dao.GetRepTargetByEndpoint(target.URL)
if err != nil {
log.Errorf("failed to get target [ %s ]: %v", target.URL, err)
t.SendInternalServerError(errors.New(""))
return
}
if ta != nil {
t.SendConflictError(fmt.Errorf("the target whose endpoint is %s already exists", target.URL))
return
}
if len(target.Password) != 0 {
target.Password, err = utils.ReversibleEncrypt(target.Password, t.secretKey)
if err != nil {
log.Errorf("failed to encrypt password: %v", err)
t.SendInternalServerError(errors.New(""))
return
}
}
id, err := dao.AddRepTarget(*target)
if err != nil {
log.Errorf("failed to add target: %v", err)
t.SendInternalServerError(errors.New(""))
return
}
t.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
}
// Put ...
func (t *TargetAPI) Put() {
id, err := t.GetIDFromURL()
if err != nil {
t.SendBadRequestError(err)
return
}
target, err := dao.GetRepTarget(id)
if err != nil {
log.Errorf("failed to get target %d: %v", id, err)
t.SendInternalServerError(errors.New(""))
return
}
if target == nil {
t.SendNotFoundError(fmt.Errorf("target %d not found", id))
return
}
if len(target.Password) != 0 {
target.Password, err = utils.ReversibleDecrypt(target.Password, t.secretKey)
if err != nil {
log.Errorf("failed to decrypt password: %v", err)
t.SendInternalServerError(errors.New(""))
return
}
}
req := struct {
Name *string `json:"name"`
Endpoint *string `json:"endpoint"`
Username *string `json:"username"`
Password *string `json:"password"`
Insecure *bool `json:"insecure"`
}{}
if err := t.DecodeJSONReq(&req); err != nil {
t.SendBadRequestError(err)
return
}
originalName := target.Name
originalURL := target.URL
if req.Name != nil {
target.Name = *req.Name
}
if req.Endpoint != nil {
target.URL = *req.Endpoint
}
if req.Username != nil {
target.Username = *req.Username
}
if req.Password != nil {
target.Password = *req.Password
}
if req.Insecure != nil {
target.Insecure = *req.Insecure
}
t.Validate(target)
if target.Name != originalName {
ta, err := dao.GetRepTargetByName(target.Name)
if err != nil {
log.Errorf("failed to get target %s: %v", target.Name, err)
t.SendInternalServerError(errors.New(""))
return
}
if ta != nil {
t.SendConflictError(errors.New("name is already used"))
return
}
}
if target.URL != originalURL {
ta, err := dao.GetRepTargetByEndpoint(target.URL)
if err != nil {
log.Errorf("failed to get target [ %s ]: %v", target.URL, err)
t.SendInternalServerError(errors.New(""))
return
}
if ta != nil {
t.SendConflictError(fmt.Errorf("the target whose endpoint is %s already exists", target.URL))
return
}
}
if len(target.Password) != 0 {
target.Password, err = utils.ReversibleEncrypt(target.Password, t.secretKey)
if err != nil {
log.Errorf("failed to encrypt password: %v", err)
t.SendInternalServerError(errors.New(""))
return
}
}
if err := dao.UpdateRepTarget(*target); err != nil {
log.Errorf("failed to update target %d: %v", id, err)
t.SendInternalServerError(errors.New(""))
return
}
}
// Delete ...
func (t *TargetAPI) Delete() {
id, err := t.GetIDFromURL()
if err != nil {
t.SendBadRequestError(err)
return
}
target, err := dao.GetRepTarget(id)
if err != nil {
log.Errorf("failed to get target %d: %v", id, err)
t.SendInternalServerError(errors.New(""))
return
}
if target == nil {
t.SendNotFoundError(fmt.Errorf("target %d not found", id))
return
}
policies, err := dao.GetRepPolicyByTarget(id)
if err != nil {
log.Errorf("failed to get policies according target %d: %v", id, err)
t.SendInternalServerError(errors.New(""))
return
}
if len(policies) > 0 {
log.Error("the target is used by policies, can not be deleted")
t.SendPreconditionFailedError(errors.New("the target is used by policies, can not be deleted"))
return
}
if err = dao.DeleteRepTarget(id); err != nil {
log.Errorf("failed to delete target %d: %v", id, err)
t.SendInternalServerError(errors.New(""))
return
}
}
func newRegistryClient(endpoint string, insecure bool, username, password string) (*registry.Registry, error) {
transport := registry.GetHTTPTransport(insecure)
credential := auth.NewBasicAuthCredential(username, password)
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: transport,
}, credential)
return registry.NewRegistry(endpoint, &http.Client{
Transport: registry.NewTransport(transport, authorizer),
})
}
// ListPolicies ...
func (t *TargetAPI) ListPolicies() {
id, err := t.GetIDFromURL()
if err != nil {
t.SendBadRequestError(err)
return
}
target, err := dao.GetRepTarget(id)
if err != nil {
log.Errorf("failed to get target %d: %v", id, err)
t.SendInternalServerError(errors.New(""))
return
}
if target == nil {
t.SendNotFoundError(fmt.Errorf("target %d not found", id))
return
}
policies, err := dao.GetRepPolicyByTarget(id)
if err != nil {
log.Errorf("failed to get policies according target %d: %v", id, err)
t.SendInternalServerError(errors.New(""))
return
}
t.Data["json"] = policies
t.ServeJSON()
}

289
src/core/api/target_test.go
View File

@ -1,289 +0,0 @@
// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"os"
"strconv"
"testing"
"github.com/goharbor/harbor/tests/apitests/apilib"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
addTargetName = "testTargets"
)
var addTargetID int
func TestTargetsPost(t *testing.T) {
var httpStatusCode int
var err error
assert := assert.New(t)
apiTest := newHarborAPI()
endPoint := os.Getenv("REGISTRY_URL")
repTargets := &apilib.RepTargetPost{Endpoint: endPoint, Name: addTargetName, Username: adminName, Password: adminPwd}
fmt.Println("Testing Targets Post API")
// -------------------case 1 : response code = 201------------------------//
fmt.Println("case 1 : response code = 201")
httpStatusCode, body, err := apiTest.AddTargets(*admin, *repTargets)
if err != nil {
t.Error("Error whihle add targets", err.Error())
t.Log(err)
} else {
assert.Equal(int(201), httpStatusCode, "httpStatusCode should be 201")
t.Log(body)
}
// -----------case 2 : response code = 409,name is already used-----------//
fmt.Println("case 2 : response code = 409,name is already used")
httpStatusCode, _, err = apiTest.AddTargets(*admin, *repTargets)
if err != nil {
t.Error("Error whihle add targets", err.Error())
t.Log(err)
} else {
assert.Equal(int(409), httpStatusCode, "httpStatusCode should be 409")
}
// -----------case 3 : response code = 409,name is already used-----------//
fmt.Println("case 3 : response code = 409,endPoint is already used")
repTargets.Username = "errName"
httpStatusCode, _, err = apiTest.AddTargets(*admin, *repTargets)
if err != nil {
t.Error("Error whihle add targets", err.Error())
t.Log(err)
} else {
assert.Equal(int(409), httpStatusCode, "httpStatusCode should be 409")
}
// --------case 4 : response code = 401,User need to log in first.--------//
fmt.Println("case 4 : response code = 401,User need to log in first.")
httpStatusCode, _, err = apiTest.AddTargets(*unknownUsr, *repTargets)
if err != nil {
t.Error("Error whihle add targets", err.Error())
t.Log(err)
} else {
assert.Equal(int(401), httpStatusCode, "httpStatusCode should be 401")
}
fmt.Printf("\n")
}
func TestTargetsGet(t *testing.T) {
var httpStatusCode int
var err error
var reslut []apilib.RepTarget
assert := assert.New(t)
apiTest := newHarborAPI()
fmt.Println("Testing Targets Get API")
// -------------------case 1 : response code = 200------------------------//
fmt.Println("case 1 : response code = 200")
httpStatusCode, reslut, err = apiTest.ListTargets(*admin, addTargetName)
if err != nil {
t.Error("Error whihle get targets", err.Error())
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
addTargetID = int(reslut[0].Id)
}
}
func TestTargetPing(t *testing.T) {
apiTest := newHarborAPI()
// 404: not exist target
target01 := struct {
ID int64 `json:"id"`
}{
ID: 10000,
}
code, err := apiTest.PingTarget(*admin, target01)
require.Nil(t, err)
assert.Equal(t, http.StatusNotFound, code)
// 400: empty endpoint
target02 := struct {
Endpoint string `json:"endpoint"`
}{
Endpoint: "",
}
code, err = apiTest.PingTarget(*admin, target02)
require.Nil(t, err)
assert.Equal(t, http.StatusBadRequest, code)
// 200
target03 := struct {
ID int64 `json:"id"`
Endpoint string `json:"endpoint"`
Username string `json:"username"`
Password string `json:"password"`
Insecure bool `json:"insecure"`
}{
ID: int64(addTargetID),
Endpoint: os.Getenv("REGISTRY_URL"),
Username: adminName,
Password: adminPwd,
Insecure: true,
}
code, err = apiTest.PingTarget(*admin, target03)
require.Nil(t, err)
assert.Equal(t, http.StatusOK, code)
}
func TestTargetGetByID(t *testing.T) {
var httpStatusCode int
var err error
assert := assert.New(t)
apiTest := newHarborAPI()
fmt.Println("Testing Targets Get API by Id")
// -------------------case 1 : response code = 200------------------------//
fmt.Println("case 1 : response code = 200")
id := strconv.Itoa(addTargetID)
httpStatusCode, err = apiTest.GetTargetByID(*admin, id)
if err != nil {
t.Error("Error whihle get target by id", err.Error())
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
}
// --------------case 2 : response code = 404,target not found------------//
fmt.Println("case 2 : response code = 404,target not found")
id = "1111"
httpStatusCode, err = apiTest.GetTargetByID(*admin, id)
if err != nil {
t.Error("Error whihle get target by id", err.Error())
t.Log(err)
} else {
assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404")
}
}
func TestTargetsPut(t *testing.T) {
var httpStatusCode int
var err error
assert := assert.New(t)
apiTest := newHarborAPI()
endPoint := "1.1.1.1"
updateRepTargets := &apilib.RepTargetPost{Endpoint: endPoint, Name: addTargetName, Username: adminName, Password: adminPwd}
id := strconv.Itoa(addTargetID)
fmt.Println("Testing Target Put API")
// -------------------case 1 : response code = 200------------------------//
fmt.Println("case 1 : response code = 200")
httpStatusCode, err = apiTest.PutTargetByID(*admin, id, *updateRepTargets)
if err != nil {
t.Error("Error whihle update target", err.Error())
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
}
// --------------case 2 : response code = 404,target not found------------//
id = "111"
fmt.Println("case 2 : response code = 404,target not found")
httpStatusCode, err = apiTest.PutTargetByID(*admin, id, *updateRepTargets)
if err != nil {
t.Error("Error whihle update target", err.Error())
t.Log(err)
} else {
assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404")
}
}
func TestTargetGetPolicies(t *testing.T) {
var httpStatusCode int
var err error
assert := assert.New(t)
apiTest := newHarborAPI()
fmt.Println("Testing Targets Get API to list policies")
// -------------------case 1 : response code = 200------------------------//
fmt.Println("case 1 : response code = 200")
id := strconv.Itoa(addTargetID)
httpStatusCode, err = apiTest.GetTargetPoliciesByID(*admin, id)
if err != nil {
t.Error("Error whihle get target by id", err.Error())
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
}
// --------------case 2 : response code = 404,target not found------------//
fmt.Println("case 2 : response code = 404,target not found")
id = "1111"
httpStatusCode, err = apiTest.GetTargetPoliciesByID(*admin, id)
if err != nil {
t.Error("Error whihle get target by id", err.Error())
t.Log(err)
} else {
assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404")
}
}
func TestTargetsDelete(t *testing.T) {
var httpStatusCode int
var err error
assert := assert.New(t)
apiTest := newHarborAPI()
id := strconv.Itoa(addTargetID)
fmt.Println("Testing Targets Delete API")
// -------------------case 1 : response code = 200------------------------//
fmt.Println("case 1 : response code = 200")
httpStatusCode, err = apiTest.DeleteTargetsByID(*admin, id)
if err != nil {
t.Error("Error whihle delete targets", err.Error())
t.Log(err)
} else {
assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200")
}
// --------------case 2 : response code = 404,target not found------------//
fmt.Println("case 2 : response code = 404,target not found")
id = "1111"
httpStatusCode, err = apiTest.DeleteTargetsByID(*admin, id)
if err != nil {
t.Error("Error whihle delete targets", err.Error())
t.Log(err)
} else {
assert.Equal(int(404), httpStatusCode, "httpStatusCode should be 404")
}
}

19
src/core/main.go
View File

@ -18,11 +18,12 @@ import (
"encoding/gob"
"fmt"
"os"
"os/signal"
"strconv"
"syscall"
"github.com/astaxie/beego"
_ "github.com/astaxie/beego/session/redis"
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils"
@ -36,8 +37,7 @@ import (
"github.com/goharbor/harbor/src/core/filter"
"github.com/goharbor/harbor/src/core/proxy"
"github.com/goharbor/harbor/src/core/service/token"
"github.com/goharbor/harbor/src/replication/core"
_ "github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication"
)
const (
@ -70,6 +70,13 @@ func updateInitPassword(userID int, password string) error {
return nil
}
func gracefulShutdown(closing chan struct{}) {
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
log.Infof("capture system signal %s, to close \"closing\" channel", <-signals)
close(closing)
}
func main() {
beego.BConfig.WebConfig.Session.SessionOn = true
beego.BConfig.WebConfig.Session.SessionName = "sid"
@ -122,8 +129,10 @@ func main() {
}
}
if err := core.Init(); err != nil {
log.Errorf("failed to initialize the replication controller: %v", err)
closing := make(chan struct{})
go gracefulShutdown(closing)
if err := replication.Init(closing); err != nil {
log.Fatalf("failed to init for replication: %v", err)
}
filter.Init()

31
src/core/router.go
View File

@ -88,9 +88,6 @@ func initRouters() {
beego.Router("/api/repositories/*/tags/:tag/manifest", &api.RepositoryAPI{}, "get:GetManifests")
beego.Router("/api/repositories/*/signatures", &api.RepositoryAPI{}, "get:GetSignatures")
beego.Router("/api/repositories/top", &api.RepositoryAPI{}, "get:GetTopRepos")
beego.Router("/api/jobs/replication/", &api.RepJobAPI{}, "get:List;put:StopJobs")
beego.Router("/api/jobs/replication/:id([0-9]+)", &api.RepJobAPI{})
beego.Router("/api/jobs/replication/:id([0-9]+)/log", &api.RepJobAPI{}, "get:GetLog")
beego.Router("/api/jobs/scan/:id([0-9]+)/log", &api.ScanJobAPI{}, "get:GetLog")
beego.Router("/api/system/gc", &api.GCAPI{}, "get:List")
@ -99,20 +96,20 @@ func initRouters() {
beego.Router("/api/system/gc/schedule", &api.GCAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/system/scanAll/schedule", &api.ScanAllAPI{}, "get:Get;put:Put;post:Post")
beego.Router("/api/policies/replication/:id([0-9]+)", &api.RepPolicyAPI{})
beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "get:List")
beego.Router("/api/policies/replication", &api.RepPolicyAPI{}, "post:Post")
beego.Router("/api/targets/", &api.TargetAPI{}, "get:List")
beego.Router("/api/targets/", &api.TargetAPI{}, "post:Post")
beego.Router("/api/targets/:id([0-9]+)", &api.TargetAPI{})
beego.Router("/api/targets/:id([0-9]+)/policies/", &api.TargetAPI{}, "get:ListPolicies")
beego.Router("/api/targets/ping", &api.TargetAPI{}, "post:Ping")
beego.Router("/api/logs", &api.LogAPI{})
beego.Router("/api/replication/adapters", &api.ReplicationAdapterAPI{}, "get:List")
beego.Router("/api/replication/executions", &api.ReplicationOperationAPI{}, "get:ListExecutions;post:CreateExecution")
beego.Router("/api/replication/executions/:id([0-9]+)", &api.ReplicationOperationAPI{}, "get:GetExecution;put:StopExecution")
beego.Router("/api/replication/executions/:id([0-9]+)/tasks", &api.ReplicationOperationAPI{}, "get:ListTasks")
beego.Router("/api/replication/executions/:id([0-9]+)/tasks/:tid([0-9]+)/log", &api.ReplicationOperationAPI{}, "get:GetTaskLog")
beego.Router("/api/replication/policies", &api.ReplicationPolicyAPI{}, "get:List;post:Create")
beego.Router("/api/replication/policies/:id([0-9]+)", &api.ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete")
beego.Router("/api/internal/configurations", &api.ConfigAPI{}, "get:GetInternalConfig;put:Put")
beego.Router("/api/configurations", &api.ConfigAPI{}, "get:Get;put:Put")
beego.Router("/api/statistics", &api.StatisticAPI{})
beego.Router("/api/replications", &api.ReplicationAPI{})
beego.Router("/api/labels", &api.LabelAPI{}, "post:Post;get:List")
beego.Router("/api/labels/:id([0-9]+)", &api.LabelAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/labels/:id([0-9]+)/resources", &api.LabelAPI{}, "get:ListResources")
@ -128,10 +125,18 @@ func initRouters() {
beego.Router("/service/notifications", &registry.NotificationHandler{})
beego.Router("/service/notifications/clair", &clair.Handler{}, "post:Handle")
beego.Router("/service/notifications/jobs/scan/:id([0-9]+)", &jobs.Handler{}, "post:HandleScan")
beego.Router("/service/notifications/jobs/replication/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplication")
beego.Router("/service/notifications/jobs/adminjob/:id([0-9]+)", &admin.Handler{}, "post:HandleAdminJob")
beego.Router("/service/notifications/jobs/replication/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationScheduleJob")
beego.Router("/service/notifications/jobs/replication/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationTask")
beego.Router("/service/token", &token.Handler{})
beego.Router("/api/registries", &api.RegistryAPI{}, "get:List;post:Post")
beego.Router("/api/registries/:id([0-9]+)", &api.RegistryAPI{}, "get:Get;put:Put;delete:Delete")
beego.Router("/api/registries/ping", &api.RegistryAPI{}, "post:Ping")
// we use "0" as the ID of the local Harbor registry, so don't add "([0-9]+)" in the path
beego.Router("/api/registries/:id/info", &api.RegistryAPI{}, "get:GetInfo")
beego.Router("/api/registries/:id/namespace", &api.RegistryAPI{}, "get:GetNamespace")
beego.Router("/v2/*", &controllers.RegistryProxy{}, "*:Handle")
// APIs for chart repository

27
src/core/service/notifications/jobs/handler.go
View File

@ -23,6 +23,9 @@ import (
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/operation/hook"
"github.com/goharbor/harbor/src/replication/policy/scheduler"
)
var statusMap = map[string]string{
@ -38,8 +41,9 @@ var statusMap = map[string]string{
// Handler handles reqeust on /service/notifications/jobs/*, which listens to the webhook of jobservice.
type Handler struct {
api.BaseController
id int64
status string
id int64
status string
rawStatus string
}
// Prepare ...
@ -59,6 +63,7 @@ func (h *Handler) Prepare() {
h.Abort("200")
return
}
h.rawStatus = data.Status
status, ok := statusMap[data.Status]
if !ok {
log.Debugf("drop the job status update event: job id-%d, status-%s", id, status)
@ -78,12 +83,22 @@ func (h *Handler) HandleScan() {
}
}
// HandleReplication handles the webhook of replication job
func (h *Handler) HandleReplication() {
log.Debugf("received replication job status update event: job-%d, status-%s", h.id, h.status)
if err := dao.UpdateRepJobStatus(h.id, h.status); err != nil {
// HandleReplicationScheduleJob handles the webhook of replication schedule job
func (h *Handler) HandleReplicationScheduleJob() {
log.Debugf("received replication schedule job status update event: schedule-job-%d, status-%s", h.id, h.status)
if err := scheduler.UpdateStatus(h.id, h.status); err != nil {
log.Errorf("Failed to update job status, id: %d, status: %s", h.id, h.status)
h.SendInternalServerError(err)
return
}
}
// HandleReplicationTask handles the webhook of replication task
func (h *Handler) HandleReplicationTask() {
log.Debugf("received replication task status update event: task-%d, status-%s", h.id, h.status)
if err := hook.UpdateTask(replication.OperationCtl, h.id, h.rawStatus); err != nil {
log.Errorf("Failed to update replication task status, id: %d, status: %s", h.id, h.status)
h.SendInternalServerError(err)
return
}
}

44
src/core/service/notifications/registry/handler.go
View File

@ -27,10 +27,11 @@ import (
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/api"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/core/notifier"
coreutils "github.com/goharbor/harbor/src/core/utils"
rep_notification "github.com/goharbor/harbor/src/replication/event/notification"
"github.com/goharbor/harbor/src/replication/event/topic"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/adapter"
rep_event "github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
)
// NotificationHandler handles request on /service/notifications/, which listens to registry's events.
@ -111,16 +112,24 @@ func (n *NotificationHandler) Post() {
return
}
// TODO: handle image delete event and chart event
go func() {
image := repository + ":" + tag
err := notifier.Publish(topic.ReplicationEventTopicOnPush, rep_notification.OnPushNotification{
Image: image,
})
if err != nil {
log.Errorf("failed to publish on push topic for resource %s: %v", image, err)
return
e := &rep_event.Event{
Type: rep_event.EventTypeImagePush,
Resource: &model.Resource{
Type: model.ResourceTypeImage,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: repository,
// TODO filling the metadata
},
Vtags: []string{tag},
},
},
}
if err := replication.EventHandler.Handle(e); err != nil {
log.Errorf("failed to handle event: %v", err)
}
log.Debugf("the on push topic for resource %s published", image)
}()
if autoScanEnabled(pro) {
@ -173,15 +182,16 @@ func filterEvents(notification *models.Notification) ([]*models.Event, error) {
}
func checkEvent(event *models.Event) bool {
// pull and push manifest
if strings.ToLower(strings.TrimSpace(event.Request.UserAgent)) != "harbor-registry-client" && (event.Action == "pull" || event.Action == "push") {
// push action
if event.Action == "push" {
return true
}
// push manifest by job-service
if strings.ToLower(strings.TrimSpace(event.Request.UserAgent)) == "harbor-registry-client" && event.Action == "push" {
return true
// if it is pull action, check the user-agent
userAgent := strings.ToLower(strings.TrimSpace(event.Request.UserAgent))
if userAgent == "harbor-registry-client" || userAgent == strings.ToLower(adapter.UserAgentReplication) {
return false
}
return false
return true
}
func autoScanEnabled(project *models.Project) bool {

139
src/jobservice/job/impl/replication/delete.go
View File

@ -1,139 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package replication
import (
"net/http"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/jobservice/env"
"github.com/goharbor/harbor/src/jobservice/logger"
)
// Deleter deletes repository or images on the destination registry
type Deleter struct {
ctx env.JobContext
repository *repository
dstRegistry *registry
logger logger.Interface
retry bool
}
// ShouldRetry : retry if the error is network error
func (d *Deleter) ShouldRetry() bool {
return d.retry
}
// MaxFails ...
func (d *Deleter) MaxFails() uint {
return 3
}
// Validate ....
func (d *Deleter) Validate(params map[string]interface{}) error {
return nil
}
// Run ...
func (d *Deleter) Run(ctx env.JobContext, params map[string]interface{}) error {
err := d.run(ctx, params)
d.retry = retry(err)
return err
}
func (d *Deleter) run(ctx env.JobContext, params map[string]interface{}) error {
if err := d.init(ctx, params); err != nil {
return err
}
return d.delete()
}
func (d *Deleter) init(ctx env.JobContext, params map[string]interface{}) error {
d.logger = ctx.GetLogger()
d.ctx = ctx
if canceled(d.ctx) {
d.logger.Warning(errCanceled.Error())
return errCanceled
}
d.repository = &repository{
name: params["repository"].(string),
}
if tags, ok := params["tags"]; ok {
tgs := tags.([]interface{})
for _, tg := range tgs {
d.repository.tags = append(d.repository.tags, tg.(string))
}
}
url := params["dst_registry_url"].(string)
insecure := params["dst_registry_insecure"].(bool)
cred := auth.NewBasicAuthCredential(
params["dst_registry_username"].(string),
params["dst_registry_password"].(string))
var err error
d.dstRegistry, err = initRegistry(url, insecure, cred, d.repository.name)
if err != nil {
d.logger.Errorf("failed to create client for destination registry: %v", err)
return err
}
d.logger.Infof("initialization completed: repository: %s, tags: %v, destination URL: %s, insecure: %v",
d.repository.name, d.repository.tags, d.dstRegistry.url, d.dstRegistry.insecure)
return nil
}
func (d *Deleter) delete() error {
repository := d.repository.name
tags := d.repository.tags
if len(tags) == 0 {
if canceled(d.ctx) {
d.logger.Warning(errCanceled.Error())
return errCanceled
}
if err := d.dstRegistry.DeleteRepository(repository); err != nil {
if e, ok := err.(*common_http.Error); ok && e.Code == http.StatusNotFound {
d.logger.Warningf("repository %s not found", repository)
return nil
}
d.logger.Errorf("failed to delete repository %s: %v", repository, err)
return err
}
d.logger.Infof("repository %s has been deleted", repository)
return nil
}
for _, tag := range tags {
if canceled(d.ctx) {
d.logger.Warning(errCanceled.Error())
return errCanceled
}
if err := d.dstRegistry.DeleteImage(repository, tag); err != nil {
if e, ok := err.(*common_http.Error); ok && e.Code == http.StatusNotFound {
d.logger.Warningf("image %s:%s not found", repository, tag)
return nil
}
d.logger.Errorf("failed to delete image %s:%s: %v", repository, tag, err)
return err
}
d.logger.Infof("image %s:%s has been deleted", repository, tag)
}
return nil
}

93
src/jobservice/job/impl/replication/registry.go
View File

@ -1,93 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package replication
import (
"fmt"
"net/url"
"strconv"
"strings"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/models"
reg "github.com/goharbor/harbor/src/common/utils/registry"
)
type repository struct {
name string
tags []string
}
// registry wraps operations of Harbor UI and docker registry into one struct
type registry struct {
reg.Repository // docker registry client
client *common_http.Client // Harbor client
url string
insecure bool
}
func (r *registry) GetProject(name string) (*models.Project, error) {
url, err := url.Parse(strings.TrimRight(r.url, "/") + "/api/projects")
if err != nil {
return nil, err
}
q := url.Query()
q.Set("name", name)
url.RawQuery = q.Encode()
projects := []*models.Project{}
if err = r.client.Get(url.String(), &projects); err != nil {
return nil, err
}
for _, project := range projects {
if project.Name == name {
return project, nil
}
}
return nil, fmt.Errorf("project %s not found", name)
}
func (r *registry) CreateProject(project *models.Project) error {
// only replicate the public property of project
pro := struct {
models.ProjectRequest
Public int `json:"public"`
}{
ProjectRequest: models.ProjectRequest{
Name: project.Name,
Metadata: map[string]string{
models.ProMetaPublic: strconv.FormatBool(project.IsPublic()),
},
},
}
// put "public" property in both metadata and public field to keep compatibility
// with old version API(<=1.2.0)
if project.IsPublic() {
pro.Public = 1
}
return r.client.Post(strings.TrimRight(r.url, "/")+"/api/projects/", pro)
}
func (r *registry) DeleteRepository(repository string) error {
return r.client.Delete(strings.TrimRight(r.url, "/") + "/api/repositories/" + repository)
}
func (r *registry) DeleteImage(repository, tag string) error {
return r.client.Delete(strings.TrimRight(r.url, "/") + "/api/repositories/" + repository + "/tags/" + tag)
}

97
src/jobservice/job/impl/replication/replicate.go
View File

@ -1,97 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package replication
import (
"fmt"
"net/http"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier/auth"
reg "github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/jobservice/env"
"github.com/goharbor/harbor/src/jobservice/logger"
)
// Replicator call UI's API to start a repliation according to the policy ID
// passed in parameters
type Replicator struct {
ctx env.JobContext
url string // the URL of UI service
insecure bool
policyID int64
client *common_http.Client
logger logger.Interface
}
// ShouldRetry ...
func (r *Replicator) ShouldRetry() bool {
return false
}
// MaxFails ...
func (r *Replicator) MaxFails() uint {
return 0
}
// Validate ....
func (r *Replicator) Validate(params map[string]interface{}) error {
return nil
}
// Run ...
func (r *Replicator) Run(ctx env.JobContext, params map[string]interface{}) error {
if err := r.init(ctx, params); err != nil {
return err
}
return r.replicate()
}
func (r *Replicator) init(ctx env.JobContext, params map[string]interface{}) error {
r.logger = ctx.GetLogger()
r.ctx = ctx
if canceled(r.ctx) {
r.logger.Warning(errCanceled.Error())
return errCanceled
}
r.policyID = (int64)(params["policy_id"].(float64))
r.url = params["url"].(string)
r.insecure = params["insecure"].(bool)
cred := auth.NewSecretAuthorizer(secret())
r.client = common_http.NewClient(&http.Client{
Transport: reg.GetHTTPTransport(r.insecure),
}, cred)
r.logger.Infof("initialization completed: policy ID: %d, URL: %s, insecure: %v",
r.policyID, r.url, r.insecure)
return nil
}
func (r *Replicator) replicate() error {
if err := r.client.Post(fmt.Sprintf("%s/api/replications", r.url), struct {
PolicyID int64 `json:"policy_id"`
}{
PolicyID: r.policyID,
}); err != nil {
r.logger.Errorf("failed to send the replication request to %s: %v", r.url, err)
return err
}
r.logger.Info("the replication request has been sent successfully")
return nil
}

113
src/jobservice/job/impl/replication/replication.go Normal file
View File

@ -0,0 +1,113 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package replication
import (
"encoding/json"
"fmt"
"github.com/goharbor/harbor/src/jobservice/env"
"github.com/goharbor/harbor/src/jobservice/opm"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/transfer"
// import chart transfer
_ "github.com/goharbor/harbor/src/replication/transfer/chart"
// import image transfer
_ "github.com/goharbor/harbor/src/replication/transfer/image"
// register the Harbor adapter
_ "github.com/goharbor/harbor/src/replication/adapter/harbor"
// register the DockerHub adapter
_ "github.com/goharbor/harbor/src/replication/adapter/dockerhub"
// register the Native adapter
_ "github.com/goharbor/harbor/src/replication/adapter/native"
)
// Replication implements the job interface
type Replication struct{}
// MaxFails returns that how many times this job can fail
func (r *Replication) MaxFails() uint {
return 3
}
// ShouldRetry always returns true which means the job is needed to be restarted when fails
func (r *Replication) ShouldRetry() bool {
return true
}
// Validate does nothing
func (r *Replication) Validate(params map[string]interface{}) error {
return nil
}
// Run gets the corresponding transfer according to the resource type
// and calls its function to do the real work
func (r *Replication) Run(ctx env.JobContext, params map[string]interface{}) error {
logger := ctx.GetLogger()
src, dst, err := parseParams(params)
if err != nil {
logger.Errorf("failed to parse parameters: %v", err)
return err
}
factory, err := transfer.GetFactory(src.Type)
if err != nil {
logger.Errorf("failed to get transfer factory: %v", err)
return err
}
stopFunc := func() bool {
cmd, exist := ctx.OPCommand()
if !exist {
return false
}
return cmd == opm.CtlCommandStop
}
transfer, err := factory(ctx.GetLogger(), stopFunc)
if err != nil {
logger.Errorf("failed to create transfer: %v", err)
return err
}
return transfer.Transfer(src, dst)
}
func parseParams(params map[string]interface{}) (*model.Resource, *model.Resource, error) {
src := &model.Resource{}
if err := parseParam(params, "src_resource", src); err != nil {
return nil, nil, err
}
dst := &model.Resource{}
if err := parseParam(params, "dst_resource", dst); err != nil {
return nil, nil, err
}
return src, dst, nil
}
func parseParam(params map[string]interface{}, name string, v interface{}) error {
value, exist := params[name]
if !exist {
return fmt.Errorf("param %s not found", name)
}
str, ok := value.(string)
if !ok {
return fmt.Errorf("the value of %s isn't string", name)
}
return json.Unmarshal([]byte(str), v)
}

100
src/jobservice/job/impl/replication/replication_test.go Normal file
View File

@ -0,0 +1,100 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package replication
import (
"testing"
"github.com/goharbor/harbor/src/jobservice/job/impl"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/transfer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseParam(t *testing.T) {
params := map[string]interface{}{}
// not exist param
err := parseParam(params, "not_exist_param", nil)
assert.NotNil(t, err)
// the param is not string
params["num"] = 1
err = parseParam(params, "num", nil)
assert.NotNil(t, err)
// not a valid json struct
type person struct {
Name string
}
params["person"] = `"name": "tom"`
p := &person{}
err = parseParam(params, "person", p)
assert.NotNil(t, err)
// pass
params["person"] = `{"name": "tom"}`
err = parseParam(params, "person", p)
assert.Nil(t, err)
assert.Equal(t, "tom", p.Name)
}
func TestParseParams(t *testing.T) {
params := map[string]interface{}{
"src_resource": `{"type":"chart"}`,
"dst_resource": `{"type":"chart"}`,
}
res, dst, err := parseParams(params)
require.Nil(t, err)
assert.Equal(t, "chart", string(res.Type))
assert.Equal(t, "chart", string(dst.Type))
}
func TestMaxFails(t *testing.T) {
rep := &Replication{}
assert.Equal(t, uint(3), rep.MaxFails())
}
func TestShouldRetry(t *testing.T) {
rep := &Replication{}
assert.True(t, rep.ShouldRetry())
}
func TestValidate(t *testing.T) {
rep := &Replication{}
assert.Nil(t, rep.Validate(nil))
}
var transferred = false
var fakedTransferFactory = func(transfer.Logger, transfer.StopFunc) (transfer.Transfer, error) {
return &fakedTransfer{}, nil
}
type fakedTransfer struct{}
func (f *fakedTransfer) Transfer(src *model.Resource, dst *model.Resource) error {
transferred = true
return nil
}
func TestRun(t *testing.T) {
err := transfer.RegisterFactory("res", fakedTransferFactory)
require.Nil(t, err)
params := map[string]interface{}{
"src_resource": `{"type":"res"}`,
"dst_resource": `{}`,
}
rep := &Replication{}
require.Nil(t, rep.Run(&impl.Context{}, params))
assert.True(t, transferred)
}

78
src/jobservice/job/impl/replication/scheduler.go Normal file
View File

@ -0,0 +1,78 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package replication
import (
"fmt"
"net/http"
"os"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier/auth"
reg "github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/jobservice/env"
"github.com/goharbor/harbor/src/jobservice/errs"
"github.com/goharbor/harbor/src/jobservice/opm"
"github.com/goharbor/harbor/src/replication/model"
)
// Scheduler is a job running in Jobservice which can be used as
// a scheduler when submitting it as a scheduled job. It receives
// a URL and data, and post the data to the URL when it is running
type Scheduler struct {
ctx env.JobContext
}
// ShouldRetry ...
func (s *Scheduler) ShouldRetry() bool {
return false
}
// MaxFails ...
func (s *Scheduler) MaxFails() uint {
return 0
}
// Validate ....
func (s *Scheduler) Validate(params map[string]interface{}) error {
return nil
}
// Run ...
func (s *Scheduler) Run(ctx env.JobContext, params map[string]interface{}) error {
cmd, exist := ctx.OPCommand()
if exist && cmd == opm.CtlCommandStop {
return errs.JobStoppedError()
}
logger := ctx.GetLogger()
url := params["url"].(string)
url = fmt.Sprintf("%s/api/replication/executions?trigger=%s", url, model.TriggerTypeScheduled)
policyID := (int64)(params["policy_id"].(float64))
cred := auth.NewSecretAuthorizer(os.Getenv("JOBSERVICE_SECRET"))
client := common_http.NewClient(&http.Client{
Transport: reg.GetHTTPTransport(true),
}, cred)
if err := client.Post(url, struct {
PolicyID int64 `json:"policy_id"`
}{
PolicyID: policyID,
}); err != nil {
logger.Errorf("failed to run the schedule job: %v", err)
return err
}
logger.Info("the schedule job finished")
return nil
}

368
src/jobservice/job/impl/replication/transfer.go
View File

@ -1,368 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package replication
import (
"errors"
"fmt"
"net"
"net/http"
"os"
"strings"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
common_http "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
httpauth "github.com/goharbor/harbor/src/common/http/modifier/auth"
"github.com/goharbor/harbor/src/common/utils"
reg "github.com/goharbor/harbor/src/common/utils/registry"
"github.com/goharbor/harbor/src/common/utils/registry/auth"
"github.com/goharbor/harbor/src/jobservice/env"
job_utils "github.com/goharbor/harbor/src/jobservice/job/impl/utils"
"github.com/goharbor/harbor/src/jobservice/logger"
)
var (
errCanceled = errors.New("the job is canceled")
)
// Transfer images from source registry to the destination one
type Transfer struct {
ctx env.JobContext
repository *repository
srcRegistry *registry
dstRegistry *registry
logger logger.Interface
retry bool
}
// ShouldRetry : retry if the error is network error
func (t *Transfer) ShouldRetry() bool {
return t.retry
}
// MaxFails ...
func (t *Transfer) MaxFails() uint {
return 3
}
// Validate ....
func (t *Transfer) Validate(params map[string]interface{}) error {
return nil
}
// Run ...
func (t *Transfer) Run(ctx env.JobContext, params map[string]interface{}) error {
err := t.run(ctx, params)
t.retry = retry(err)
return err
}
func (t *Transfer) run(ctx env.JobContext, params map[string]interface{}) error {
// initialize
if err := t.init(ctx, params); err != nil {
return err
}
// try to create project on destination registry
if err := t.createProject(); err != nil {
return err
}
// replicate the images
for _, tag := range t.repository.tags {
digest, manifest, err := t.pullManifest(tag)
if err != nil {
return err
}
if err := t.transferLayers(tag, manifest.References()); err != nil {
return err
}
if err := t.pushManifest(tag, digest, manifest); err != nil {
return err
}
}
return nil
}
func (t *Transfer) init(ctx env.JobContext, params map[string]interface{}) error {
t.logger = ctx.GetLogger()
t.ctx = ctx
if canceled(t.ctx) {
t.logger.Warning(errCanceled.Error())
return errCanceled
}
// init images that need to be replicated
t.repository = &repository{
name: params["repository"].(string),
}
if tags, ok := params["tags"]; ok {
tgs := tags.([]interface{})
for _, tg := range tgs {
t.repository.tags = append(t.repository.tags, tg.(string))
}
}
var err error
// init source registry client
srcURL := params["src_registry_url"].(string)
srcInsecure := params["src_registry_insecure"].(bool)
srcCred := httpauth.NewSecretAuthorizer(secret())
srcTokenServiceURL := ""
if stsu, ok := params["src_token_service_url"]; ok {
srcTokenServiceURL = stsu.(string)
}
if len(srcTokenServiceURL) > 0 {
t.srcRegistry, err = initRegistry(srcURL, srcInsecure, srcCred, t.repository.name, srcTokenServiceURL)
} else {
t.srcRegistry, err = initRegistry(srcURL, srcInsecure, srcCred, t.repository.name)
}
if err != nil {
t.logger.Errorf("failed to create client for source registry: %v", err)
return err
}
// init destination registry client
dstURL := params["dst_registry_url"].(string)
dstInsecure := params["dst_registry_insecure"].(bool)
dstCred := auth.NewBasicAuthCredential(
params["dst_registry_username"].(string),
params["dst_registry_password"].(string))
t.dstRegistry, err = initRegistry(dstURL, dstInsecure, dstCred, t.repository.name)
if err != nil {
t.logger.Errorf("failed to create client for destination registry: %v", err)
return err
}
// get the tag list first if it is null
if len(t.repository.tags) == 0 {
tags, err := t.srcRegistry.ListTag()
if err != nil {
t.logger.Errorf("an error occurred while listing tags for the source repository: %v", err)
return err
}
if len(tags) == 0 {
err = fmt.Errorf("empty tag list for repository %s", t.repository.name)
t.logger.Error(err)
return err
}
t.repository.tags = tags
}
t.logger.Infof("initialization completed: repository: %s, tags: %v, source registry: URL-%s insecure-%v, destination registry: URL-%s insecure-%v",
t.repository.name, t.repository.tags, t.srcRegistry.url, t.srcRegistry.insecure, t.dstRegistry.url, t.dstRegistry.insecure)
return nil
}
func initRegistry(url string, insecure bool, credential modifier.Modifier,
repository string, tokenServiceURL ...string) (*registry, error) {
registry := &registry{
url: url,
insecure: insecure,
}
// use the same transport for clients connecting to docker registry and Harbor UI
transport := reg.GetHTTPTransport(insecure)
authorizer := auth.NewStandardTokenAuthorizer(&http.Client{
Transport: transport,
}, credential, tokenServiceURL...)
uam := &job_utils.UserAgentModifier{
UserAgent: "harbor-registry-client",
}
repositoryClient, err := reg.NewRepository(repository, url,
&http.Client{
Transport: reg.NewTransport(transport, authorizer, uam),
})
if err != nil {
return nil, err
}
registry.Repository = *repositoryClient
registry.client = common_http.NewClient(
&http.Client{
Transport: transport,
}, credential)
return registry, nil
}
func (t *Transfer) createProject() error {
if canceled(t.ctx) {
t.logger.Warning(errCanceled.Error())
return errCanceled
}
p, _ := utils.ParseRepository(t.repository.name)
project, err := t.srcRegistry.GetProject(p)
if err != nil {
t.logger.Errorf("failed to get project %s from source registry: %v", p, err)
return err
}
if err = t.dstRegistry.CreateProject(project); err != nil {
// other jobs may be also doing the same thing when the current job
// is creating project or the project has already exist, so when the
// response code is 409, continue to do next step
if e, ok := err.(*common_http.Error); ok && e.Code == http.StatusConflict {
t.logger.Warningf("the status code is 409 when creating project %s on destination registry, try to do next step", p)
return nil
}
t.logger.Errorf("an error occurred while creating project %s on destination registry: %v", p, err)
return err
}
t.logger.Infof("project %s is created on destination registry", p)
return nil
}
func (t *Transfer) pullManifest(tag string) (string, distribution.Manifest, error) {
if canceled(t.ctx) {
t.logger.Warning(errCanceled.Error())
return "", nil, errCanceled
}
acceptMediaTypes := []string{schema1.MediaTypeManifest, schema2.MediaTypeManifest}
digest, mediaType, payload, err := t.srcRegistry.PullManifest(tag, acceptMediaTypes)
if err != nil {
t.logger.Errorf("an error occurred while pulling manifest of %s:%s from source registry: %v",
t.repository.name, tag, err)
return "", nil, err
}
t.logger.Infof("manifest of %s:%s pulled successfully from source registry: %s",
t.repository.name, tag, digest)
if strings.Contains(mediaType, "application/json") {
mediaType = schema1.MediaTypeManifest
}
manifest, _, err := reg.UnMarshal(mediaType, payload)
if err != nil {
t.logger.Errorf("an error occurred while parsing manifest: %v", err)
return "", nil, err
}
return digest, manifest, nil
}
func (t *Transfer) transferLayers(tag string, blobs []distribution.Descriptor) error {
repository := t.repository.name
// all blobs(layers and config)
for _, blob := range blobs {
if canceled(t.ctx) {
t.logger.Warning(errCanceled.Error())
return errCanceled
}
digest := blob.Digest.String()
if blob.MediaType == schema2.MediaTypeForeignLayer {
t.logger.Infof("blob %s of %s:%s is an foreign layer, skip", digest, repository, tag)
continue
}
exist, err := t.dstRegistry.BlobExist(digest)
if err != nil {
t.logger.Errorf("an error occurred while checking existence of blob %s of %s:%s on destination registry: %v",
digest, repository, tag, err)
return err
}
if exist {
t.logger.Infof("blob %s of %s:%s already exists on the destination registry, skip",
digest, repository, tag)
continue
}
t.logger.Infof("transferring blob %s of %s:%s to the destination registry ...",
digest, repository, tag)
size, data, err := t.srcRegistry.PullBlob(digest)
if err != nil {
t.logger.Errorf("an error occurred while pulling blob %s of %s:%s from the source registry: %v",
digest, repository, tag, err)
return err
}
if data != nil {
defer data.Close()
}
if err = t.dstRegistry.PushBlob(digest, size, data); err != nil {
t.logger.Errorf("an error occurred while pushing blob %s of %s:%s to the distination registry: %v",
digest, repository, tag, err)
return err
}
t.logger.Infof("blob %s of %s:%s transferred to the destination registry completed",
digest, repository, tag)
}
return nil
}
func (t *Transfer) pushManifest(tag, digest string, manifest distribution.Manifest) error {
if canceled(t.ctx) {
t.logger.Warning(errCanceled.Error())
return errCanceled
}
repository := t.repository.name
dgt, exist, err := t.dstRegistry.ManifestExist(tag)
if err != nil {
t.logger.Warningf("an error occurred while checking the existence of manifest of %s:%s on the destination registry: %v, try to push manifest",
repository, tag, err)
} else {
if exist && dgt == digest {
t.logger.Infof("manifest of %s:%s exists on the destination registry, skip manifest pushing",
repository, tag)
return nil
}
}
mediaType, data, err := manifest.Payload()
if err != nil {
t.logger.Errorf("an error occurred while getting payload of manifest for %s:%s : %v",
repository, tag, err)
return err
}
if _, err = t.dstRegistry.PushManifest(tag, mediaType, data); err != nil {
t.logger.Errorf("an error occurred while pushing manifest of %s:%s to the destination registry: %v",
repository, tag, err)
return err
}
t.logger.Infof("manifest of %s:%s has been pushed to the destination registry",
repository, tag)
return nil
}
func canceled(ctx env.JobContext) bool {
_, canceled := ctx.OPCommand()
return canceled
}
func retry(err error) bool {
if err == nil {
return false
}
_, ok := err.(net.Error)
return ok
}
func secret() string {
return os.Getenv("JOBSERVICE_SECRET")
}

11
src/jobservice/runtime/bootstrap.go
View File

@ -205,12 +205,11 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(ctx *env.Context, cfg *config.Con
}
if err := redisWorkerPool.RegisterJobs(
map[string]interface{}{
job.ImageScanJob: (*scan.ClairJob)(nil),
job.ImageScanAllJob: (*scan.All)(nil),
job.ImageTransfer: (*replication.Transfer)(nil),
job.ImageDelete: (*replication.Deleter)(nil),
job.ImageReplicate: (*replication.Replicator)(nil),
job.ImageGC: (*gc.GarbageCollector)(nil),
job.ImageScanJob: (*scan.ClairJob)(nil),
job.ImageScanAllJob: (*scan.All)(nil),
job.ImageGC: (*gc.GarbageCollector)(nil),
job.Replication: (*replication.Replication)(nil),
job.ReplicationScheduler: (*replication.Scheduler)(nil),
}); err != nil {
// exit
return nil, err

8
src/portal/lib/README.md
View File

@ -258,9 +258,9 @@ export const DefaultServiceConfig: IServiceConfig = {
systemInfoEndpoint: "/api/systeminfo",
repositoryBaseEndpoint: "/api/repositories",
logBaseEndpoint: "/api/logs",
targetBaseEndpoint: "/api/targets",
targetBaseEndpoint: "/api/registries",
replicationRuleEndpoint: "/api/policies/replication",
replicationJobEndpoint: "/api/jobs/replication",
replicationBaseEndpoint: "/api/replication/executions",
vulnerabilityScanningBaseEndpoint: "/api/repositories",
configurationEndpoint: "/api/configurations",
enablei18Support: false,
@ -293,11 +293,11 @@ It supports partially overriding. For the items not overridden, default values w
* **logBaseEndpoint:** The base endpoint of the service used to handle the recent access logs. Default is "/api/logs".
* **targetBaseEndpoint:** The base endpoint of the service used to handle the registry endpoints. Default is "/api/targets".
* **targetBaseEndpoint:** The base endpoint of the service used to handle the registry endpoints. Default is "/api/registries".
* **replicationRuleEndpoint:** The base endpoint of the service used to handle the replication rules. Default is "/api/policies/replication".
* **replicationJobEndpoint:** The base endpoint of the service used to handle the replication jobs. Default is "/api/jobs/replication".
* **replicationBaseEndpoint:** The base endpoint of the service used to handle the replication executions. Default is "/api/replication/executions".
* **vulnerabilityScanningBaseEndpoint:** The base endpoint of the service used to handle the vulnerability scanning results.Default value is "/api/repositories".

4
src/portal/lib/src/confirmation-dialog/confirmation-dialog.component.html
View File

@ -26,5 +26,9 @@
<button type="button" class="btn btn-outline" (click)="cancel()" [hidden]="isDelete">{{'BUTTON.CANCEL' | translate}}</button>
<button type="button" class="btn btn-primary" (click)="confirm()" [hidden]="isDelete">{{'BUTTON.REPLICATE' | translate}}</button>
</ng-template>
<ng-template [ngSwitchCase]="5">
<button type="button" class="btn btn-outline" (click)="cancel()" [hidden]="isDelete">{{'BUTTON.CANCEL' | translate}}</button>
<button type="button" class="btn btn-primary" (click)="confirm()" [hidden]="isDelete">{{'BUTTON.STOP' | translate}}</button>
</ng-template>
</div>
</clr-modal>

35
src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.html
View File

@ -11,6 +11,18 @@
</div>
<form #targetForm="ngForm">
<section class="form-block">
<!-- provider -->
<div class="form-group">
<label class="form-group-label-override required">{{'DESTINATION.PROVIDER' | translate}}</label>
<div class="form-select">
<div class="select providerSelect pull-left">
<select name="adapter" id="adapter" [(ngModel)]="target.type" [disabled]="testOngoing || controlEnabled">
<option *ngFor="let adapter of adapterList" value="{{adapter}}">{{adapter}}</option>
</select>
</div>
</div>
</div>
<!-- Endpoint name -->
<div class="form-group">
<label for="destination_name" class="col-md-4 form-group-label-override required">{{ 'DESTINATION.NAME' |
translate }}</label>
@ -23,30 +35,39 @@
</span>
</label>
</div>
<!--Description-->
<div class="form-group">
<label class="form-group-label-override">{{'REPLICATION.DESCRIPTION' | translate}}</label>
<textarea type="text" class="inputWidth" row=3 name="description" [(ngModel)]="target.description"></textarea>
</div>
<!-- Endpoint Url -->
<div class="form-group">
<label for="destination_url" class="col-md-4 form-group-label-override required">{{ 'DESTINATION.URL' |
translate }}</label>
<label class="col-md-8" for="destination_url" aria-haspopup="true" role="tooltip" [class.invalid]="targetEndpoint.errors && (targetEndpoint.dirty || targetEndpoint.touched)"
[class.valid]="targetEndpoint.valid" class="tooltip tooltip-validation tooltip-sm tooltip-bottom-left">
<input type="text" id="destination_url" [disabled]="testOngoing" [readonly]="!editable" [(ngModel)]="target.endpoint"
<input type="text" id="destination_url" [disabled]="testOngoing || controlEnabled" [readonly]="!editable" [(ngModel)]="target.url"
size="20" name="endpointUrl" #targetEndpoint="ngModel" required placeholder="http(s)://192.168.1.1">
<span class="tooltip-content" *ngIf="targetEndpoint.errors && targetEndpoint.errors.required && (targetEndpoint.dirty || targetEndpoint.touched)">
{{ 'DESTINATION.URL_IS_REQUIRED' | translate }}
</span>
</label>
</div>
<!-- access_key -->
<div class="form-group">
<label for="destination_username" class="col-md-4 form-group-label-override">{{ 'DESTINATION.USERNAME' |
<label for="destination_access_key" class="col-md-4 form-group-label-override">{{ 'DESTINATION.ACCESS_ID' |
translate }}</label>
<input type="text" class="col-md-8" id="destination_username" [disabled]="testOngoing" [readonly]="!editable"
[(ngModel)]="target.username" size="20" name="username" #username="ngModel">
<input type="text" placeholder="Access ID" class="col-md-8" id="destination_access_key" [disabled]="testOngoing" [readonly]="!editable"
[(ngModel)]="target.credential.access_key" size="23" name="access_key" #access_key="ngModel">
</div>
<!-- access_secret -->
<div class="form-group">
<label for="destination_password" class="col-md-4 form-group-label-override">{{ 'DESTINATION.PASSWORD' |
<label for="destination_password" class="col-md-4 form-group-label-override">{{ 'DESTINATION.ACCESS_SECRET' |
translate }}</label>
<input type="password" class="col-md-8" id="destination_password" [disabled]="testOngoing" [readonly]="!editable"
[(ngModel)]="target.password" size="20" name="password" #password="ngModel">
<input type="password" placeholder="Access Secret" class="col-md-8" id="destination_password" [disabled]="testOngoing" [readonly]="!editable"
[(ngModel)]="target.credential.access_secret" size="23" name="access_secret" #access_secret="ngModel">
</div>
<!-- Verify Remote Cert -->
<div class="form-group">
<label for="destination_insecure" id="destination_insecure_checkbox">{{'CONFIG.VERIFY_REMOTE_CERT' |
translate }}</label>

10
src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.scss
View File

@ -10,4 +10,12 @@
.form-height {
height: 30px;
}
}
.providerSelect {
width: 180px;
}
.inputWidth {
width: 182px;
}

21
src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.spec.ts
View File

@ -22,14 +22,20 @@ import { of } from "rxjs";
describe("CreateEditEndpointComponent (inline template)", () => {
let mockData: Endpoint = {
id: 1,
endpoint: "https://10.117.4.151",
name: "target_01",
username: "admin",
password: "",
credential: {
access_key: "admin",
access_secret: "",
type: "basic"
},
description: "test",
insecure: false,
type: 0
name: "target_01",
type: "Harbor",
url: "https://10.117.4.151"
};
let mockAdapters = ['harbor', 'docker hub'];
let comp: CreateEditEndpointComponent;
let fixture: ComponentFixture<CreateEditEndpointComponent>;
@ -40,6 +46,7 @@ describe("CreateEditEndpointComponent (inline template)", () => {
let endpointService: EndpointService;
let spy: jasmine.Spy;
let spyAdapter: jasmine.Spy;
beforeEach(async(() => {
TestBed.configureTestingModule({
@ -62,6 +69,10 @@ describe("CreateEditEndpointComponent (inline template)", () => {
comp = fixture.componentInstance;
endpointService = fixture.debugElement.injector.get(EndpointService);
spyAdapter = spyOn(endpointService, "getAdapters").and.returnValue(
of(mockAdapters)
);
spy = spyOn(endpointService, "getEndpoint").and.returnValue(
of(mockData)
);

147
src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts
View File

@ -18,7 +18,8 @@ import {
ViewChild,
AfterViewChecked,
ChangeDetectorRef,
OnDestroy
OnDestroy,
OnInit
} from "@angular/core";
import { NgForm } from "@angular/forms";
import { Subscription } from "rxjs";
@ -30,7 +31,6 @@ import { InlineAlertComponent } from "../inline-alert/inline-alert.component";
import { Endpoint } from "../service/interface";
import { clone, compareValue, isEmptyObject } from "../utils";
const FAKE_PASSWORD = "rjGcfuRu";
@Component({
@ -39,16 +39,17 @@ const FAKE_PASSWORD = "rjGcfuRu";
styleUrls: ["./create-edit-endpoint.component.scss"]
})
export class CreateEditEndpointComponent
implements AfterViewChecked, OnDestroy {
implements AfterViewChecked, OnDestroy, OnInit {
modalTitle: string;
controlEnabled: boolean = false;
createEditDestinationOpened: boolean;
staticBackdrop: boolean = true;
closable: boolean = false;
editable: boolean;
adapterList: string[];
target: Endpoint = this.initEndpoint();
selectedType: string;
initVal: Endpoint;
targetForm: NgForm;
@ViewChild("targetForm") currentForm: NgForm;
@ -71,6 +72,17 @@ export class CreateEditEndpointComponent
private ref: ChangeDetectorRef
) {}
ngOnInit(): void {
this.endpointService.getAdapters().subscribe(
adapters => {
this.adapterList = adapters || [];
},
error => {
this.errorHandler.error(error);
}
);
}
public get isValid(): boolean {
return (
!this.testOngoing &&
@ -98,12 +110,16 @@ export class CreateEditEndpointComponent
initEndpoint(): Endpoint {
return {
endpoint: "",
name: "",
username: "",
password: "",
credential: {
access_key: "",
access_secret: "",
type: "basic"
},
description: "",
insecure: false,
type: 0
name: "",
type: "harbor",
url: ""
};
}
@ -125,7 +141,6 @@ export class CreateEditEndpointComponent
this.initVal = this.initEndpoint();
this.formValues = null;
this.endpointId = "";
this.inlineAlert.close();
}
@ -153,18 +168,21 @@ export class CreateEditEndpointComponent
this.translateService
.get("DESTINATION.TITLE_EDIT")
.subscribe(res => (this.modalTitle = res));
this.endpointService.getEndpoint(targetId)
.subscribe(target => {
this.endpointService.getEndpoint(targetId).subscribe(
target => {
this.target = target;
// Keep data cache
this.initVal = clone(target);
this.initVal.password = FAKE_PASSWORD;
this.target.password = FAKE_PASSWORD;
this.initVal.credential.access_secret = FAKE_PASSWORD;
this.target.credential.access_secret = FAKE_PASSWORD;
// Open the modal now
this.open();
this.controlEnabled = true;
this.forceRefreshView(2000);
}, error => this.errorHandler.error(error));
},
error => this.errorHandler.error(error)
);
} else {
this.endpointId = "";
this.translateService
@ -172,15 +190,16 @@ export class CreateEditEndpointComponent
.subscribe(res => (this.modalTitle = res));
// Directly open the modal
this.open();
this.controlEnabled = false;
}
}
testConnection() {
let payload: Endpoint = this.initEndpoint();
if (!this.endpointId) {
payload.endpoint = this.target.endpoint;
payload.username = this.target.username;
payload.password = this.target.password;
payload.url = this.target.url;
payload.credential.access_key = this.target.credential.access_key;
payload.credential.access_secret = this.target.credential.access_secret;
payload.insecure = this.target.insecure;
} else {
let changes: { [key: string]: any } = this.getChanges();
@ -197,18 +216,20 @@ export class CreateEditEndpointComponent
}
this.testOngoing = true;
this.endpointService.pingEndpoint(payload)
.subscribe(response => {
this.endpointService.pingEndpoint(payload).subscribe(
response => {
this.inlineAlert.showInlineSuccess({
message: "DESTINATION.TEST_CONNECTION_SUCCESS"
});
this.forceRefreshView(2000);
this.testOngoing = false;
}, error => {
},
error => {
this.inlineAlert.showInlineError("DESTINATION.TEST_CONNECTION_FAILURE");
this.forceRefreshView(2000);
this.testOngoing = false;
});
}
);
}
onSubmit() {
@ -223,10 +244,9 @@ export class CreateEditEndpointComponent
if (this.onGoing) {
return; // Avoid duplicated submitting
}
this.onGoing = true;
this.endpointService.createEndpoint(this.target)
.subscribe(response => {
this.endpointService.createEndpoint(this.target).subscribe(
response => {
this.translateService
.get("DESTINATION.CREATED_SUCCESS")
.subscribe(res => this.errorHandler.info(res));
@ -234,14 +254,13 @@ export class CreateEditEndpointComponent
this.onGoing = false;
this.close();
this.forceRefreshView(2000);
}, error => {
},
error => {
this.onGoing = false;
let errorMessageKey = this.handleErrorMessageKey(error.status);
this.translateService.get(errorMessageKey).subscribe(res => {
this.inlineAlert.showInlineError(res);
});
this.inlineAlert.showInlineError(error);
this.forceRefreshView(2000);
});
}
);
}
updateEndpoint() {
@ -257,6 +276,7 @@ export class CreateEditEndpointComponent
if (isEmptyObject(changes)) {
return;
}
let changekeys: { [key: string]: any } = Object.keys(changes);
changekeys.forEach((key: string) => {
@ -268,8 +288,8 @@ export class CreateEditEndpointComponent
}
this.onGoing = true;
this.endpointService.updateEndpoint(this.target.id, payload)
.subscribe(response => {
this.endpointService.updateEndpoint(this.target.id, payload).subscribe(
response => {
this.translateService
.get("DESTINATION.UPDATED_SUCCESS")
.subscribe(res => this.errorHandler.info(res));
@ -277,25 +297,13 @@ export class CreateEditEndpointComponent
this.close();
this.onGoing = false;
this.forceRefreshView(2000);
}, error => {
let errorMessageKey = this.handleErrorMessageKey(error.status);
this.translateService.get(errorMessageKey).subscribe(res => {
this.inlineAlert.showInlineError(res);
});
},
error => {
this.inlineAlert.showInlineError(error);
this.onGoing = false;
this.forceRefreshView(2000);
});
}
handleErrorMessageKey(status: number): string {
switch (status) {
case 409:
return "DESTINATION.CONFLICT_NAME";
case 400:
return "DESTINATION.INVALID_NAME";
default:
return "UNKNOWN_ERROR";
}
}
);
}
onCancel() {
@ -338,7 +346,6 @@ export class CreateEditEndpointComponent
if (!compareValue(this.formValues, data)) {
this.formValues = data;
this.inlineAlert.close();
}
}
}
@ -353,20 +360,36 @@ export class CreateEditEndpointComponent
}
for (let prop of Object.keys(this.target)) {
let field: any = this.initVal[prop];
if (!compareValue(field, this.target[prop])) {
changes[prop] = this.target[prop];
// Number
if (typeof field === "number") {
changes[prop] = +changes[prop];
}
if (typeof field !== "object") {
if (!compareValue(field, this.target[prop])) {
changes[prop] = this.target[prop];
// Number
if (typeof field === "number") {
changes[prop] = +changes[prop];
}
// Trim string value
if (typeof field === "string") {
changes[prop] = ("" + changes[prop]).trim();
// Trim string value
if (typeof field === "string") {
changes[prop] = ("" + changes[prop]).trim();
}
}
} else {
for (let pro of Object.keys(field)) {
if (!compareValue(field[pro], this.target[prop][pro])) {
changes[pro] = this.target[prop][pro];
// Number
if (typeof field[pro] === "number") {
changes[pro] = +changes[pro];
}
// Trim string value
if (typeof field[pro] === "string") {
changes[pro] = ("" + changes[pro]).trim();
}
}
}
}
}
return changes;
}
}

212
src/portal/lib/src/create-edit-rule/create-edit-rule.component.html
View File

@ -2,23 +2,13 @@
<h3 class="modal-title">{{headerTitle | translate}}</h3>
<hbr-inline-alert class="modal-title" (confirmEvt)="confirmCancel($event)"></hbr-inline-alert>
<div class="modal-body modal-body-height">
<clr-alert [hidden]='!deletedLabelCount' [clrAlertType]="'alert-warning'" [clrAlertSizeSmall]="true"
[clrAlertClosable]="false" [(clrAlertClosed)]="alertClosed">
<div class="alert-item">
<span class="alert-text">{{deletedLabelInfo}}</span>
<div class="alert-actions">
<a class="alert-action" (click)=" alertClosed = true">{{'REPLICATION.ACKNOWLEDGE' | translate}}</a>
</div>
</div>
</clr-alert>
<form [formGroup]="ruleForm" novalidate>
<section class="form-block">
<div class="form-group form-group-override">
<label class="form-group-label-override required">{{'REPLICATION.NAME' | translate}}</label>
<label aria-haspopup="true" role="tooltip" class="tooltip tooltip-validation tooltip-md tooltip-bottom-left"
[class.invalid]='(ruleForm.controls.name.touched && ruleForm.controls.name.invalid) || !isRuleNameValid'>
<input type="text" id="ruleName" pattern="^[a-z0-9]+(?:[._-][a-z0-9]+)*$" class="inputWidth" required
maxlength="255" formControlName="name" #ruleName (keyup)='checkRuleName()' autocomplete="off">
<label aria-haspopup="true" role="tooltip" class="tooltip tooltip-validation tooltip-md tooltip-bottom-left" [class.invalid]='(ruleForm.controls.name.touched && ruleForm.controls.name.invalid) || !isRuleNameValid'>
<input type="text" id="ruleName" pattern="^[a-z0-9]+(?:[._-][a-z0-9]+)*$" class="inputWidth" required maxlength="255" formControlName="name"
#ruleName (keyup)='checkRuleName()' autocomplete="off">
<span class="tooltip-content">{{ruleNameTooltip | translate}}</span>
</label>
<span class="spinner spinner-inline spinner-pos" [hidden]="!inNameChecking"></span>
@ -28,128 +18,120 @@
<label class="form-group-label-override">{{'REPLICATION.DESCRIPTION' | translate}}</label>
<textarea type="text" id="ruleDescription" class="inputWidth" row=3 formControlName="description"></textarea>
</div>
<!--Projects-->
<!-- replication mode -->
<div class="form-group form-group-override">
<label class="form-group-label-override required">{{'REPLICATION.SOURCE_PROJECT' | translate}}</label>
<div formArrayName="projects">
<div class="projectInput inputWidth" *ngFor="let project of projects.controls; let i= index"
[formGroupName]="i" (mouseleave)="leaveInput()">
<label aria-haspopup="true" role="tooltip" class="tooltip tooltip-validation tooltip-md tooltip-bottom-left"
[class.invalid]='noProjectInfo'>
<input *ngIf="!projectId" formControlName="name" type="text" class="inputWidth" value="name" required
pattern="^[a-z0-9]+(?:[._-][a-z0-9]+)*$" (keyup)='handleValidation()' (focus)="focusClear($event)"
autocomplete="off">
<input *ngIf="projectId" formControlName="name" type="text" class="inputWidth" value="name" readonly>
<span class="tooltip-content">{{noProjectInfo | translate}}</span>
</label>
<div class="selectBox inputWidth" [style.display]="selectedProjectList.length ? 'block' : 'none'">
<ul>
<li *ngFor="let project of selectedProjectList" (click)="selectedProjectName(project?.name)">{{project?.name}}</li>
</ul>
</div>
</div>
<label class="form-group-label-override">{{'REPLICATION.REPLI_MODE' | translate}}</label>
<div class="radio-inline" [class.disabled]="policyId >= 0">
<input type="radio" id="push_base" name="replicationMode" [value]=true [disabled]="policyId >= 0" [(ngModel)]="isPushMode" (change)="modeChange()" [ngModelOptions]="{standalone: true}">
<label for="push_base">Push-based</label>
</div>
<div class="radio-inline" [class.disabled]="policyId >= 0">
<input type="radio" id="pull_base" name="replicationMode" [value]=false [disabled]="policyId >= 0" [(ngModel)]="isPushMode" [ngModelOptions]="{standalone: true}">
<label for="pull_base">Pull-based</label>
</div>
</div>
<!--images/Filter-->
<div class="form-group form-group-override">
<label class="form-group-label-override">{{'REPLICATION.SOURCE_IMAGES_FILTER' | translate}}</label>
<div formArrayName="filters">
<div class="filterSelect" *ngFor="let filter of filters.controls; let i=index">
<div [formGroupName]="i">
<div class="select floatSetPar">
<select formControlName="kind" #selectedValue (change)="filterChange($event, selectedValue.value)" id="{{i}}"
name="{{filterListData[i]?.name}}">
<option *ngFor="let opt of filterListData[i]?.options;" value="{{opt}}">{{opt}}</option>
</select>
</div>
<label aria-haspopup="true" role="tooltip" class="tooltip tooltip-validation tooltip-md tooltip-bottom-left"
[class.invalid]='(filter.value.dirty || filter.value.touched) && filter.value.invalid'>
<input type="text" #filterValue required size="14" formControlName="value" [attr.disabled]="(filterListData[i]?.name=='label') ?'' : null">
<span class="tooltip-content">{{'TOOLTIP.EMPTY' | translate}}</span>
</label>
<div class="arrowSet" [hidden]="!(filterListData[i]?.name=='label')" (click)="openLabelList(selectedValue.value, i, $event)">
<clr-icon shape="angle" ></clr-icon>
</div>
<clr-icon shape="warning-standard" class="is-solid is-warning warning-icon" size="14" [hidden]="!deletedLabelCount || !(filterListData[i]?.name=='label')"></clr-icon>
<clr-icon shape="times-circle" class="is-solid" (click)="deleteFilter(i)"></clr-icon>
<div *ngIf="!withAdmiral">
<hbr-filter-label [projectId]="projectId" [selectedLabelInfo]="filterLabelInfo" [isOpen]="filterListData[i].isOpen"
(selectedLabels)="selectedLabelList($event, i)" (closePanelEvent)="filterListData[i].isOpen = false"></hbr-filter-label>
</div>
</div>
</div>
</div>
<clr-icon id="add-label-list" shape="plus-circle" class="is-solid plus-position" [hidden]="isFilterHide" (click)="addNewFilter()"></clr-icon>
</div>
<!--Targets-->
<div class="form-group form-group-override">
<label class="form-group-label-override required">{{'DESTINATION.ENDPOINT' | translate}}</label>
<div formArrayName="targets" class="form-select">
<div class="select endpointSelect pull-left" *ngFor="let target of targets.controls; let i= index"
[formGroupName]="i">
<select id="ruleTarget" (change)="targetChange($event)" formControlName="id">
<option *ngFor="let target of targetList" value="{{target.id}}">{{target.name}}-{{target.endpoint}}</option>
<!--source registry-->
<div *ngIf="!isPushMode" class="form-group form-group-override">
<label class="form-group-label-override required">{{'REPLICATION.SOURCE_REGISTRY' | translate}}</label>
<div class="form-select">
<div class="select endpointSelect pull-left">
<select id="src_registry_id" (change)="sourceChange($event)" formControlName="src_registry" [compareWith]="equals">
<option *ngFor="let source of sourceList" [ngValue]="source">{{source.name}}-{{source.url}}</option>
</select>
</div>
</div>
<label *ngIf="noEndpointInfo.length != 0" class="colorRed alertLabel">{{noEndpointInfo | translate}}</label>
<span class="alertLabel goLink" *ngIf="noEndpointInfo.length != 0" (click)="goRegistry()">{{'REPLICATION.ENDPOINTS'
| translate}}</span>
<span class="alertLabel goLink" *ngIf="noEndpointInfo.length != 0" (click)="goRegistry()">{{'REPLICATION.ENDPOINTS' | translate}}</span>
</div>
<!--images/Filter-->
<div class="form-group form-group-override">
<label class="form-group-label-override">{{'REPLICATION.SOURCE_RESOURCE_FILTER' | translate}}</label>
<div formArrayName="filters">
<div class="filterSelect" *ngFor="let filter of filters.controls; let i=index">
<div [formGroupName]="i">
<div class="width-70">
<label>{{"REPLICATION." + supportedFilters[i]?.type.toUpperCase() | translate}}:</label>
</div>
<label *ngIf="supportedFilters[i]?.style==='input'" aria-haspopup="true" role="tooltip" class="tooltip tooltip-validation tooltip-md tooltip-bottom-left"
[class.invalid]='(filter.value.dirty || filter.value.touched) && filter.value.invalid'>
<input type="text" #filterValue required size="14" formControlName="value">
<span class="tooltip-content">{{'TOOLTIP.EMPTY' | translate}}</span>
</label>
<div class="select inline-block" *ngIf="supportedFilters[i]?.style==='radio'">
<select formControlName="value" #selectedValue id="{{i}}" name="{{supportedFilters[i]?.type}}">
<option value=""></option>
<option *ngFor="let value of supportedFilters[i]?.values;" value="{{value}}">{{value}}</option>
</select>
</div>
</div>
</div>
</div>
</div>
<!--destination registry-->
<div *ngIf="isPushMode" class="form-group form-group-override">
<label class="form-group-label-override required">{{'REPLICATION.DEST_REGISTRY' | translate}}</label>
<div class="form-select">
<div class="select endpointSelect pull-left">
<select id="dest_registry" (change)="targetChange($event)" formControlName="dest_registry" [compareWith]="equals">
<option *ngFor="let target of targetList" [ngValue]="target">{{target.name}}-{{target.url}}</option>
</select>
</div>
</div>
<label *ngIf="noEndpointInfo.length != 0" class="colorRed alertLabel">{{noEndpointInfo | translate}}</label>
<span class="alertLabel goLink" *ngIf="noEndpointInfo.length != 0" (click)="goRegistry()">{{'REPLICATION.ENDPOINTS' | translate}}</span>
</div>
<!--destination namespaces -->
<div class="form-group form-group-override">
<label class="form-group-label-override">{{'REPLICATION.DEST_NAMESPACE' | translate}}</label>
<div class="form-select">
<div class="endpointSelect pull-left">
<input formControlName="dest_namespace" type="text" id="dest_namespace" pattern="^[a-z0-9]+(?:[._-][a-z0-9]+)*$" class="inputWidth"
maxlength="255">
</div>
</div>
</div>
<!--Trigger-->
<div class="form-group form-group-override">
<label class="form-group-label-override">{{'REPLICATION.TRIGGER_MODE' | translate}}</label>
<label class="form-group-label-override required">{{'REPLICATION.TRIGGER_MODE' | translate}}</label>
<div formGroupName="trigger">
<!--on trigger-->
<div class="select floatSetPar">
<select id="ruleTrigger" formControlName="kind" (change)="selectTrigger($event)">
<option value="Manual">{{'REPLICATION.MANUAL' | translate}}</option>
<option value="Immediate">{{'REPLICATION.IMMEDIATE' | translate}}</option>
<option value="Scheduled">{{'REPLICATION.SCHEDULE' | translate}}</option>
<div class="select width-115">
<select id="ruleTrigger" formControlName="type">
<option *ngFor="let trigger of supportedTriggers" [value]="trigger">{{'REPLICATION.' + trigger.toUpperCase() | translate }}</option>
</select>
</div>
<!--on push-->
<div formGroupName="schedule_param" class="schedule-style">
<div class="select floatSet" [hidden]="!isScheduleOpt">
<select name="scheduleType" formControlName="type" (change)="selectSchedule($event)">
<option value="Daily">{{'REPLICATION.DAILY' | translate}}</option>
<option value="Weekly">{{'REPLICATION.WEEKLY' | translate}}</option>
</select>
<div formGroupName="trigger_settings">
<div [hidden]="isNotSchedule()" class="form-group form-cron">
<label class="required">Cron String</label>
<label for="targetCron" aria-haspopup="true" role="tooltip"class="tooltip tooltip-validation tooltip-md tooltip-top-right"
[class.invalid]="!isNotSchedule() && cronTouched || !cronInputValid(ruleForm.value.trigger?.trigger_settings?.cron || '')" >
<input type="text" name=targetCron id="targetCron" required class="form-control cron-input" formControlName="cron">
<span class="tooltip-content">
{{'TOOLTIP.CRON_REQUIRED' | translate }}
</span>
</label>
</div>
<!--weekly-->
<span [hidden]="!weeklySchedule || !isScheduleOpt">on &nbsp;</span>
<div [hidden]="!weeklySchedule || !isScheduleOpt" class="select floatSet weekday-width">
<select name="scheduleDay" formControlName="weekday">
<option value="1">{{'WEEKLY.MONDAY' | translate}}</option>
<option value="2">{{'WEEKLY.TUESDAY' | translate}}</option>
<option value="3">{{'WEEKLY.WEDNESDAY' | translate}}</option>
<option value="4">{{'WEEKLY.THURSDAY' | translate}}</option>
<option value="5">{{'WEEKLY.FRIDAY' | translate}}</option>
<option value="6">{{'WEEKLY.SATURDAY' | translate}}</option>
<option value="7">{{'WEEKLY.SUNDAY' | translate}}</option>
</select>
</div>
<!--daily/time-->
<span [hidden]="!isScheduleOpt">at &nbsp;</span>
<input [hidden]="!isScheduleOpt" type="time" formControlName="offtime" required value="08:00" />
</div>
</div>
<div [hidden]="!isImmediate" class="clr-form-control rule-width">
<div [hidden]="isNotEventBased()" class="clr-form-control rule-width">
<clr-checkbox-wrapper>
<input type="checkbox" clrCheckbox [checked]="false" id="ruleDeletion" formControlName="replicate_deletion" class="clr-checkbox">
<input type="checkbox" clrCheckbox [checked]="false" id="ruleDeletion" formControlName="deletion" class="clr-checkbox">
<label for="ruleDeletion" class="clr-control-label">{{'REPLICATION.DELETE_REMOTE_IMAGES' | translate}}</label>
</clr-checkbox-wrapper>
</div>
<div class="clr-form-control rule-width">
<clr-checkbox-wrapper>
<input type="checkbox" clrCheckbox [checked]="true" id="ruleExit" formControlName="replicate_existing_image_now"
class="clr-checkbox">
<label for="ruleExit" class="clr-control-label">{{'REPLICATION.REPLICATE_IMMEDIATE' | translate}}</label>
</clr-checkbox-wrapper>
<div class="rule-width">
<clr-checkbox-wrapper>
<input type="checkbox" clrCheckbox [checked]="true" id="enablePolicy" formControlName="enabled" class="clr-checkbox">
<label for="enablePolicy" class="clr-control-label">{{'REPLICATION.ENABLED' | translate}}</label>
</clr-checkbox-wrapper>
</div>
<div class="rule-width">
<clr-checkbox-wrapper>
<input type="checkbox" clrCheckbox [checked]="true" id="overridePolicy" formControlName="override" class="clr-checkbox">
<label for="overridePolicy" class="clr-control-label">{{'REPLICATION.OVERRIDE' | translate}}</label>
</clr-checkbox-wrapper>
</div>
</div>
<div class="loading-center">
@ -159,9 +141,7 @@
</form>
</div>
<div class="modal-footer">
<button type="button" id="ruleBtnCancel" class="btn btn-outline" [disabled]="this.inProgress" (click)="onCancel()">{{
'BUTTON.CANCEL' | translate }}</button>
<button type="submit" id="ruleBtnOk" class="btn btn-primary" (click)="onSubmit()" [disabled]="!ruleForm.valid || !isValid || !hasFormChange()">{{
'BUTTON.SAVE' | translate }}</button>
<button type="button" id="ruleBtnCancel" class="btn btn-outline" [disabled]="this.inProgress" (click)="onCancel()">{{ 'BUTTON.CANCEL' | translate }}</button>
<button type="submit" id="ruleBtnOk" class="btn btn-primary" (click)="onSubmit()" [disabled]="!isValid || !hasFormChange()">{{ 'BUTTON.SAVE' | translate }}</button>
</div>
</clr-modal>

55
src/portal/lib/src/create-edit-rule/create-edit-rule.component.scss
View File

@ -1,5 +1,5 @@
.select {
width: 186px;
width: 190px;
}
.select .optionMore {
@ -40,7 +40,7 @@ h4 {
.endpointSelect {
width: 270px;
margin-right: 20px;
margin-right: 10px;
}
.filterSelect {
@ -49,11 +49,11 @@ h4 {
}
.filterSelect clr-icon {
margin-left: 15px;
margin-left: 10px;
}
.filterSelect label {
width: 136px;
width: 190px;
}
.filterSelect label input {
@ -68,9 +68,15 @@ h4 {
padding-left: 0;
}
.floatSetPar {
.width-70 {
display: inline-block;
width: 120px;
width: 70px;
margin-right: 10px;
}
.width-115 {
display: inline-block;
width: 115px;
margin-right: 10px;
}
@ -146,12 +152,19 @@ h4 {
}
.form-group-override {
padding-left: 170px !important;
padding-left: 200px;
}
.form-group>label:first-child {
font-size: 14px;
width: 6.5rem;
.form-group {
>label:first-child{
font-size: 14px;
width: 7rem;
}
.radio {
label {
margin-right:10px;
}
}
}
.form-select {
@ -199,7 +212,7 @@ clr-modal {
margin-left: -15px;
}
.plus-position {
.mr-t-11{
margin-top: 11px;
}
@ -214,4 +227,22 @@ clr-modal {
.loading-center {
display: block;
text-align: center;
}
}
.width-315 {
display: flex;
align-items: center;
width:315px;
}
.mr-t-10 {
margin-top: 10px;
}
.inline-block {
display: inline-block;
}
.form-cron {
padding-left:3.8rem;
}

230
src/portal/lib/src/create-edit-rule/create-edit-rule.component.spec.ts
View File

@ -32,14 +32,12 @@ import {
EndpointService,
EndpointDefaultService
} from "../service/endpoint.service";
import {
ProjectDefaultService,
ProjectService
} from "../service/project.service";
import { OperationService } from "../operation/operation.service";
import {FilterLabelComponent} from "./filter-label.component";
import {LabelService} from "../service/label.service";
import {LabelPieceComponent} from "../label-piece/label-piece.component";
import { RouterTestingModule } from '@angular/router/testing';
import { of } from "rxjs";
describe("CreateEditRuleComponent (inline template)", () => {
@ -48,74 +46,51 @@ describe("CreateEditRuleComponent (inline template)", () => {
id: 1,
name: "sync_01",
description: "",
projects: [
{
project_id: 1,
owner_id: 0,
name: "project_01",
creation_time: "",
deleted: 0,
owner_name: "",
togglable: false,
update_time: "",
current_user_role_id: 0,
repo_count: 0,
has_project_admin_role: false,
is_member: false,
role_name: "",
metadata: {
public: "",
enable_content_trust: "",
prevent_vul: "",
severity: "",
auto_scan: ""
}
}
],
targets: [
{
id: 1,
endpoint: "https://10.117.4.151",
name: "target_01",
username: "admin",
password: "",
insecure: false,
type: 0
}
],
src_registry: {id: 2},
src_namespaces: ["name1", "name2"],
trigger: {
kind: "Manual",
schedule_param: null
type: "Manual",
trigger_settings: {}
},
filters: [],
replicate_existing_image_now: false,
replicate_deletion: false
deletion: false,
enabled: true,
override: true
}
];
let mockJobs: ReplicationJobItem[] = [
{
id: 1,
status: "stopped",
repository: "library/busybox",
policy_id: 1,
operation: "transfer",
tags: null
trigger: "Manual",
total: 0,
failed: 0,
succeed: 0,
in_progress: 0,
stopped: 0
},
{
id: 2,
status: "stopped",
repository: "library/busybox",
policy_id: 1,
operation: "transfer",
tags: null
trigger: "Manual",
total: 1,
failed: 0,
succeed: 1,
in_progress: 0,
stopped: 0
},
{
id: 3,
status: "stopped",
repository: "library/busybox",
policy_id: 2,
operation: "transfer",
tags: null
trigger: "Manual",
total: 1,
failed: 1,
succeed: 0,
in_progress: 0,
stopped: 0
}
];
@ -127,39 +102,55 @@ describe("CreateEditRuleComponent (inline template)", () => {
let mockEndpoints: Endpoint[] = [
{
id: 1,
endpoint: "https://10.117.4.151",
name: "target_01",
username: "admin",
password: "",
credential: {
access_key: "admin",
access_secret: "",
type: "basic"
},
description: "test",
insecure: false,
type: 0
name: "target_01",
type: "Harbor",
url: "https://10.117.4.151"
},
{
id: 2,
endpoint: "https://10.117.5.142",
name: "target_02",
username: "AAA",
password: "",
credential: {
access_key: "AAA",
access_secret: "",
type: "basic"
},
description: "test",
insecure: false,
type: 0
name: "target_02",
type: "Harbor",
url: "https://10.117.5.142"
},
{
id: 3,
endpoint: "https://101.1.11.111",
name: "target_03",
username: "admin",
password: "",
credential: {
access_key: "admin",
access_secret: "",
type: "basic"
},
description: "test",
insecure: false,
type: 0
name: "target_03",
type: "Harbor",
url: "https://101.1.11.111"
},
{
id: 4,
endpoint: "http://4.4.4.4",
name: "target_04",
username: "",
password: "",
credential: {
access_key: "admin",
access_secret: "",
type: "basic"
},
description: "test",
insecure: true,
type: 0
name: "target_04",
type: "Harbor",
url: "https://4.4.4.4"
}
];
@ -167,48 +158,49 @@ describe("CreateEditRuleComponent (inline template)", () => {
id: 1,
name: "sync_01",
description: "",
projects: [
{
project_id: 1,
owner_id: 0,
name: "project_01",
creation_time: "",
deleted: 0,
owner_name: "",
togglable: false,
update_time: "",
current_user_role_id: 0,
repo_count: 0,
has_project_admin_role: false,
is_member: false,
role_name: "",
metadata: {
public: "",
enable_content_trust: "",
prevent_vul: "",
severity: "",
auto_scan: ""
}
}
],
targets: [
{
id: 1,
endpoint: "https://10.117.4.151",
name: "target_01",
username: "admin",
password: "",
insecure: false,
type: 0
}
],
src_namespaces: ["namespace1", "namespace2"],
src_registry: {id: 10 },
dest_registry: {id: 0 },
trigger: {
kind: "Manual",
schedule_param: null
type: "Manual",
trigger_settings: {}
},
filters: [],
replicate_existing_image_now: false,
replicate_deletion: false
deletion: false,
enabled: true,
override: true
};
let mockRegistryInfo = {
"type": "harbor",
"description": "",
"supported_resource_filters": [
{
"type": "Name",
"style": "input"
},
{
"type": "Version",
"style": "input"
},
{
"type": "Label",
"style": "input"
},
{
"type": "Resource",
"style": "radio",
"values": [
"repository",
"chart"
]
}
],
"supported_triggers": [
"manual",
"scheduled",
"event_based"
]
};
let fixture: ComponentFixture<ReplicationComponent>;
@ -224,16 +216,18 @@ describe("CreateEditRuleComponent (inline template)", () => {
let spyOneRule: jasmine.Spy;
let spyJobs: jasmine.Spy;
let spyAdapter: jasmine.Spy;
let spyEndpoint: jasmine.Spy;
let config: IServiceConfig = {
replicationJobEndpoint: "/api/jobs/replication/testing",
targetBaseEndpoint: "/api/targets/testing"
replicationBaseEndpoint: "/api/replication/testing",
targetBaseEndpoint: "/api/registries/testing"
};
beforeEach(async(() => {
TestBed.configureTestingModule({
imports: [SharedModule, NoopAnimationsModule],
imports: [SharedModule, NoopAnimationsModule, RouterTestingModule],
declarations: [
ReplicationComponent,
ListReplicationRuleComponent,
@ -250,7 +244,6 @@ describe("CreateEditRuleComponent (inline template)", () => {
{ provide: SERVICE_CONFIG, useValue: config },
{ provide: ReplicationService, useClass: ReplicationDefaultService },
{ provide: EndpointService, useClass: EndpointDefaultService },
{ provide: ProjectService, useClass: ProjectDefaultService },
{ provide: JobLogService, useClass: JobLogDefaultService },
{ provide: OperationService },
{ provide: LabelService }
@ -278,10 +271,11 @@ describe("CreateEditRuleComponent (inline template)", () => {
replicationService,
"getReplicationRule"
).and.returnValue(of(mockRule));
spyJobs = spyOn(replicationService, "getJobs").and.returnValues(
of(mockJob)
);
spyJobs = spyOn(replicationService, "getExecutions").and.returnValues(
of(mockJob));
spyAdapter = spyOn(replicationService, "getRegistryInfo").and.returnValues(
of(mockRegistryInfo));
spyEndpoint = spyOn(endpointService, "getEndpoints").and.returnValues(
of(mockEndpoints)
);

830
src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts
View File

File diff suppressed because it is too large Load Diff

18
src/portal/lib/src/endpoint/endpoint.component.html
View File

@ -17,17 +17,27 @@
<button type="button" class="btn btn-sm btn-secondary" [disabled]="!(selectedRow.length ===1)" (click)="editTargets(selectedRow)" ><clr-icon shape="pencil" size="16"></clr-icon>&nbsp;{{'DESTINATION.EDIT' | translate}}</button>
<button type="button" class="btn btn-sm btn-secondary" [disabled]="!selectedRow.length" (click)="deleteTargets(selectedRow)"><clr-icon shape="times" size="16"></clr-icon>&nbsp;{{'DESTINATION.DELETE' | translate}}</button>
</clr-dg-action-bar>
<clr-dg-column [clrDgField]="'name'">{{'DESTINATION.NAME' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'endpoint'">{{'DESTINATION.URL' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'name'" class="flex-min-width">{{'DESTINATION.NAME' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'status'" class="flex-min-width">{{'DESTINATION.STATUS' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'url'" class="flex-min-width">{{'DESTINATION.URL' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'type'">{{'DESTINATION.PROVIDER' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'insecure'">{{'CONFIG.VERIFY_REMOTE_CERT' | translate }}</clr-dg-column>
<clr-dg-column [clrDgField]="'credential.type'">{{'DESTINATION.AUTHENTICATION' | translate }}</clr-dg-column>
<clr-dg-column [clrDgSortBy]="creationTimeComparator">{{'DESTINATION.CREATION_TIME' | translate}}</clr-dg-column>
<clr-dg-placeholder>{{'DESTINATION.PLACEHOLDER' | translate }}</clr-dg-placeholder>
<clr-dg-row *clrDgItems="let t of targets" [clrDgItem]='t'>
<clr-dg-cell>{{t.name}}</clr-dg-cell>
<clr-dg-cell>{{t.endpoint}}</clr-dg-cell>
<clr-dg-cell class="flex-min-width">{{t.name}}</clr-dg-cell>
<clr-dg-cell [ngSwitch]="t.status" class="flex-min-width">
<div *ngSwitchCase="'unhealthy'"><clr-icon shape="exclamation-circle" class="is-error text-alignment" size="22"></clr-icon> Unhealthy</div>
<div *ngSwitchCase="'healthy'"><clr-icon shape="success-standard" class="is-success text-alignment" size="18"></clr-icon> Healthy</div>
<div *ngSwitchCase="'unknown' || ''"><clr-icon shape="exclamation-triangle" class="is-warning text-alignment" size="22"></clr-icon> Unknown</div>
</clr-dg-cell>
<clr-dg-cell class="flex-min-width">{{t.url}}</clr-dg-cell>
<clr-dg-cell>{{t.type}}</clr-dg-cell>
<clr-dg-cell>
{{!t.insecure}}
</clr-dg-cell>
<clr-dg-cell>{{t.credential.type}}</clr-dg-cell>
<clr-dg-cell>{{t.creation_time | date: 'short'}}</clr-dg-cell>
</clr-dg-row>
<clr-dg-footer>

4
src/portal/lib/src/endpoint/endpoint.component.scss
View File

@ -25,4 +25,8 @@
.endpoint-view {
position: relative;
}
.flex-min-width {
min-width: 180px;
}

78
src/portal/lib/src/endpoint/endpoint.component.spec.ts
View File

@ -25,54 +25,76 @@ describe("EndpointComponent (inline template)", () => {
let mockData: Endpoint[] = [
{
id: 1,
endpoint: "https://10.117.4.151",
credential: {
access_key: "admin",
access_secret: "",
type: "basic"
},
description: "test",
insecure: false,
name: "target_01",
username: "admin",
password: "",
insecure: true,
type: 0
type: "Harbor",
url: "https://10.117.4.151"
},
{
id: 2,
endpoint: "https://10.117.5.142",
name: "target_02",
username: "AAA",
password: "",
credential: {
access_key: "AAA",
access_secret: "",
type: "basic"
},
description: "test",
insecure: false,
type: 0
name: "target_02",
type: "Harbor",
url: "https://10.117.5.142"
},
{
id: 3,
endpoint: "https://101.1.11.111",
name: "target_03",
username: "admin",
password: "",
credential: {
access_key: "admin",
access_secret: "",
type: "basic"
},
description: "test",
insecure: false,
type: 0
name: "target_03",
type: "Harbor",
url: "https://101.1.11.111"
},
{
id: 4,
endpoint: "http://4.4.4.4",
name: "target_04",
username: "",
password: "",
credential: {
access_key: "admin",
access_secret: "",
type: "basic"
},
description: "test",
insecure: false,
type: 0
name: "target_04",
type: "Harbor",
url: "https://4.4.4.4"
}
];
let mockOne: Endpoint[] = [
{
id: 1,
endpoint: "https://10.117.4.151",
name: "target_01",
username: "admin",
password: "",
credential: {
access_key: "admin",
access_secret: "",
type: "basic"
},
description: "test",
insecure: false,
type: 0
name: "target_01",
type: "Harbor",
url: "https://10.117.4.151"
}
];
let mockAdapters = ['harbor', 'docker hub'];
let comp: EndpointComponent;
let fixture: ComponentFixture<EndpointComponent>;
let config: IServiceConfig = {
@ -81,6 +103,7 @@ describe("EndpointComponent (inline template)", () => {
let endpointService: EndpointService;
let spy: jasmine.Spy;
let spyAdapter: jasmine.Spy;
let spyOnRules: jasmine.Spy;
let spyOne: jasmine.Spy;
beforeEach(async(() => {
@ -111,6 +134,11 @@ describe("EndpointComponent (inline template)", () => {
spy = spyOn(endpointService, "getEndpoints").and.returnValues(
of(mockData)
);
spyAdapter = spyOn(endpointService, "getAdapters").and.returnValue(
of(mockAdapters)
);
spyOnRules = spyOn(
endpointService,
"getEndpointWithReplicationRules"

17
src/portal/lib/src/endpoint/endpoint.component.ts
View File

@ -76,12 +76,16 @@ export class EndpointComponent implements OnInit, OnDestroy {
get initEndpoint(): Endpoint {
return {
endpoint: "",
name: "",
username: "",
password: "",
credential: {
access_key: "",
access_secret: "",
type: ""
},
description: "",
insecure: false,
type: 0
name: "",
type: "",
url: "",
};
}
@ -90,7 +94,6 @@ export class EndpointComponent implements OnInit, OnDestroy {
private translateService: TranslateService,
private operationService: OperationService,
private ref: ChangeDetectorRef) {
this.forceRefreshView(1000);
}
ngOnInit(): void {
@ -114,8 +117,8 @@ export class EndpointComponent implements OnInit, OnDestroy {
this.endpointService.getEndpoints(this.targetName)
.subscribe(targets => {
this.targets = targets || [];
this.forceRefreshView(1000);
this.loading = false;
this.forceRefreshView(1000);
}, error => {
this.errorHandler.error(error);
this.loading = false;

8
src/portal/lib/src/harbor-library.module.ts
View File

@ -79,13 +79,13 @@ import { OperationService } from './operation/operation.service';
* this default configuration.
*/
export const DefaultServiceConfig: IServiceConfig = {
baseEndpoint: "/api",
systemInfoEndpoint: "/api/systeminfo",
repositoryBaseEndpoint: "/api/repositories",
logBaseEndpoint: "/api/logs",
targetBaseEndpoint: "/api/targets",
replicationBaseEndpoint: "/api/replications",
replicationRuleEndpoint: "/api/policies/replication",
replicationJobEndpoint: "/api/jobs/replication",
targetBaseEndpoint: "/api/registries",
replicationBaseEndpoint: "/api/replication",
replicationRuleEndpoint: "/api/replication/policies",
vulnerabilityScanningBaseEndpoint: "/api/repositories",
projectPolicyEndpoint: "/api/projects/configs",
projectBaseEndpoint: "/api/projects",

49
src/portal/lib/src/list-replication-rule/list-replication-rule.component.html
View File

@ -6,28 +6,41 @@
<button type="button" class="btn btn-sm btn-secondary" *ngIf="hasDeleteReplicationPermission" [disabled]="!selectedRow" (click)="deleteRule(selectedRow)"><clr-icon shape="times" size="16"></clr-icon>&nbsp;{{'REPLICATION.DELETE_POLICY' | translate}}</button>
<button type="button" class="btn btn-sm btn-secondary" *ngIf="hasExecuteReplicationPermission" [disabled]="!selectedRow" (click)="replicateRule(selectedRow)"><clr-icon shape="export" size="16"></clr-icon>&nbsp;{{'REPLICATION.REPLICATE' | translate}}</button>
</clr-dg-action-bar>
<clr-dg-column [clrDgField]="'name'">{{'REPLICATION.NAME' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'status'">{{'REPLICATION.STATUS' | translate}}</clr-dg-column>
<clr-dg-column *ngIf="!projectScope">{{'REPLICATION.PROJECT' | translate}}</clr-dg-column>
<clr-dg-column>{{'REPLICATION.NAME' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'status'" class="status-width">{{'REPLICATION.STATUS' | translate}}</clr-dg-column>
<clr-dg-column class="col-width">{{'REPLICATION.SRC_REGISTRY' | translate}}</clr-dg-column>
<clr-dg-column class="col-width">{{'REPLICATION.REPLICATION_MODE' | translate}}</clr-dg-column>
<clr-dg-column class="min-width">{{'REPLICATION.DESTINATION_NAMESPACE' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'trigger'">{{'REPLICATION.REPLICATION_TRIGGER' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'override'" class="status-width">{{'REPLICATION.OVERRIDE' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'description'">{{'REPLICATION.DESCRIPTION' | translate}}</clr-dg-column>
<clr-dg-column >{{'REPLICATION.DESTINATION_NAME' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'trigger'">{{'REPLICATION.TRIGGER_MODE' | translate}}</clr-dg-column>
<clr-dg-placeholder>{{'REPLICATION.PLACEHOLDER' | translate }}</clr-dg-placeholder>
<clr-dg-row *clrDgItems="let p of changedRules" [clrDgItem]="p" [style.backgroundColor]="(projectScope && withReplicationJob && selectedId === p.id) ? '#eee' : ''">
<clr-dg-row *clrDgItems="let p of changedRules; let i=index" [clrDgItem]="p" [style.backgroundColor]="(projectScope && withReplicationJob && selectedId === p.id) ? '#eee' : ''">
<clr-dg-cell>{{p.name}}</clr-dg-cell>
<clr-dg-cell>
<div [ngSwitch]="hasDeletedLabel(p)">
<clr-tooltip *ngSwitchCase="'disabled'" class="tooltip-lg">
<clr-icon clrTooltipTrigger shape="exclamation-triangle" class="is-warning text-alignment" size="22"></clr-icon>Disabled
<clr-tooltip-content clrPosition="top-right" clrSize="xs" *clrIfOpen>
<span>{{'REPLICATION.RULE_DISABLED' | translate}}</span>
</clr-tooltip-content>
</clr-tooltip>
<div *ngSwitchCase="'enabled'" ><clr-icon shape="success-standard" class="is-success text-alignment" size="18"></clr-icon> Enabled</div>
<clr-dg-cell class="status-width">
<div [ngSwitch]="p.enabled">
<clr-tooltip *ngSwitchCase="false" class="tooltip-lg">
<clr-icon clrTooltipTrigger shape="exclamation-triangle" class="is-warning text-alignment" size="22"></clr-icon>Disabled
<clr-tooltip-content clrPosition="top-right" clrSize="xs" *clrIfOpen>
<span>{{'REPLICATION.RULE_DISABLED' | translate}}</span>
</clr-tooltip-content>
</clr-tooltip>
<div *ngSwitchCase="true" ><clr-icon shape="success-standard" class="is-success text-alignment" size="18"></clr-icon> Enabled</div>
</div>
</clr-dg-cell>
<clr-dg-cell *ngIf="!projectScope">
<a href="javascript:void(0)" (click)="$event.stopPropagation(); redirectTo(p)">{{p.projects?.length>0 ? p.projects[0].name : ''}}</a>
<clr-dg-cell class="col-width">
{{p.src_registry ? p.src_registry.name : ''}}
</clr-dg-cell>
<clr-dg-cell class="col-width">
{{p.src_registry && p.src_registry.id > 0 ? 'pull-based' : 'push-based'}}
</clr-dg-cell>
<clr-dg-cell class="min-width">
{{p.dest_registry ? p.dest_registry.name : ''}} : {{p.dest_namespace? p.dest_namespace: '-'}}
</clr-dg-cell>
<clr-dg-cell>{{p.trigger ? p.trigger.type : ''}}</clr-dg-cell>
<clr-dg-cell [ngSwitch]="p.override">
<clr-icon shape="check-circle" *ngSwitchCase="true" size="20" class="color-green"></clr-icon>
<clr-icon shape="times-circle" *ngSwitchCase="false" size="16" class="icon-style"></clr-icon>
</clr-dg-cell>
<clr-dg-cell>
{{p.description ? trancatedDescription(p.description) : '-'}}
@ -38,8 +51,6 @@
</clr-tooltip-content>
</clr-tooltip>
</clr-dg-cell>
<clr-dg-cell>{{p.targets?.length>0 ? p.targets[0].name : ''}}</clr-dg-cell>
<clr-dg-cell>{{p.trigger ? p.trigger.kind : ''}}</clr-dg-cell>
</clr-dg-row>
<clr-dg-footer>
<span *ngIf="pagination.totalItems">{{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'REPLICATION.OF' | translate}} </span>{{pagination.totalItems }} {{'REPLICATION.ITEMS' | translate}}

15
src/portal/lib/src/list-replication-rule/list-replication-rule.component.scss
View File

@ -4,4 +4,17 @@
.text-alignment {
vertical-align: text-bottom;
}
}
.min-width {
width: 224px;
}
.col-width {
width: 140px;
}
.status-width {
width: 105px;
}
.icon-style {
color: #C92100;
}

79
src/portal/lib/src/list-replication-rule/list-replication-rule.component.spec.ts
View File

@ -21,83 +21,29 @@ describe('ListReplicationRuleComponent (inline template)', () => {
let mockRules: ReplicationRule[] = [
{
"id": 1,
"projects": [{
"project_id": 33,
"owner_id": 1,
"name": "aeas",
"deleted": 0,
"togglable": false,
"current_user_role_id": 0,
"repo_count": 0,
"metadata": {
"public": false,
"enable_content_trust": "",
"prevent_vul": "",
"severity": "",
"auto_scan": ""},
"owner_name": "",
"creation_time": null,
"update_time": null,
"has_project_admin_role": true,
"is_member": true,
"role_name": ""
}],
"targets": [{
"endpoint": "",
"id": 0,
"insecure": false,
"name": "khans3",
"username": "",
"password": "",
"type": 0,
}],
"name": "sync_01",
"description": "",
"filters": null,
"trigger": {"kind": "Manual", "schedule_param": null},
"trigger": {"type": "Manual", "trigger_settings": null},
"error_job_count": 2,
"replicate_deletion": false,
"replicate_existing_image_now": false,
"deletion": false,
"src_namespaces": ["name1", "name2"],
"src_registry": {id: 3},
"enabled": true,
"override": true
},
{
"id": 2,
"projects": [{
"project_id": 33,
"owner_id": 1,
"name": "aeas",
"deleted": 0,
"togglable": false,
"current_user_role_id": 0,
"repo_count": 0,
"metadata": {
"public": false,
"enable_content_trust": "",
"prevent_vul": "",
"severity": "",
"auto_scan": ""},
"owner_name": "",
"creation_time": null,
"update_time": null,
"has_project_admin_role": true,
"is_member": true,
"role_name": ""
}],
"targets": [{
"endpoint": "",
"id": 0,
"insecure": false,
"name": "khans3",
"username": "",
"password": "",
"type": 0,
}],
"name": "sync_02",
"description": "",
"filters": null,
"trigger": {"kind": "Manual", "schedule_param": null},
"trigger": {"type": "Manual", "trigger_settings": null},
"error_job_count": 2,
"replicate_deletion": false,
"replicate_existing_image_now": false,
"deletion": false,
"src_namespaces": ["name1", "name2"],
"dest_registry": {id: 3},
"enabled": true,
"override": true
},
];
@ -137,6 +83,7 @@ describe('ListReplicationRuleComponent (inline template)', () => {
comp = fixture.componentInstance;
replicationService = fixture.debugElement.injector.get(ReplicationService);
spyRules = spyOn(replicationService, 'getReplicationRules').and.returnValues(of(mockRules));
fixture.detectChanges();
});

6
src/portal/lib/src/list-replication-rule/list-replication-rule.component.ts
View File

@ -27,9 +27,8 @@ import {
import { Comparator } from "../service/interface";
import { TranslateService } from "@ngx-translate/core";
import { map, catchError } from "rxjs/operators";
import { Observable, forkJoin, throwError as observableThrowError } from "rxjs";
import { Observable, forkJoin, of, throwError as observableThrowError } from "rxjs";
import { ReplicationService } from "../service/replication.service";
import {
ReplicationJob,
ReplicationJobItem,
@ -198,7 +197,8 @@ export class ListReplicationRuleComponent implements OnInit, OnChanges {
let ruleData: ReplicationJobItem[];
this.canDeleteRule = true;
let count = 0;
return this.replicationService.getJobs(id)
return this.replicationService.getExecutions(id)
.pipe(map(response => {
ruleData = response.data;
if (ruleData.length) {

5
src/portal/lib/src/replication/index.ts
View File

@ -1,8 +1,11 @@
import { Type } from '@angular/core';
import { ReplicationComponent } from './replication.component';
import { ReplicationTasksComponent } from './replication-tasks/replication-tasks.component';
export * from './replication.component';
export * from './replication-tasks/replication-tasks.component';
export const REPLICATION_DIRECTIVES: Type<any>[] = [
ReplicationComponent
ReplicationComponent,
ReplicationTasksComponent
];

140
src/portal/lib/src/replication/replication-tasks/replication-tasks.component.html Normal file
View File

@ -0,0 +1,140 @@
<div class="replication-tasks">
<section class="overview-section">
<div class="title-wrapper">
<div>
<div>
<a (click)="onBack()" class="onback"><{{'PROJECT_DETAIL.REPLICATION'|
translate}}</a>
</div>
<div class="title-block">
<div>
<h2 class="custom-h2 h2-style">{{'REPLICATION.REPLICATION_EXECUTION'|
translate}}</h2>
<span class="id-divider"></span>
<h2 class="custom-h2 h2-style">{{executionId}}</h2>
</div>
<div>
<div class="status-progress" *ngIf="executions && executions['status'] === 'InProgress'">
<span class="spinner spinner-inline"></span>
<span>{{'REPLICATION.IN_PROGRESS'| translate}}</span>
</div>
<div class="status-success" *ngIf="executions && executions['status'] === 'Succeed'">
<clr-icon size="18" shape="success-standard" class="color-green"></clr-icon>
<span>{{'REPLICATION.SUCCESS'| translate}}</span>
</div>
<div class="status-failed" *ngIf="executions && executions['status'] === 'Failed'">
<clr-icon size="18" shape="error-standard" class="color-red"></clr-icon>
<span>{{'REPLICATION.FAILURE'| translate}}</span>
</div>
</div>
<div>
<button class="btn btn-primary btn-sm" (click)="stopJob()"
[disabled]="stopOnGoing">{{'REPLICATION.STOPJOB' | translate}}</button>
</div>
</div>
</div>
</div>
<div class="execution-block">
<div class="executions-detail">
<div>
<label>{{'REPLICATION.TRIGGER_MODE' | translate}} :</label>
<span>{{trigger| translate}}</span>
</div>
<div>
<label>{{'REPLICATION.CREATION_TIME' | translate}} :</label>
<span>{{startTime | date: 'short'}}</span>
</div>
</div>
<div class="flex-block">
<section class="execution-detail-label">
<section class="detail-row">
<div class="num-success common-style"></div>
<label class="detail-span">{{'REPLICATION.SUCCESS'| translate}}</label>
<div class="execution-details">{{successNum}}</div>
</section>
<section class="detail-row">
<div class="num-failed common-style"></div>
<label class="detail-span">{{'REPLICATION.FAILURE'| translate}}</label>
<div class="execution-details">{{failedNum}}</div>
</section>
<section class="detail-row">
<div class="num-progress common-style"></div>
<label class="detail-span">{{'REPLICATION.IN_PROGRESS'| translate}}</label>
<div class="execution-details">{{progressNum}}</div>
</section>
<section class="detail-row">
<div class="num-stopped common-style"></div>
<label class="detail-span">{{'REPLICATION.STOPPED'| translate}}</label>
<div class="execution-details">{{stoppedNum}}</div>
</section>
</section>
</div>
</div>
</section>
<div class="tasks-detail">
<h3 class="modal-title">Tasks</h3>
<div class="row flex-items-xs-between flex-items-xs-bottom">
<div class="action-select">
<div class="select filter-tag" [hidden]="!isOpenFilterTag">
<select (change)="selectFilter($event)">
<option value="resource_type">{{'REPLICATION.RESOURCE_TYPE' |translate}}</option>
<option value="status">{{'REPLICATION.STATUS' | translate}}</option>
</select>
</div>
<hbr-filter [withDivider]="true" (openFlag)="openFilter($event)"
filterPlaceholder='{{"REPLICATION.FILTER_PLACEHOLDER" | translate}}'
(filterEvt)="doSearch($event)" [currentValue]="searchTask"></hbr-filter>
<span class="refresh-btn">
<clr-icon shape="refresh" (click)="refreshTasks()"></clr-icon>
</span>
</div>
</div>
<clr-datagrid (clrDgRefresh)="clrLoadTasks($event)" [(clrDgSelected)]="selectedRow" [clrDgLoading]="loading">
<clr-dg-column>{{'REPLICATION.TASK_ID'| translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'resource_type'">{{'REPLICATION.RESOURCE_TYPE'
| translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'src_resource'">{{'REPLICATION.SOURCE' |
translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'dst_resource'">{{'REPLICATION.DESTINATION' |
translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'operation'">{{'REPLICATION.OPERATION' |
translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'status'">{{'REPLICATION.STATUS' |
translate}}</clr-dg-column>
<clr-dg-column [clrDgSortBy]="startTimeComparator">{{'REPLICATION.CREATION_TIME'
| translate}}</clr-dg-column>
<clr-dg-column [clrDgSortBy]="endTimeComparator">{{'REPLICATION.END_TIME'
| translate}}</clr-dg-column>
<clr-dg-column>{{'REPLICATION.LOGS' | translate}}</clr-dg-column>
<clr-dg-row *clrDgItems="let t of tasks">
<clr-dg-cell>{{t.id}}</clr-dg-cell>
<clr-dg-cell>{{t.resource_type}}</clr-dg-cell>
<clr-dg-cell>{{t.src_resource}}</clr-dg-cell>
<clr-dg-cell>{{t.dst_resource}}</clr-dg-cell>
<clr-dg-cell>{{t.operation}}</clr-dg-cell>
<clr-dg-cell>{{t.status}}</clr-dg-cell>
<clr-dg-cell>{{t.start_time | date: 'short'}}</clr-dg-cell>
<clr-dg-cell>{{t.end_time | date: 'short'}}</clr-dg-cell>
<clr-dg-cell>
<span *ngIf="t.status=='InProgress'; else elseBlock" class="label">{{'REPLICATION.NO_LOGS'
| translate}}</span>
<ng-template #elseBlock>
<a target="_blank" [href]="viewLog(t.id)">
<clr-icon shape="list"></clr-icon>
</a>
</ng-template>
</clr-dg-cell>
</clr-dg-row>
<clr-dg-footer>
<span *ngIf="pagination.totalItems">{{pagination.firstItem + 1}}
-
{{pagination.lastItem +1 }} {{'ROBOT_ACCOUNT.OF' |
translate}} </span>
{{pagination.totalItems }} {{'ROBOT_ACCOUNT.ITEMS' | translate}}
<clr-dg-pagination #pagination [clrDgPageSize]="pageSize" [(clrDgPage)]="currentPage" [clrDgTotalItems]="totalCount"></clr-dg-pagination>
</clr-dg-footer>
</clr-datagrid>
</div>
</div>

110
src/portal/lib/src/replication/replication-tasks/replication-tasks.component.scss Normal file
View File

@ -0,0 +1,110 @@
.replication-tasks {
.overview-section {
.title-wrapper {
.onback{
color: #007cbb;
font-size: 14px;
cursor: pointer;
}
.title-block {
display: flex;
align-items: center;
>div:first-child {
width: 250px;
}
>div:nth-child(2) {
width: 140px;
span {
color: #007cbb;
font-size: 12px;
margin-left: 10px;
}
}
.id-divider {
display: inline-block;
height: 25px;
width: 2px;
background-color: #cccccc;
margin: 0 20px;
}
.h2-style {
display: inline-block;
}
}
}
.execution-block {
margin-top: 24px;
display: flex;
flex-wrap: row wrap;
.execution-detail-label {
margin-right: 10px;
text-align: left;
.detail-row {
display: flex;
height: 27px;
.common-style {
width: 12px;
height: 12px;
margin-top: 16px;
}
.num-success {
background-color: #308700;
}
.num-failed {
background-color: #C92101;
}
.num-progress {
background-color: #1C5898;
}
.num-stopped {
background-color: #A1A1A1;
}
.detail-span {
flex:0 0 100px;
margin: 10px 0 0 10px;
}
.execution-details {
width: 200px;
margin: 8px 35px;
}
}
}
.executions-detail {
width: 400px;
label {
display: inline-block;
color: black;
width: 120px;
}
>div {
margin-top: 10px;
}
}
}
}
.tasks-detail {
margin-top: 65px;
.action-select {
padding-right: 18px;
height: 24px;
display: flex;
justify-content: flex-end;
.filter-tag {
float: left;
margin-top: 8px;
}
.refresh-btn {
cursor: pointer;
margin-top: 7px;
}
.refresh-btn:hover {
color: #007CBB;
}
}
clr-datagrid {
margin-top: 20px;
}
}
}

243
src/portal/lib/src/replication/replication-tasks/replication-tasks.component.ts Normal file
View File

@ -0,0 +1,243 @@
import { Component, OnInit, Input, OnDestroy } from '@angular/core';
import { Router } from '@angular/router';
import { ReplicationService } from "../../service/replication.service";
import { TranslateService } from '@ngx-translate/core';
import { finalize } from "rxjs/operators";
import { Subscription, timer } from "rxjs";
import { ErrorHandler } from "../../error-handler/error-handler";
import { ReplicationJob, ReplicationTasks, Comparator, ReplicationJobItem, State } from "../../service/interface";
import { CustomComparator, DEFAULT_PAGE_SIZE, calculatePage, doFiltering, doSorting } from "../../utils";
import { RequestQueryParams } from "../../service/RequestQueryParams";
const taskStatus: any = {
PENDING: "pending",
RUNNING: "running",
SCHEDULED: "scheduled"
};
@Component({
selector: 'replication-tasks',
templateUrl: './replication-tasks.component.html',
styleUrls: ['./replication-tasks.component.scss']
})
export class ReplicationTasksComponent implements OnInit, OnDestroy {
isOpenFilterTag: boolean;
selectedRow: [];
currentPage: number = 1;
currentPagePvt: number = 0;
totalCount: number = 0;
pageSize: number = DEFAULT_PAGE_SIZE;
currentState: State;
loading = true;
searchTask: string;
defaultFilter = "resource_type";
tasks: ReplicationTasks;
taskItem: ReplicationTasks[] = [];
tasksCopy: ReplicationTasks[] = [];
stopOnGoing: boolean;
executions: ReplicationJobItem[];
timerDelay: Subscription;
@Input() executionId: string;
startTimeComparator: Comparator<ReplicationJob> = new CustomComparator<
ReplicationJob
>("start_time", "date");
endTimeComparator: Comparator<ReplicationJob> = new CustomComparator<
ReplicationJob
>("end_time", "date");
constructor(
private translate: TranslateService,
private router: Router,
private replicationService: ReplicationService,
private errorHandler: ErrorHandler,
) { }
ngOnInit(): void {
this.searchTask = '';
this.getExecutionDetail();
}
getExecutionDetail(): void {
if (this.executionId) {
this.replicationService.getExecutionById(this.executionId)
.subscribe(res => {
this.executions = res.data;
},
error => {
this.errorHandler.error(error);
});
}
}
public get trigger(): string {
return this.executions && this.executions['trigger']
? this.executions['trigger']
: "";
}
public get startTime(): Date {
return this.executions && this.executions['start_time']
? this.executions['start_time']
: null;
}
public get successNum(): string {
return this.executions && this.executions['succeed'];
}
public get failedNum(): string {
return this.executions && this.executions['failed'];
}
public get progressNum(): string {
return this.executions && this.executions['in_progress'];
}
public get stoppedNum(): string {
return this.executions && this.executions['stopped'];
}
stopJob() {
this.stopOnGoing = true;
this.replicationService.stopJobs(this.executionId)
.subscribe(response => {
this.stopOnGoing = false;
this.getExecutionDetail();
this.translate.get("REPLICATION.STOP_SUCCESS", { param: this.executionId }).subscribe((res: string) => {
this.errorHandler.info(res);
});
},
error => {
this.errorHandler.error(error);
});
}
viewLog(taskId: number | string): string {
return this.replicationService.getJobBaseUrl() + "/executions/" + this.executionId + "/tasks/" + taskId + "/log";
}
ngOnDestroy() {
if (this.timerDelay) {
this.timerDelay.unsubscribe();
}
}
clrLoadTasks(state: State): void {
if (!state || !state.page) {
return;
}
// Keep it for future filter
this.currentState = state;
let pageNumber: number = calculatePage(state);
if (pageNumber !== this.currentPagePvt) {
// load data
let params: RequestQueryParams = new RequestQueryParams();
params.set("page", '' + pageNumber);
params.set("page_size", '' + this.pageSize);
if (this.searchTask && this.searchTask !== "") {
params.set(this.defaultFilter, this.searchTask);
}
this.loading = true;
this.replicationService.getReplicationTasks(this.executionId, params)
.pipe(finalize(() => (this.loading = false)))
.subscribe(res => {
this.totalCount = res.length;
this.tasks = res; // Keep the data
this.taskItem = this.tasks.filter(tasks => tasks.resource_type !== "");
if (!this.timerDelay) {
this.timerDelay = timer(10000, 10000).subscribe(() => {
let count: number = 0;
this.tasks.forEach(tasks => {
if (
tasks.status.toLowerCase() === taskStatus.PENDING ||
tasks.status.toLowerCase() === taskStatus.RUNNING ||
tasks.status.toLowerCase() === taskStatus.SCHEDULED
) {
count++;
}
});
if (count > 0) {
this.clrLoadTasks(this.currentState);
} else {
this.timerDelay.unsubscribe();
this.timerDelay = null;
}
});
}
this.taskItem = doFiltering<ReplicationTasks>(this.taskItem, state);
this.taskItem = doSorting<ReplicationTasks>(this.taskItem, state);
this.currentPagePvt = pageNumber;
},
error => {
this.errorHandler.error(error);
});
} else {
this.taskItem = this.tasks.filter(tasks => tasks.resource_type !== "");
// Do customized filter
this.taskItem = doFiltering<ReplicationTasks>(this.taskItem, state);
// Do customized sorting
this.taskItem = doSorting<ReplicationTasks>(this.taskItem, state);
}
}
onBack(): void {
this.router.navigate(["harbor", "replications"]);
}
selectFilter($event: any): void {
this.defaultFilter = $event['target'].value;
this.doSearch(this.searchTask);
}
// refresh icon
refreshTasks(): void {
this.searchTask = '';
this.loading = true;
this.replicationService.getReplicationTasks(this.executionId)
.subscribe(res => {
this.tasks = res;
this.loading = false;
},
error => {
this.loading = false;
this.errorHandler.error(error);
});
}
public doSearch(value: string): void {
if (!value) {
return;
}
this.searchTask = value.trim();
this.loading = true;
this.currentPage = 1;
if (this.currentPagePvt === 1) {
// Force reloading
let st: State = this.currentState;
if (!st) {
st = {
page: {}
};
}
st.page.from = 0;
st.page.to = this.pageSize - 1;
st.page.size = this.pageSize;
this.currentPagePvt = 0;
this.clrLoadTasks(st);
}
}
openFilter(isOpen: boolean): void {
if (isOpen) {
this.isOpenFilterTag = true;
} else {
this.isOpenFilterTag = false;
}
}
}

85
src/portal/lib/src/replication/replication.component.html
View File

@ -23,56 +23,60 @@
<div class="col-lg-12 col-md-12 col-sm-12 col-xs-12 jobList" [hidden]='hiddenJobList'>
<div *ngIf="withReplicationJob" class="col-lg-12 col-md-12 col-sm-12 col-xs-12">
<div class="row flex-items-xs-between jobsRow">
<h5 class="flex-items-xs-bottom option-left-down">{{'REPLICATION.REPLICATION_JOBS' | translate}}</h5>
<div class="flex-items-xs-bottom option-right-down">
<button class="btn btn-link" (click)="toggleSearchJobOptionalName(currentJobSearchOption)">{{toggleJobSearchOption[currentJobSearchOption] | translate}}</button>
<hbr-filter [withDivider]="true" filterPlaceholder='{{"REPLICATION.FILTER_JOBS_PLACEHOLDER" | translate}}' (filterEvt)="doSearchJobs($event)"
[currentValue]="search.repoName"></hbr-filter>
<span class="refresh-btn" (click)="refreshJobs()">
<clr-icon shape="refresh"></clr-icon>
</span>
</div>
</div>
<div class="row flex-items-xs-right row-right" [hidden]="currentJobSearchOption === 0">
<div class="select select-status">
<select (change)="doFilterJobStatus($event)">
<option *ngFor="let j of jobStatus" value="{{j.key}}" [selected]="currentJobStatus.key === j.key">{{j.description | translate}}</option>
</select>
</div>
<div class="flex-items-xs-middle">
<hbr-datetime [dateInput]="search.startTime" (search)="doJobSearchByStartTime($event)"></hbr-datetime>
<hbr-datetime [dateInput]="search.endTime" [oneDayOffset]="true" (search)="doJobSearchByEndTime($event)"></hbr-datetime>
<h5 class="flex-items-xs-bottom option-left-down">{{'REPLICATION.REPLICATION_EXECUTIONS' | translate}}</h5>
<div class="row flex-items-xs-between flex-items-xs-bottom">
<div class="execution-select">
<div class="select filter-tag" [hidden]="!isOpenFilterTag">
<select (change)="doFilterJob($event)">
<option value="trigger">{{'REPLICATION.REPLICATION_TRIGGER' |translate}}</option>
<option value="status">{{'REPLICATION.STATUS' | translate}}</option>
</select>
</div>
<hbr-filter [withDivider]="true" (openFlag)="openFilter($event)"
filterPlaceholder='{{"REPLICATION.FILTER_EXECUTIONS_PLACEHOLDER" | translate}}'
(filterEvt)="doSearchExecutions($event)" [currentValue]="currentTerm"></hbr-filter>
<span class="refresh-btn">
<clr-icon shape="refresh" (click)="refreshJobs()"></clr-icon>
</span>
</div>
</div>
</div>
</div>
<div *ngIf="withReplicationJob" class="col-lg-12 col-md-12 col-sm-12 col-xs-12">
<clr-datagrid [clrDgLoading]="jobsLoading" (clrDgRefresh)="clrLoadJobs($event)">
<clr-datagrid [(clrDgSelected)]="selectedRow" [clrDgLoading]="jobsLoading" (clrDgRefresh)="clrLoadJobs($event)">
<clr-dg-action-bar>
<div class="btn-group">
<button type="button" class="btn btn-sm btn-secondary" *ngIf="isSystemAdmin" [disabled]="!(jobs && jobs.length>0) || isStopOnGoing"
(click)="stopJobs()">{{'REPLICATION.STOPJOB' | translate}}</button>
<button type="button" class="btn btn-sm btn-secondary" *ngIf="isSystemAdmin" [disabled]="!(jobs && jobs.length>0) || isStopOnGoing || !selectedRow.length"
(click)="openStopExecutionsDialog(selectedRow)">{{'REPLICATION.STOPJOB' | translate}}</button>
</div>
</clr-dg-action-bar>
<clr-dg-column [clrDgField]="'repository'">{{'REPLICATION.NAME' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'status'">{{'REPLICATION.STATUS' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'operation'">{{'REPLICATION.OPERATION' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'id'">{{'REPLICATION.ID' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'trigger'">{{'REPLICATION.REPLICATION_TRIGGER' | translate}}</clr-dg-column>
<clr-dg-column [clrDgSortBy]="creationTimeComparator">{{'REPLICATION.CREATION_TIME' | translate}}</clr-dg-column>
<clr-dg-column [clrDgSortBy]="updateTimeComparator">{{'REPLICATION.UPDATE_TIME' | translate}}</clr-dg-column>
<clr-dg-column>{{'REPLICATION.LOGS' | translate}}</clr-dg-column>
<clr-dg-column>{{'REPLICATION.DURATION' | translate}}</clr-dg-column>
<clr-dg-column>{{'REPLICATION.SUCCESS_RATE' | translate}}</clr-dg-column>
<clr-dg-column>{{'REPLICATION.TOTAL' | translate}}</clr-dg-column>
<clr-dg-column [clrDgField]="'status'">{{'REPLICATION.STATUS' | translate}}</clr-dg-column>
<clr-dg-placeholder>{{'REPLICATION.JOB_PLACEHOLDER' | translate }}</clr-dg-placeholder>
<clr-dg-row *ngFor="let j of jobs">
<clr-dg-cell>{{j.repository}}</clr-dg-cell>
<clr-dg-cell>{{j.status}}</clr-dg-cell>
<clr-dg-cell>{{j.operation}}</clr-dg-cell>
<clr-dg-cell>{{j.creation_time | date: 'short'}}</clr-dg-cell>
<clr-dg-cell>{{j.update_time | date: 'short'}}</clr-dg-cell>
<clr-dg-row *ngFor="let j of jobs" [clrDgItem]="j">
<clr-dg-cell>
<span *ngIf="j.status=='pending'; else elseBlock" class="label">{{'REPLICATION.NO_LOGS' | translate}}</span>
<ng-template #elseBlock>
<a target="_blank" [href]="viewLog(j.id)">
<clr-icon shape="list"></clr-icon>
</a>
</ng-template>
<a href="javascript:void(0)" (click)="goToLink(j.id)">{{j.id}}</a>
</clr-dg-cell>
<clr-dg-cell>{{j.trigger}}</clr-dg-cell>
<clr-dg-cell>{{j.start_time | date: 'short'}}</clr-dg-cell>
<clr-dg-cell>{{getDuration(j)}}</clr-dg-cell>
<clr-dg-cell>
{{(j.succeed > 0 ? j.succeed / j.total : 0) | percent }}
</clr-dg-cell>
<clr-dg-cell>{{j.total}}</clr-dg-cell>
<clr-dg-cell>
{{j.status}}
<clr-tooltip>
<clr-icon *ngIf="j.status_text" clrTooltipTrigger shape="info-circle" size="20"></clr-icon>
<clr-tooltip-content [clrPosition]="'left'" clrSize="md" *clrIfOpen>
<span>{{j.status_text}}</span>
</clr-tooltip-content>
</clr-tooltip>
</clr-dg-cell>
</clr-dg-row>
<clr-dg-footer>
@ -83,7 +87,8 @@
</clr-datagrid>
</div>
</div>
<hbr-create-edit-rule *ngIf="isSystemAdmin" [withAdmiral]="withAdmiral" [projectId]="projectId" [projectName]="projectName"
<hbr-create-edit-rule *ngIf="isSystemAdmin" [withAdmiral]="withAdmiral"
(goToRegistry)="goRegistry()" (reload)="reloadRules($event)"></hbr-create-edit-rule>
<confirmation-dialog #replicationConfirmDialog (confirmAction)="confirmReplication($event)"></confirmation-dialog>
<confirmation-dialog #StopConfirmDialog (confirmAction)="confirmStop($event)"></confirmation-dialog>
</div>

27
src/portal/lib/src/replication/replication.component.scss
View File

@ -13,6 +13,25 @@
padding-right: 16px;
}
.execution-select {
padding-right: 18px;
height: 24px;
display: flex;
justify-content: flex-end;
padding-top: 25px;
.filter-tag {
float: left;
margin-top: 8px;
}
.refresh-btn {
cursor: pointer;
margin-top: 7px;
&:hover {
color: #007CBB;
}
}
}
.rightPos{
position: absolute;
right: 35px;
@ -28,7 +47,7 @@
}
.row-right {
padding-right: 50px;
margin-left: 564px;
}
.replication-row {
@ -48,4 +67,10 @@
margin-top: 24px;
}
}
}
clr-datagrid {
::ng-deep .datagrid-table {
position: static;
}
}

194
src/portal/lib/src/replication/replication.component.spec.ts
View File

@ -22,167 +22,106 @@ import {ProjectDefaultService, ProjectService} from "../service/project.service"
import {OperationService} from "../operation/operation.service";
import {FilterLabelComponent} from "../create-edit-rule/filter-label.component";
import {LabelPieceComponent} from "../label-piece/label-piece.component";
import { RouterTestingModule } from '@angular/router/testing';
import { of } from 'rxjs';
describe('Replication Component (inline template)', () => {
let mockRules: ReplicationRule[] = [
{
"id": 1,
"projects": [{
"project_id": 33,
"owner_id": 1,
"name": "aeas",
"deleted": 0,
"togglable": false,
"current_user_role_id": 0,
"repo_count": 0,
"metadata": {
"public": false,
"enable_content_trust": "",
"prevent_vul": "",
"severity": "",
"auto_scan": ""},
"owner_name": "",
"creation_time": null,
"update_time": null,
"has_project_admin_role": true,
"is_member": true,
"role_name": ""
}],
"targets": [{
"id": 1,
"endpoint": "https://10.117.4.151",
"name": "target_01",
"username": "admin",
"password": "",
"insecure": false,
"type": 0
}],
"name": "sync_01",
"description": "",
"filters": null,
"trigger": {"kind": "Manual", "schedule_param": null},
"trigger": {"type": "Manual", "trigger_settings": null},
"error_job_count": 2,
"replicate_deletion": false,
"replicate_existing_image_now": false,
"deletion": false,
"src_registry": {id: 3},
"src_namespaces": ["name1"],
"enabled": true,
"override": true
},
{
"id": 2,
"projects": [{
"project_id": 33,
"owner_id": 1,
"name": "aeas",
"deleted": 0,
"togglable": false,
"current_user_role_id": 0,
"repo_count": 0,
"metadata": {
"public": false,
"enable_content_trust": "",
"prevent_vul": "",
"severity": "",
"auto_scan": ""},
"owner_name": "",
"creation_time": null,
"update_time": null,
"has_project_admin_role": true,
"is_member": true,
"role_name": ""
}],
"targets": [{
"id": 1,
"endpoint": "https://10.117.4.151",
"name": "target_01",
"username": "admin",
"password": "",
"insecure": false,
"type": 0
}],
"name": "sync_02",
"description": "",
"filters": null,
"trigger": {"kind": "Manual", "schedule_param": null},
"trigger": {"type": "Manual", "trigger_settings": null},
"error_job_count": 2,
"replicate_deletion": false,
"replicate_existing_image_now": false,
"deletion": false,
"dest_registry": {id: 5},
"src_namespaces": ["name1"],
"enabled": true,
"override": true
}
];
let mockJobs: ReplicationJobItem[] = [
{
"id": 1,
"status": "error",
"repository": "library/nginx",
"policy_id": 1,
"operation": "transfer",
"update_time": new Date("2017-05-23 12:20:33"),
"tags": null
id: 1,
status: "stopped",
policy_id: 1,
trigger: "Manual",
total: 0,
failed: 0,
succeed: 0,
in_progress: 0,
stopped: 0
},
{
"id": 2,
"status": "finished",
"repository": "library/mysql",
"policy_id": 1,
"operation": "transfer",
"update_time": new Date("2017-05-27 12:20:33"),
"tags": null
id: 2,
status: "stopped",
policy_id: 1,
trigger: "Manual",
total: 1,
failed: 0,
succeed: 1,
in_progress: 0,
stopped: 0
},
{
"id": 3,
"status": "stopped",
"repository": "library/busybox",
"policy_id": 2,
"operation": "transfer",
"update_time": new Date("2017-04-23 12:20:33"),
"tags": null
id: 3,
status: "stopped",
policy_id: 2,
trigger: "Manual",
total: 1,
failed: 1,
succeed: 0,
in_progress: 0,
stopped: 0
}
];
let mockEndpoints: Endpoint[] = [
{
"id": 1,
"endpoint": "https://10.117.4.151",
"name": "target_01",
"username": "admin",
"password": "",
"credential": {
"access_key": "admin",
"access_secret": "",
"type": "basic"
},
"description": "test",
"insecure": false,
"type": 0
"name": "target_01",
"type": "Harbor",
"url": "https://10.117.4.151"
},
{
"id": 2,
"endpoint": "https://10.117.5.142",
"name": "target_02",
"username": "AAA",
"password": "",
"credential": {
"access_key": "admin",
"access_secret": "",
"type": "basic"
},
"description": "test",
"insecure": false,
"type": 0
"name": "target_02",
"type": "Harbor",
"url": "https://10.117.5.142"
},
];
// let mockProjects: Project[] = [
// { "project_id": 1,
// "owner_id": 0,
// "name": 'project_01',
// "creation_time": '',
// "deleted": 0,
// "owner_name": '',
// "togglable": false,
// "update_time": '',
// "current_user_role_id": 0,
// "repo_count": 0,
// "has_project_admin_role": false,
// "is_member": false,
// "role_name": '',
// "metadata": {
// "public": '',
// "enable_content_trust": '',
// "prevent_vul": '',
// "severity": '',
// "auto_scan": '',
// }
// }];
let mockJob: ReplicationJob = {
metadata: {xTotalCount: 3},
data: mockJobs
@ -198,7 +137,7 @@ describe('Replication Component (inline template)', () => {
let spyRules: jasmine.Spy;
let spyJobs: jasmine.Spy;
let spyEndpoint: jasmine.Spy;
let spyEndpoints: jasmine.Spy;
let deGrids: DebugElement[];
let deRules: DebugElement;
@ -208,15 +147,15 @@ describe('Replication Component (inline template)', () => {
let elJob: HTMLElement;
let config: IServiceConfig = {
replicationRuleEndpoint: '/api/policies/replication/testing',
replicationJobEndpoint: '/api/jobs/replication/testing'
replicationRuleEndpoint: '/api/policies/replication/testing'
};
beforeEach(async(() => {
TestBed.configureTestingModule({
imports: [
SharedModule,
NoopAnimationsModule
NoopAnimationsModule,
RouterTestingModule
],
declarations: [
ReplicationComponent,
@ -254,10 +193,10 @@ describe('Replication Component (inline template)', () => {
endpointService = fixtureCreate.debugElement.injector.get(EndpointService);
spyRules = spyOn(replicationService, 'getReplicationRules').and.returnValues(of(mockRules));
spyJobs = spyOn(replicationService, 'getJobs').and.returnValues(of(mockJob));
spyJobs = spyOn(replicationService, 'getExecutions').and.returnValues(of(mockJob));
spyEndpoint = spyOn(endpointService, 'getEndpoints').and.returnValues(of(mockEndpoints));
spyEndpoints = spyOn(endpointService, 'getEndpoints').and.returnValues(of(mockEndpoints));
fixture.detectChanges();
fixture.whenStable().then(() => {
fixture.detectChanges();
@ -325,7 +264,6 @@ describe('Replication Component (inline template)', () => {
fixture.detectChanges();
fixture.whenStable().then(() => {
fixture.detectChanges();
comp.doFilterJobStatus('finished');
let el: HTMLElement = deJobs.nativeElement;
fixture.detectChanges();
expect(el).toBeTruthy();
@ -337,8 +275,6 @@ describe('Replication Component (inline template)', () => {
fixture.detectChanges();
fixture.whenStable().then(() => {
fixture.detectChanges();
comp.doJobSearchByStartTime('2017-05-01');
comp.doJobSearchByEndTime('2015-05-25');
let el: HTMLElement = deJobs.nativeElement;
fixture.detectChanges();
expect(el).toBeTruthy();

312
src/portal/lib/src/replication/replication.component.ts
View File

@ -21,9 +21,8 @@ import {
EventEmitter
} from "@angular/core";
import { Comparator, State } from "../service/interface";
import { Subscription, forkJoin, timer, Observable} from "rxjs";
import { finalize, catchError, map } from "rxjs/operators";
import { Subscription, forkJoin, timer, Observable, throwError } from "rxjs";
import { TranslateService } from "@ngx-translate/core";
import { ListReplicationRuleComponent } from "../list-replication-rule/list-replication-rule.component";
@ -54,41 +53,28 @@ import {
import { ConfirmationMessage } from "../confirmation-dialog/confirmation-message";
import { ConfirmationDialogComponent } from "../confirmation-dialog/confirmation-dialog.component";
import { ConfirmationAcknowledgement } from "../confirmation-dialog/confirmation-state-message";
import {operateChanges, OperationState, OperateInfo} from "../operation/operate";
import {OperationService} from "../operation/operation.service";
import { catchError, map } from "rxjs/operators";
import { throwError as observableThrowError } from "rxjs";
import {
operateChanges,
OperationState,
OperateInfo
} from "../operation/operate";
import { OperationService } from "../operation/operation.service";
import { Router } from "@angular/router";
const ONE_HOUR_SECONDS: number = 3600;
const ONE_MINUTE_SECONDS: number = 60;
const ONE_DAY_SECONDS: number = 24 * ONE_HOUR_SECONDS;
const ruleStatus: { [key: string]: any } = [
{ key: "all", description: "REPLICATION.ALL_STATUS" },
{ key: "1", description: "REPLICATION.ENABLED" },
{ key: "0", description: "REPLICATION.DISABLED" }
];
const jobStatus: { [key: string]: any } = [
{ key: "all", description: "REPLICATION.ALL" },
{ key: "pending", description: "REPLICATION.PENDING" },
{ key: "running", description: "REPLICATION.RUNNING" },
{ key: "error", description: "REPLICATION.ERROR" },
{ key: "retrying", description: "REPLICATION.RETRYING" },
{ key: "stopped", description: "REPLICATION.STOPPED" },
{ key: "finished", description: "REPLICATION.FINISHED" },
{ key: "canceled", description: "REPLICATION.CANCELED" }
];
const optionalSearch: {} = {
0: "REPLICATION.ADVANCED",
1: "REPLICATION.SIMPLE"
};
export class SearchOption {
ruleId: number | string;
ruleName: string = "";
repoName: string = "";
trigger: string = "";
status: string = "";
startTime: string = "";
startTimestamp: string = "";
endTime: string = "";
endTimestamp: string = "";
page: number = 1;
pageSize: number = DEFAULT_PAGE_SIZE;
}
@ -115,15 +101,16 @@ export class ReplicationComponent implements OnInit, OnDestroy {
@Output() goToRegistry = new EventEmitter<any>();
search: SearchOption = new SearchOption();
isOpenFilterTag: boolean;
ruleStatus = ruleStatus;
currentRuleStatus: { key: string; description: string };
jobStatus = jobStatus;
currentJobStatus: { key: string; description: string };
currentTerm: string;
defaultFilter = "trigger";
changedRules: ReplicationRule[];
selectedRow: ReplicationJobItem[] = [];
rules: ReplicationRule[];
loading: boolean;
isStopOnGoing: boolean;
@ -131,25 +118,24 @@ export class ReplicationComponent implements OnInit, OnDestroy {
jobs: ReplicationJobItem[];
toggleJobSearchOption = optionalSearch;
currentJobSearchOption: number;
@ViewChild(ListReplicationRuleComponent)
listReplicationRule: ListReplicationRuleComponent;
@ViewChild(CreateEditRuleComponent)
createEditPolicyComponent: CreateEditRuleComponent;
@ViewChild("replicationConfirmDialog")
replicationConfirmDialog: ConfirmationDialogComponent;
@ViewChild("StopConfirmDialog")
StopConfirmDialog: ConfirmationDialogComponent;
creationTimeComparator: Comparator<ReplicationJob> = new CustomComparator<
ReplicationJob
>("creation_time", "date");
>("start_time", "date");
updateTimeComparator: Comparator<ReplicationJob> = new CustomComparator<
ReplicationJob
>("update_time", "date");
>("end_time", "date");
// Server driven pagination
currentPage: number = 1;
@ -160,10 +146,12 @@ export class ReplicationComponent implements OnInit, OnDestroy {
timerDelay: Subscription;
constructor(
private router: Router,
private errorHandler: ErrorHandler,
private replicationService: ReplicationService,
private operationService: OperationService,
private translateService: TranslateService) {}
private translateService: TranslateService
) {}
public get showPaginationIndex(): boolean {
return this.totalCount > 0;
@ -171,8 +159,6 @@ export class ReplicationComponent implements OnInit, OnDestroy {
ngOnInit() {
this.currentRuleStatus = this.ruleStatus[0];
this.currentJobStatus = this.jobStatus[0];
this.currentJobSearchOption = 0;
}
ngOnDestroy() {
@ -197,6 +183,11 @@ export class ReplicationComponent implements OnInit, OnDestroy {
this.goToRegistry.emit();
}
goToLink(exeId: number): void {
let linkUrl = ["harbor", "replications", exeId, "tasks"];
this.router.navigate(linkUrl);
}
// Server driven data loading
clrLoadJobs(state: State): void {
if (!state || !state.page || !this.search.ruleId) {
@ -213,42 +204,23 @@ export class ReplicationComponent implements OnInit, OnDestroy {
// Pagination
params.set("page", "" + pageNumber);
params.set("page_size", "" + this.pageSize);
// Search by status
if (this.search.status.trim()) {
params.set("status", this.search.status);
}
// Search by repository
if (this.search.repoName.trim()) {
params.set("repository", this.search.repoName);
}
// Search by timestamps
if (this.search.startTimestamp.trim()) {
params.set("start_time", this.search.startTimestamp);
}
if (this.search.endTimestamp.trim()) {
params.set("end_time", this.search.endTimestamp);
if (this.currentTerm && this.currentTerm !== "") {
params.set(this.defaultFilter, this.currentTerm);
}
this.jobsLoading = true;
// Do filtering and sorting
this.jobs = doFiltering<ReplicationJobItem>(this.jobs, state);
this.jobs = doSorting<ReplicationJobItem>(this.jobs, state);
this.jobsLoading = false;
this.replicationService.getJobs(this.search.ruleId, params)
.subscribe(response => {
this.replicationService.getExecutions(this.search.ruleId, params).subscribe(
response => {
this.totalCount = response.metadata.xTotalCount;
this.jobs = response.data;
if (!this.timerDelay) {
this.timerDelay = timer(10000, 10000).subscribe(() => {
let count: number = 0;
this.jobs.forEach(job => {
if (
job.status === "pending" ||
job.status === "running" ||
job.status === "retrying"
job.status === "InProgress"
) {
count++;
}
@ -261,18 +233,30 @@ export class ReplicationComponent implements OnInit, OnDestroy {
}
});
}
// Do filtering and sorting
this.jobs = doFiltering<ReplicationJobItem>(this.jobs, state);
this.jobs = doSorting<ReplicationJobItem>(this.jobs, state);
this.jobsLoading = false;
}, error => {
},
error => {
this.jobsLoading = false;
this.errorHandler.error(error);
});
}
);
}
public doSearchExecutions(terms: string): void {
if (!terms) {
return;
}
this.currentTerm = terms.trim();
// Trigger data loading and start from first page
this.jobsLoading = true;
this.currentPage = 1;
this.jobsLoading = true;
// Force reloading
this.loadFirstPage();
}
loadFirstPage(): void {
let st: State = this.currentState;
if (!st) {
@ -291,10 +275,6 @@ export class ReplicationComponent implements OnInit, OnDestroy {
if (rule && rule.id) {
this.hiddenJobList = false;
this.search.ruleId = rule.id || "";
this.search.repoName = "";
this.search.status = "";
this.currentJobSearchOption = 0;
this.currentJobStatus = { key: "all", description: "REPLICATION.ALL" };
this.loadFirstPage();
}
}
@ -322,7 +302,7 @@ export class ReplicationComponent implements OnInit, OnDestroy {
let rule: ReplicationRule = message.data;
if (rule) {
forkJoin(this.replicationOperate(rule)).subscribe((item) => {
forkJoin(this.replicationOperate(rule)).subscribe(item => {
this.selectOneRule(rule);
});
}
@ -332,30 +312,39 @@ export class ReplicationComponent implements OnInit, OnDestroy {
replicationOperate(rule: ReplicationRule): Observable<any> {
// init operation info
let operMessage = new OperateInfo();
operMessage.name = 'OPERATION.REPLICATION';
operMessage.name = "OPERATION.REPLICATION";
operMessage.data.id = rule.id;
operMessage.state = OperationState.progressing;
operMessage.data.name = rule.name;
this.operationService.publishInfo(operMessage);
return this.replicationService.replicateRule(+rule.id)
.pipe(map(response => {
this.translateService.get('BATCH.REPLICATE_SUCCESS')
.subscribe(res => operateChanges(operMessage, OperationState.success));
})
, catchError(error => {
if (error && error.status === 412) {
return forkJoin(this.translateService.get('BATCH.REPLICATE_FAILURE'),
this.translateService.get('REPLICATION.REPLICATE_SUMMARY_FAILURE'))
.pipe(map(function (res) {
operateChanges(operMessage, OperationState.failure, res[1]);
}));
} else {
return this.translateService.get('BATCH.REPLICATE_FAILURE').pipe(map(res => {
return this.replicationService.replicateRule(+rule.id).pipe(
map(response => {
this.translateService
.get("BATCH.REPLICATE_SUCCESS")
.subscribe(res =>
operateChanges(operMessage, OperationState.success)
);
}),
catchError(error => {
if (error && error.status === 412) {
return forkJoin(
this.translateService.get("BATCH.REPLICATE_FAILURE"),
this.translateService.get("REPLICATION.REPLICATE_SUMMARY_FAILURE")
).pipe(
map(function(res) {
operateChanges(operMessage, OperationState.failure, res[1]);
})
);
} else {
return this.translateService.get("BATCH.REPLICATE_FAILURE").pipe(
map(res => {
operateChanges(operMessage, OperationState.failure, res);
}));
})
);
}
}));
})
);
}
customRedirect(rule: ReplicationRule) {
@ -367,21 +356,17 @@ export class ReplicationComponent implements OnInit, OnDestroy {
this.listReplicationRule.retrieveRules(ruleName);
}
doFilterJobStatus($event: any) {
if ($event && $event.target && $event.target["value"]) {
let status = $event.target["value"];
this.currentJobStatus = this.jobStatus.find((r: any) => r.key === status);
if (this.currentJobStatus.key === "all") {
status = "";
}
this.search.status = status;
this.doSearchJobs(this.search.repoName);
}
doFilterJob($event: any): void {
this.defaultFilter = $event["target"].value;
this.doSearchJobs(this.currentTerm);
}
doSearchJobs(repoName: string) {
this.search.repoName = repoName;
doSearchJobs(terms: string) {
if (!terms) {
return;
}
this.currentTerm = terms.trim();
this.currentPage = 1;
this.loadFirstPage();
}
@ -391,17 +376,68 @@ export class ReplicationComponent implements OnInit, OnDestroy {
this.hiddenJobList = true;
}
stopJobs() {
if (this.jobs && this.jobs.length) {
this.isStopOnGoing = true;
this.replicationService.stopJobs(this.jobs[0].policy_id)
.subscribe(res => {
this.refreshJobs();
this.isStopOnGoing = false;
}, error => this.errorHandler.error(error));
openStopExecutionsDialog(targets: ReplicationJobItem[]) {
let ExecutionId = targets.map(robot => robot.id).join(",");
let StopExecutionsMessage = new ConfirmationMessage(
"REPLICATION.STOP_TITLE",
"REPLICATION.STOP_SUMMARY",
ExecutionId,
targets,
ConfirmationTargets.STOP_EXECUTIONS,
ConfirmationButtons.STOP_CANCEL
);
this.StopConfirmDialog.open(StopExecutionsMessage);
}
confirmStop(message: ConfirmationAcknowledgement) {
if (
message &&
message.state === ConfirmationState.CONFIRMED &&
message.source === ConfirmationTargets.STOP_EXECUTIONS
) {
this.StopExecutions(message.data);
}
}
StopExecutions(targets: ReplicationJobItem[]): void {
if (targets && targets.length < 1) {
return;
}
this.isStopOnGoing = true;
if (this.jobs && this.jobs.length) {
let ExecutionsStop$ = targets.map(target => this.StopOperate(target));
forkJoin(ExecutionsStop$)
.pipe(
catchError(err => throwError(err)),
finalize(() => {
this.refreshJobs();
this.isStopOnGoing = false;
this.selectedRow = [];
})
)
.subscribe(() => {});
}
}
StopOperate(targets: ReplicationJobItem): any {
let operMessage = new OperateInfo();
operMessage.name = "OPERATION.STOP_EXECUTIONS";
operMessage.data.id = targets.id;
operMessage.state = OperationState.progressing;
operMessage.data.name = targets.id;
this.operationService.publishInfo(operMessage);
return this.replicationService
.stopJobs(targets.id)
.pipe(
map(
() => operateChanges(operMessage, OperationState.success),
err => operateChanges(operMessage, OperationState.failure, err)
)
);
}
reloadRules(isReady: boolean) {
if (isReady) {
this.search.ruleName = "";
@ -414,14 +450,7 @@ export class ReplicationComponent implements OnInit, OnDestroy {
}
refreshJobs() {
this.currentJobStatus = this.jobStatus[0];
this.search.startTime = " ";
this.search.endTime = " ";
this.search.repoName = "";
this.search.startTimestamp = "";
this.search.endTimestamp = "";
this.search.status = "";
this.currentTerm = "";
this.currentPage = 1;
let st: State = {
@ -434,23 +463,36 @@ export class ReplicationComponent implements OnInit, OnDestroy {
this.clrLoadJobs(st);
}
toggleSearchJobOptionalName(option: number) {
option === 1
? (this.currentJobSearchOption = 0)
: (this.currentJobSearchOption = 1);
openFilter(isOpen: boolean): void {
if (isOpen) {
this.isOpenFilterTag = true;
} else {
this.isOpenFilterTag = false;
}
}
getDuration(j: ReplicationJobItem) {
if (!j) {
return;
}
if (j.status === "Failed") {
return "-";
}
let start_time = new Date(j.start_time).getTime();
let end_time = new Date(j.end_time).getTime();
let timesDiff = end_time - start_time;
let timesDiffSeconds = timesDiff / 1000;
let minutes = Math.floor(((timesDiffSeconds % ONE_DAY_SECONDS) % ONE_HOUR_SECONDS) / ONE_MINUTE_SECONDS);
let seconds = Math.floor(timesDiffSeconds % ONE_MINUTE_SECONDS);
if (minutes > 0) {
return minutes + "m" + seconds + "s";
}
doJobSearchByStartTime(fromTimestamp: string) {
this.search.startTimestamp = fromTimestamp;
this.loadFirstPage();
}
if (seconds > 0) {
return seconds + "s";
}
doJobSearchByEndTime(toTimestamp: string) {
this.search.endTimestamp = toTimestamp;
this.loadFirstPage();
}
viewLog(jobId: number | string): string {
return this.replicationService.getJobBaseUrl() + "/" + jobId + "/log";
if (seconds <= 0 && timesDiff > 0) {
return timesDiff + 'ms';
}
}
}

11
src/portal/lib/src/service.config.ts
View File

@ -2,6 +2,7 @@ import { InjectionToken } from '@angular/core';
export let SERVICE_CONFIG = new InjectionToken("service.config");
export interface IServiceConfig {
baseEndpoint?: string;
/**
* The base endpoint of service used to retrieve the system configuration information.
* The configurations may include but not limit:
@ -66,16 +67,6 @@ export interface IServiceConfig {
*/
replicationRuleEndpoint?: string;
/**
* The base endpoint of the service used to handle the replication jobs.
*
*
* * {string}
* @memberOf IServiceConfig
*/
replicationJobEndpoint?: string;
/**
* The base endpoint of the service used to handle vulnerability scanning.
*

21
src/portal/lib/src/service/endpoint.service.ts
View File

@ -12,6 +12,7 @@ import { RequestQueryParams } from "./RequestQueryParams";
import { Endpoint, ReplicationRule } from "./interface";
import { catchError, map } from "rxjs/operators";
/**
* Define the service methods to handle the endpoint related things.
*
@ -58,6 +59,17 @@ export abstract class EndpointService {
*
* @memberOf EndpointService
*/
abstract getAdapters(): Observable<any>;
/**
* Create new endpoint.
*
* @abstract
* ** deprecated param {Adapter} adapter
* returns {(Observable<any> | any)}
*
* @memberOf EndpointService
*/
abstract createEndpoint(
endpoint: Endpoint
): Observable<any>;
@ -133,7 +145,7 @@ export class EndpointDefaultService extends EndpointService {
super();
this._endpointUrl = config.targetBaseEndpoint
? config.targetBaseEndpoint
: "/api/targets";
: "/api/registries";
}
public getEndpoints(
@ -166,6 +178,13 @@ export class EndpointDefaultService extends EndpointService {
, catchError(error => observableThrowError(error)));
}
public getAdapters(): Observable<any> {
return this.http
.get(`/api/replication/adapters`)
.pipe(map(response => response.json())
, catchError(error => observableThrowError(error)));
}
public createEndpoint(
endpoint: Endpoint
): Observable<any> {

78
src/portal/lib/src/service/interface.ts
View File

@ -75,13 +75,22 @@ export interface Tag extends Base {
* extends {Base}
*/
export interface Endpoint extends Base {
endpoint: string;
name: string;
username?: string;
password?: string;
credential: {
access_key?: string,
access_secret?: string,
type: string;
};
description: string;
insecure: boolean;
type: number;
[key: string]: any;
name: string;
type: string;
url: string;
}
export interface Filter {
type: string;
style: string;
values ?: string[];
}
/**
@ -97,33 +106,35 @@ export interface ReplicationRule extends Base {
id?: number;
name: string;
description: string;
projects: Project[];
targets: Endpoint[];
trigger: Trigger;
filters: Filter[];
replicate_existing_image_now?: boolean;
replicate_deletion?: boolean;
deletion?: boolean;
src_registry?: any;
dest_registry?: any;
src_namespaces: string [];
dest_namespace?: string;
enabled: boolean;
override: boolean;
}
export class Filter {
kind: string;
pattern: string;
constructor(kind: string, pattern: string) {
this.kind = kind;
this.pattern = pattern;
type: string;
value?: string;
constructor(type: string) {
this.type = type;
}
}
export class Trigger {
kind: string;
schedule_param:
type: string;
trigger_settings:
| any
| {
[key: string]: any | any[];
};
constructor(kind: string, param: any | { [key: string]: any | any[] }) {
this.kind = kind;
this.schedule_param = param;
constructor(type: string, param: any | { [key: string]: any | any[] }) {
this.type = type;
this.trigger_settings = param;
}
}
@ -146,13 +157,34 @@ export interface ReplicationJob {
*/
export interface ReplicationJobItem extends Base {
[key: string]: any | any[];
id: number;
status: string;
repository: string;
policy_id: number;
operation: string;
tags: string;
trigger: string;
total: number;
failed: number;
succeed: number;
in_progress: number;
stopped: number;
}
/**
* Interface for replication tasks item.
*
**
* interface ReplicationTasks
*/
export interface ReplicationTasks extends Base {
[key: string]: any | any[];
operation: string;
id: number;
execution_id: number;
resource_type: string;
src_resource: string;
dst_resource: string;
job_id: number;
status: string;
}
/**
* Interface for storing metadata of response.
*

4
src/portal/lib/src/service/job-log.service.spec.ts
View File

@ -6,7 +6,7 @@ import { SERVICE_CONFIG, IServiceConfig } from '../service.config';
describe('JobLogService', () => {
const mockConfig: IServiceConfig = {
replicationJobEndpoint: "/api/jobs/replication/testing",
replicationBaseEndpoint: "/api/replication/testing",
scanJobEndpoint: "/api/jobs/scan/testing"
};
@ -33,7 +33,7 @@ describe('JobLogService', () => {
it('should be initialized', inject([JobLogDefaultService], (service: JobLogService) => {
expect(service).toBeTruthy();
expect(config.replicationJobEndpoint).toEqual("/api/jobs/replication/testing");
expect(config.replicationBaseEndpoint).toEqual("/api/replication/testing");
expect(config.scanJobEndpoint).toEqual("/api/jobs/scan/testing");
}));
});

6
src/portal/lib/src/service/job-log.service.ts
View File

@ -47,9 +47,9 @@ export class JobLogDefaultService extends JobLogService {
@Inject(SERVICE_CONFIG) config: IServiceConfig
) {
super();
this._replicationJobBaseUrl = config.replicationJobEndpoint
? config.replicationJobEndpoint
: "/api/jobs/replication";
this._replicationJobBaseUrl = config.replicationBaseEndpoint
? config.replicationBaseEndpoint
: "/api/replication";
this._scanningJobBaseUrl = config.scanJobEndpoint
? config.scanJobEndpoint
: "/api/jobs/scan";

4
src/portal/lib/src/service/replication.service.spec.ts
View File

@ -7,7 +7,7 @@ import { SERVICE_CONFIG, IServiceConfig } from '../service.config';
describe('ReplicationService', () => {
const mockConfig: IServiceConfig = {
replicationRuleEndpoint: "/api/policies/replication/testing",
replicationJobEndpoint: "/api/jobs/replication/testing"
replicationBaseEndpoint: "/api/replication/testing"
};
let config: IServiceConfig;
@ -38,6 +38,6 @@ describe('ReplicationService', () => {
it('should inject the right config', () => {
expect(config).toBeTruthy();
expect(config.replicationRuleEndpoint).toEqual("/api/policies/replication/testing");
expect(config.replicationJobEndpoint).toEqual("/api/jobs/replication/testing");
expect(config.replicationBaseEndpoint).toEqual("/api/replication/testing");
});
});

149
src/portal/lib/src/service/replication.service.ts
View File

@ -9,12 +9,12 @@ import {
import {
ReplicationJob,
ReplicationRule,
ReplicationJobItem
ReplicationJobItem,
ReplicationTasks
} from "./interface";
import { RequestQueryParams } from "./RequestQueryParams";
import { map, catchError } from "rxjs/operators";
import { Observable, throwError as observableThrowError } from "rxjs";
/**
* Define the service methods to handle the replication (rule and job) related things.
*
@ -57,6 +57,19 @@ export abstract class ReplicationService {
ruleId: number | string
): Observable<ReplicationRule>;
/**
* Get the specified replication task.
*
* @abstract
* returns {(Observable<ReplicationRule>)}
*
* @memberOf ReplicationService
*/
abstract getReplicationTasks(
executionId: number | string,
queryParams?: RequestQueryParams
): Observable<ReplicationTasks>;
/**
* Create new replication rule.
*
@ -122,11 +135,14 @@ export abstract class ReplicationService {
*/
abstract disableReplicationRule(
ruleId: number | string
): Observable<any> ;
): Observable<any>;
abstract replicateRule(
ruleId: number | string
): Observable<any> ;
): Observable<any>;
abstract getRegistryInfo(id: number): Observable<any>;
/**
* Get the jobs for the specified replication rule.
@ -144,11 +160,24 @@ export abstract class ReplicationService {
*
* @memberOf ReplicationService
*/
abstract getJobs(
abstract getExecutions(
ruleId: number | string,
queryParams?: RequestQueryParams
): Observable<ReplicationJob>;
/**
* Get the specified execution.
*
* @abstract
* ** deprecated param {(number | string)} endpointId
* returns {(Observable<ReplicationJob> | ReplicationJob)}
*
* @memberOf ReplicationService
*/
abstract getExecutionById(
executionId: number | string
): Observable<ReplicationJob>;
/**
* Get the log of the specified job.
*
@ -178,8 +207,8 @@ export abstract class ReplicationService {
@Injectable()
export class ReplicationDefaultService extends ReplicationService {
_ruleBaseUrl: string;
_jobBaseUrl: string;
_replicateUrl: string;
_baseUrl: string;
constructor(
private http: Http,
@ -188,13 +217,11 @@ export class ReplicationDefaultService extends ReplicationService {
super();
this._ruleBaseUrl = config.replicationRuleEndpoint
? config.replicationRuleEndpoint
: "/api/policies/replication";
this._jobBaseUrl = config.replicationJobEndpoint
? config.replicationJobEndpoint
: "/api/jobs/replication";
: "/api/replication/policies";
this._replicateUrl = config.replicationBaseEndpoint
? config.replicationBaseEndpoint
: "/api/replications";
: "/api/replication";
this._baseUrl = config.baseEndpoint ? config.baseEndpoint : "/api";
}
// Private methods
@ -205,12 +232,20 @@ export class ReplicationDefaultService extends ReplicationService {
rule != null &&
rule.name !== undefined &&
rule.name.trim() !== "" &&
rule.targets.length !== 0
(!!rule.dest_registry || !!rule.src_registry)
);
}
public getRegistryInfo(id): Observable<any> {
let requestUrl: string = `${this._baseUrl}/registries/${id}/info`;
return this.http
.get(requestUrl)
.pipe(map(response => response.json())
, catchError(error => observableThrowError(error)));
}
public getJobBaseUrl() {
return this._jobBaseUrl;
return this._replicateUrl;
}
public getReplicationRules(
@ -234,7 +269,7 @@ export class ReplicationDefaultService extends ReplicationService {
return this.http
.get(this._ruleBaseUrl, buildHttpRequestOptions(queryParams))
.pipe(map(response => response.json() as ReplicationRule[])
, catchError(error => observableThrowError(error)));
, catchError(error => observableThrowError(error)));
}
public getReplicationRule(
@ -248,7 +283,22 @@ export class ReplicationDefaultService extends ReplicationService {
return this.http
.get(url, HTTP_GET_OPTIONS)
.pipe(map(response => response.json() as ReplicationRule)
, catchError(error => observableThrowError(error)));
, catchError(error => observableThrowError(error)));
}
public getReplicationTasks(
executionId: number | string,
queryParams?: RequestQueryParams
): Observable<ReplicationTasks> {
if (!executionId) {
return observableThrowError("Bad argument");
}
let url: string = `${this._replicateUrl}/executions/${executionId}/tasks`;
return this.http
.get(url,
queryParams ? buildHttpRequestOptions(queryParams) : HTTP_GET_OPTIONS)
.pipe(map(response => response.json() as ReplicationTasks)
, catchError(error => observableThrowError(error)));
}
public createReplicationRule(
@ -265,7 +315,7 @@ export class ReplicationDefaultService extends ReplicationService {
HTTP_JSON_OPTIONS
)
.pipe(map(response => response)
, catchError(error => observableThrowError(error)));
, catchError(error => observableThrowError(error)));
}
public updateReplicationRule(
@ -280,7 +330,7 @@ export class ReplicationDefaultService extends ReplicationService {
return this.http
.put(url, JSON.stringify(rep), HTTP_JSON_OPTIONS)
.pipe(map(response => response)
, catchError(error => observableThrowError(error)));
, catchError(error => observableThrowError(error)));
}
public deleteReplicationRule(
@ -294,7 +344,7 @@ export class ReplicationDefaultService extends ReplicationService {
return this.http
.delete(url, HTTP_JSON_OPTIONS)
.pipe(map(response => response)
, catchError(error => observableThrowError(error)));
, catchError(error => observableThrowError(error)));
}
public replicateRule(
@ -304,11 +354,11 @@ export class ReplicationDefaultService extends ReplicationService {
return observableThrowError("Bad argument");
}
let url: string = `${this._replicateUrl}`;
let url: string = `${this._replicateUrl}/executions`;
return this.http
.post(url, { policy_id: ruleId }, HTTP_JSON_OPTIONS)
.pipe(map(response => response)
, catchError(error => observableThrowError(error)));
, catchError(error => observableThrowError(error)));
}
public enableReplicationRule(
@ -323,7 +373,7 @@ export class ReplicationDefaultService extends ReplicationService {
return this.http
.put(url, { enabled: enablement }, HTTP_JSON_OPTIONS)
.pipe(map(response => response)
, catchError(error => observableThrowError(error)));
, catchError(error => observableThrowError(error)));
}
public disableReplicationRule(
@ -337,10 +387,10 @@ export class ReplicationDefaultService extends ReplicationService {
return this.http
.put(url, { enabled: 0 }, HTTP_JSON_OPTIONS)
.pipe(map(response => response)
, catchError(error => observableThrowError(error)));
, catchError(error => observableThrowError(error)));
}
public getJobs(
public getExecutions(
ruleId: number | string,
queryParams?: RequestQueryParams
): Observable<ReplicationJob> {
@ -351,10 +401,45 @@ export class ReplicationDefaultService extends ReplicationService {
if (!queryParams) {
queryParams = new RequestQueryParams();
}
let url: string = `${this._replicateUrl}/executions`;
queryParams.set("policy_id", "" + ruleId);
return this.http
.get(this._jobBaseUrl, buildHttpRequestOptions(queryParams))
.get(url, buildHttpRequestOptions(queryParams))
.pipe(map(response => {
let result: ReplicationJob = {
metadata: {
xTotalCount: 0
},
data: []
};
if (response && response.headers) {
let xHeader: string = response.headers.get("X-Total-Count");
if (xHeader) {
result.metadata.xTotalCount = parseInt(xHeader, 0);
}
}
result.data = response.json() as ReplicationJobItem[];
if (result.metadata.xTotalCount === 0) {
if (result.data && result.data.length > 0) {
result.metadata.xTotalCount = result.data.length;
}
}
return result;
})
, catchError(error => observableThrowError(error)));
}
public getExecutionById(
executionId: number | string
): Observable<ReplicationJob> {
if (!executionId || executionId <= 0) {
return observableThrowError("Bad request argument.");
}
let requestUrl: string = `${this._replicateUrl}/executions/${executionId}`;
return this.http
.get(requestUrl, HTTP_GET_OPTIONS)
.pipe(map(response => {
let result: ReplicationJob = {
metadata: {
@ -388,23 +473,27 @@ export class ReplicationDefaultService extends ReplicationService {
return observableThrowError("Bad argument");
}
let logUrl = `${this._jobBaseUrl}/${jobId}/log`;
let logUrl = `${this._replicateUrl}/${jobId}/log`;
return this.http
.get(logUrl, HTTP_GET_OPTIONS)
.pipe(map(response => response.text())
, catchError(error => observableThrowError(error)));
, catchError(error => observableThrowError(error)));
}
public stopJobs(
jobId: number | string
): Observable<any> {
if (!jobId || jobId <= 0) {
return observableThrowError("Bad request argument.");
}
let requestUrl: string = `${this._replicateUrl}/executions/${jobId}`;
return this.http
.put(
this._jobBaseUrl,
JSON.stringify({ policy_id: jobId, status: "stop" }),
requestUrl,
HTTP_JSON_OPTIONS
)
.pipe(map(response => response)
, catchError(error => observableThrowError(error)));
, catchError(error => observableThrowError(error)));
}
}

5
src/portal/lib/src/shared/shared.const.ts
View File

@ -43,7 +43,8 @@ export const enum ConfirmationTargets {
CONFIG_ROUTE,
CONFIG_TAB,
HELM_CHART,
HELM_CHART_VERSION
HELM_CHART_VERSION,
STOP_EXECUTIONS
}
export const enum ActionType {
@ -69,7 +70,7 @@ export const enum ConfirmationState {
}
export const enum ConfirmationButtons {
CONFIRM_CANCEL, YES_NO, DELETE_CANCEL, CLOSE, REPLICATE_CANCEL
CONFIRM_CANCEL, YES_NO, DELETE_CANCEL, CLOSE, REPLICATE_CANCEL, STOP_CANCEL
}
export const LabelColor = [

5
src/portal/lib/src/shared/shared.utils.ts
View File

@ -23,6 +23,11 @@ export const errorHandler = function (error: any): string {
if (!error) {
return "UNKNOWN_ERROR";
}
if (error && error._body) {
return error._body;
}
if (!(error.statusCode || error.status)) {
// treat as string message
return '' + error;

13
src/portal/src/app/harbor-routing.module.ts
View File

@ -33,8 +33,9 @@ import { ResetPasswordComponent } from './account/password-setting/reset-passwor
import { GroupComponent } from './group/group.component';
import { TotalReplicationPageComponent } from './replication/total-replication/total-replication-page.component';
import { ReplicationTasksPageComponent } from './replication/replication-tasks-page/replication-tasks-page.component';
import { DestinationPageComponent } from './replication/destination/destination-page.component';
import { ReplicationPageComponent } from './replication/replication-page.component';
import { AuditLogComponent } from './log/audit-log.component';
import { LogPageComponent } from './log/log-page.component';
@ -109,6 +110,12 @@ const harborRoutes: Routes = [
canActivate: [SystemAdminGuard],
canActivateChild: [SystemAdminGuard],
},
{
path: 'replications/:id/:tasks',
component: ReplicationTasksPageComponent,
canActivate: [SystemAdminGuard],
canActivateChild: [SystemAdminGuard],
},
{
path: 'tags/:id/:repo',
component: TagRepositoryComponent,
@ -170,10 +177,6 @@ const harborRoutes: Routes = [
path: 'repositories/:repo/tags',
component: TagRepositoryComponent,
},
{
path: 'replications',
component: ReplicationPageComponent,
},
{
path: 'members',
component: MemberComponent

3
src/portal/src/app/project/project-detail/project-detail.component.html
View File

@ -13,9 +13,6 @@
<li class="nav-item" *ngIf="hasMemberListPermission">
<a class="nav-link" routerLink="members" routerLinkActive="active">{{'PROJECT_DETAIL.USERS' | translate}}</a>
</li>
<li class="nav-item" *ngIf="hasReplicationListPermission">
<a class="nav-link" routerLink="replications" routerLinkActive="active">{{'PROJECT_DETAIL.REPLICATION' | translate}}</a>
</li>
<li class="nav-item" *ngIf="(hasLabelListPermission && hasLabelCreatePermission) && !withAdmiral">
<a class="nav-link" routerLink="labels" routerLinkActive="active">{{'PROJECT_DETAIL.LABELS' | translate}}</a>
</li>

Some files were not shown because too many files have changed in this diff Show More