From e7fafd19419bc74fd727f658ee464f407aec40e1 Mon Sep 17 00:00:00 2001 From: guanxiatao Date: Wed, 7 Aug 2019 20:30:26 +0800 Subject: [PATCH 1/3] webhook policy, job, event support Signed-off-by: guanxiatao --- docs/swagger.yaml | 468 +++++++++++-- src/common/config/metadata/metadatalist.go | 1 + src/common/const.go | 2 + src/common/dao/base.go | 6 +- .../dao/notification/notification_job.go | 122 ++++ .../dao/notification/notification_job_test.go | 263 ++++++++ .../dao/notification/notification_policy.go | 69 ++ .../notification/notification_policy_test.go | 291 ++++++++ .../dao/notification/notification_test.go | 13 + src/common/http/client.go | 31 + src/common/http/client_test.go | 14 + src/common/models/base.go | 2 + src/common/models/hook_notification.go | 111 +++ src/common/models/hook_notification_test.go | 114 ++++ src/common/models/project.go | 10 +- src/common/rbac/const.go | 1 + src/common/rbac/project/util.go | 6 + src/common/rbac/project/visitor_role.go | 8 + src/core/api/chart_repository.go | 1 + src/core/api/harborapi_test.go | 10 + src/core/api/notification_job.go | 108 +++ src/core/api/notification_job_test.go | 107 +++ src/core/api/notification_policy.go | 384 +++++++++++ src/core/api/notification_policy_test.go | 637 ++++++++++++++++++ src/core/api/repository.go | 19 + src/core/api/systeminfo.go | 4 +- src/core/config/config.go | 5 + src/core/config/config_test.go | 1 + src/core/main.go | 8 +- src/core/notifier/event/event.go | 154 +++++ src/core/notifier/event/event_test.go | 212 ++++++ .../handler/notification/http_handler.go | 59 ++ .../handler/notification/http_handler_test.go | 97 +++ .../handler/notification/image_handler.go | 18 + .../notification/image_handler_test.go | 193 ++++++ .../handler/notification/processor.go | 174 +++++ src/core/notifier/model/event.go | 61 ++ src/core/notifier/model/topic.go | 26 + src/core/notifier/topic/topics.go | 28 + src/core/notifier/topics.go | 11 - src/core/router.go | 12 +- .../service/notifications/jobs/handler.go | 15 + .../service/notifications/registry/handler.go | 39 ++ src/pkg/notification/hook/hook.go | 85 +++ src/pkg/notification/job/manager.go | 20 + src/pkg/notification/job/manager/manager.go | 55 ++ .../notification/job/manager/manager_test.go | 22 + src/pkg/notification/model/const.go | 16 + src/pkg/notification/notification.go | 63 ++ src/pkg/notification/policy/manager.go | 25 + .../notification/policy/manager/manager.go | 159 +++++ .../policy/manager/manager_test.go | 22 + 52 files changed, 4307 insertions(+), 75 deletions(-) mode change 100644 => 100755 src/common/const.go create mode 100755 src/common/dao/notification/notification_job.go create mode 100644 src/common/dao/notification/notification_job_test.go create mode 100755 src/common/dao/notification/notification_policy.go create mode 100644 src/common/dao/notification/notification_policy_test.go create mode 100644 src/common/dao/notification/notification_test.go create mode 100644 src/common/http/client_test.go create mode 100755 src/common/models/hook_notification.go create mode 100644 src/common/models/hook_notification_test.go mode change 100644 => 100755 src/common/rbac/const.go mode change 100644 => 100755 src/common/rbac/project/visitor_role.go mode change 100644 => 100755 src/core/api/chart_repository.go create mode 100755 src/core/api/notification_job.go create mode 100644 src/core/api/notification_job_test.go create mode 100755 src/core/api/notification_policy.go create mode 100644 src/core/api/notification_policy_test.go mode change 100644 => 100755 src/core/api/repository.go mode change 100644 => 100755 src/core/config/config.go mode change 100644 => 100755 src/core/main.go create mode 100644 src/core/notifier/event/event.go create mode 100644 src/core/notifier/event/event_test.go create mode 100755 src/core/notifier/handler/notification/http_handler.go create mode 100644 src/core/notifier/handler/notification/http_handler_test.go create mode 100644 src/core/notifier/handler/notification/image_handler.go create mode 100644 src/core/notifier/handler/notification/image_handler_test.go create mode 100644 src/core/notifier/handler/notification/processor.go create mode 100755 src/core/notifier/model/event.go create mode 100644 src/core/notifier/model/topic.go create mode 100644 src/core/notifier/topic/topics.go delete mode 100644 src/core/notifier/topics.go mode change 100644 => 100755 src/core/router.go mode change 100644 => 100755 src/core/service/notifications/jobs/handler.go mode change 100644 => 100755 src/core/service/notifications/registry/handler.go create mode 100755 src/pkg/notification/hook/hook.go create mode 100755 src/pkg/notification/job/manager.go create mode 100755 src/pkg/notification/job/manager/manager.go create mode 100644 src/pkg/notification/job/manager/manager_test.go create mode 100644 src/pkg/notification/model/const.go create mode 100755 src/pkg/notification/notification.go create mode 100755 src/pkg/notification/policy/manager.go create mode 100755 src/pkg/notification/policy/manager/manager.go create mode 100644 src/pkg/notification/policy/manager/manager_test.go diff --git a/docs/swagger.yaml b/docs/swagger.yaml index b8e16e8ed..01071390b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -3565,7 +3565,7 @@ paths: name: whitelist description: The whitelist with new content schema: - $ref: "#/definitions/CVEWhitelist" + $ref: "#/definitions/CVEWhitelist" responses: '200': description: Successfully updated the CVE whitelist. @@ -3628,60 +3628,319 @@ paths: '500': description: Unexpected internal errors. '/quotas/{id}': - get: - summary: Get the specified quota - description: Get the specified quota - tags: + get: + summary: Get the specified quota + description: Get the specified quota + tags: - quota - parameters: - - name: id - in: path - type: integer - required: true - description: Quota ID - responses: - '200': - description: Successfully retrieved the quota. - schema: - $ref: '#/definitions/Quota' - '401': - description: User need to log in first. - '403': - description: User does not have permission to call this API - '404': - description: Quota does not exist. - '500': - description: Unexpected internal errors. - put: - summary: Update the specified quota - description: Update hard limits of the specified quota - tags: - - quota - parameters: - - name: id - in: path - type: integer - required: true - description: Quota ID - - name: hard - in: body - required: true - description: The new hard limits for the quota - schema: - $ref: '#/definitions/QuotaUpdateReq' - responses: - '200': - description: Updated quota hard limits successfully. - '400': - description: Illegal format of quota update request. - '401': - description: User need to log in first. - '403': - description: User does not have permission to the quota. - '404': - description: Quota ID does not exist. - '500': - description: Unexpected internal errors. + parameters: + - name: id + in: path + type: integer + required: true + description: Quota ID + responses: + '200': + description: Successfully retrieved the quota. + schema: + $ref: '#/definitions/Quota' + '401': + description: User need to log in first. + '403': + description: User does not have permission to call this API + '404': + description: Quota does not exist. + '500': + description: Unexpected internal errors. + put: + summary: Update the specified quota + description: Update hard limits of the specified quota + tags: + - quota + parameters: + - name: id + in: path + type: integer + required: true + description: Quota ID + - name: hard + in: body + required: true + description: The new hard limits for the quota + schema: + $ref: '#/definitions/QuotaUpdateReq' + responses: + '200': + description: Updated quota hard limits successfully. + '400': + description: Illegal format of quota update request. + '401': + description: User need to log in first. + '403': + description: User does not have permission to the quota. + '404': + description: Quota ID does not exist. + '500': + description: Unexpected internal errors. + '/projects/{project_id}/webhook/policies': + get: + sumary: List project webhook policies. + description: | + This endpoint returns webhook policies of a project. + parameters: + - name: project_id + in: path + type: integer + format: int64 + required: true + description: Relevant project ID. + tags: + - Products + responses: + '200': + description: List project webhook policies successfully. + schema: + type: array + items: + $ref: '#/definitions/WebhookPolicy' + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '403': + description: User have no permission to list webhook policies of the project. + '500': + description: Unexpected internal errors. + post: + sumary: Create project webhook policy. + description: | + This endpoint create a webhook policy if the project does not have one. + parameters: + - name: project_id + in: path + type: integer + format: int64 + required: true + description: Relevant project ID + - name: policy + in: body + description: Properties "targets" and "event_types" needed. + required: true + schema: + $ref: '#/definitions/WebhookPolicy' + tags: + - Products + responses: + '201': + description: Project webhook policy create successfully. + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '403': + description: User have no permission to create webhook policy of the project. + '500': + description: Unexpected internal errors. + '/projects/{project_id}/webhook/policies/{policy_id}': + get: + summary: Get project webhook policy + description: | + This endpoint returns specified webhook policy of a project. + parameters: + - name: project_id + in: path + description: Relevant project ID. + required: true + type: integer + format: int64 + - name: policy_id + in: path + description: The id of webhook policy. + required: true + type: int64 + format: int64 + tags: + - Products + responses: + '200': + description: Get webhook policy successfully. + schema: + $ref: '#/definitions/WebhookPolicy' + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '403': + description: User have no permission to get webhook policy of the project. + '404': + description: Webhook policy ID does not exist. + '500': + description: Internal server errors. + put: + summary: Update webhook policy of a project. + description: | + This endpoint is aimed to update the webhook policy of a project. + parameters: + - name: project_id + in: path + description: Relevant project ID. + required: true + type: integer + format: int64 + - name: policy_id + in: path + description: The id of webhook policy. + required: true + type: int64 + format: int64 + - name: policy + in: body + description: All properties needed except "id", "project_id", "creation_time", "update_time". + required: true + schema: + $ref: '#/definitions/WebhookPolicy' + tags: + - Products + responses: + '200': + description: Update webhook policy successfully. + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '403': + description: User have no permission to update webhook policy of the project. + '404': + description: Webhook policy ID does not exist. + '500': + description: Internal server errors. + delete: + summary: Delete webhook policy of a project + description: | + This endpoint is aimed to delete webhookpolicy of a project. + parameters: + - name: project_id + in: path + description: Relevant project ID. + required: true + type: integer + format: int64 + - name: policy_id + in: path + description: The id of webhook policy. + required: true + type: int64 + format: int64 + tags: + - Products + responses: + '200': + description: Delete webhook policy successfully. + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '403': + description: User have no permission to delete webhook policy of the project. + '404': + description: Webhook policy ID does not exist. + '500': + description: Internal server errors. + '/projects/{project_id}/webhook/policies/test': + post: + summary: Test project webhook connection + description: | + This endpoint tests webhook connection of a project. + parameters: + - name: project_id + in: path + description: Relevant project ID. + required: true + type: integer + format: int64 + - name: policy + in: body + description: Only property "targets" needed. + required: true + schema: + $ref: '#/definitions/WebhookPolicy' + tags: + - Products + responses: + '200': + description: Test webhook connection successfully. + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '403': + description: User have no permission to get webhook policy of the project. + '500': + description: Internal server errors. + '/projects/{project_id}/webhook/lasttrigger': + get: + summary: Get project webhook policy last trigger info + description: | + This endpoint returns last trigger information of project webhook policy. + parameters: + - name: project_id + in: path + description: Relevant project ID. + required: true + type: integer + format: int64 + tags: + - Products + responses: + '200': + description: Test webhook connection successfully. + schema: + type: array + items: + $ref: '#/definitions/WebhookLastTrigger' + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '403': + description: User have no permission to get webhook policy of the project. + '500': + description: Internal server errors. + '/projects/{project_id}/webhook/jobs': + get: + sumary: List project webhook jobs + description: | + This endpoint returns webhook jobs of a project. + parameters: + - name: project_id + in: path + type: integer + format: int64 + required: true + description: Relevant project ID. + - name: policy_id + in: query + type: integer + format: int64 + required: true + description: The policy ID. + tags: + - Products + responses: + '200': + description: List project webhook jobs successfully. + schema: + type: array + items: + $ref: '#/definitions/WebhookJob' + '400': + description: Illegal format of provided ID value. + '401': + description: User need to log in first. + '403': + description: User have no permission to list webhook jobs of the project. + '500': + description: Unexpected internal errors. responses: OK: description: 'Success' @@ -5385,4 +5644,103 @@ definitions: description: the creation time of the quota update_time: type: string - description: the update time of the quota \ No newline at end of file + description: the update time of the quota + WebhookTargetObject: + type: object + description: The webhook policy target object. + properties: + type: + type: string + description: The webhook target notify type. + address: + type: string + description: The webhook target address. + auth_header: + type: string + description: The webhook auth header. + skip_cert_verify: + type: boolean + description: Whether or not to skip cert verify. + WebhookPolicy: + type: object + description: The webhook policy object + properties: + id: + type: integer + format: int64 + description: The webhook policy ID. + name: + type: string + description: The name of webhook policy. + description: + type: string + description: The description of webhook policy. + project_id: + type: integer + description: The project ID of webhook policy. + targets: + type: array + items: + $ref: '#/definitions/WebhookTargetObject' + event_types: + type: array + items: + type: string + creator: + type: string + description: The creator of the webhook policy. + creation_time: + type: string + description: The create time of the webhook policy. + update_time: + type: string + description: The update time of the webhook policy. + enabled: + type: boolean + description: Whether the webhook policy is enabled or not. + WebhookLastTrigger: + type: object + description: The webhook policy and last trigger time group by event type. + properties: + event_type: + type: string + description: The webhook event type. + enabled: + type: boolean + description: Whether or not the webhook policy enabled. + creation_time: + type: string + description: The creation time of webhook policy. + last_trigger_time: + type: string + description: The last trigger time of webhook policy. + WebhookJob: + type: object + description: The webhook job. + properties: + id: + type: integer + format: int64 + description: The webhook job ID. + policy_id: + type: integer + fromat: int64 + description: The webhook policy ID. + event_type: + type: string + description: The webhook job event type. + notify_type: + type: string + description: The webhook job notify type. + status: + type: string + description: The webhook job status. + job_detail: + type: string + description: The webhook job notify detailed data. + creation_time: + type: string + description: The webhook job creation time. + update_time: + type: string + description: The webhook job update time. diff --git a/src/common/config/metadata/metadatalist.go b/src/common/config/metadata/metadatalist.go index f95446d86..3aa42f619 100644 --- a/src/common/config/metadata/metadatalist.go +++ b/src/common/config/metadata/metadatalist.go @@ -149,6 +149,7 @@ var ( {Name: common.WithNotary, Scope: SystemScope, Group: BasicGroup, EnvKey: "WITH_NOTARY", DefaultValue: "false", ItemType: &BoolType{}, Editable: true}, // the unit of expiration is minute, 43200 minutes = 30 days {Name: common.RobotTokenDuration, Scope: UserScope, Group: BasicGroup, EnvKey: "ROBOT_TOKEN_DURATION", DefaultValue: "43200", ItemType: &IntType{}, Editable: true}, + {Name: common.NotificationEnable, Scope: UserScope, Group: BasicGroup, EnvKey: "NOTIFICATION_ENABLE", DefaultValue: "true", ItemType: &BoolType{}, Editable: true}, {Name: common.CountPerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "COUNT_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, {Name: common.StoragePerProject, Scope: UserScope, Group: QuotaGroup, EnvKey: "STORAGE_PER_PROJECT", DefaultValue: "-1", ItemType: &QuotaType{}, Editable: true}, diff --git a/src/common/const.go b/src/common/const.go old mode 100644 new mode 100755 index 19871270b..f2778a48e --- a/src/common/const.go +++ b/src/common/const.go @@ -144,6 +144,8 @@ const ( ChartUploadCtxKey = contextKey("chart_upload_event") + // Global notification enable configuration + NotificationEnable = "notification_enable" // Quota setting items for project CountPerProject = "count_per_project" StoragePerProject = "storage_per_project" diff --git a/src/common/dao/base.go b/src/common/dao/base.go index 804a73208..253b02692 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -167,11 +167,13 @@ func ClearTable(table string) error { return err } -func paginateForRawSQL(sql string, limit, offset int64) string { +// PaginateForRawSQL ... +func PaginateForRawSQL(sql string, limit, offset int64) string { return fmt.Sprintf("%s limit %d offset %d", sql, limit, offset) } -func paginateForQuerySetter(qs orm.QuerySeter, page, size int64) orm.QuerySeter { +// PaginateForQuerySetter ... +func PaginateForQuerySetter(qs orm.QuerySeter, page, size int64) orm.QuerySeter { if size > 0 { qs = qs.Limit(size) if page > 0 { diff --git a/src/common/dao/notification/notification_job.go b/src/common/dao/notification/notification_job.go new file mode 100755 index 000000000..1bd8c5039 --- /dev/null +++ b/src/common/dao/notification/notification_job.go @@ -0,0 +1,122 @@ +package notification + +import ( + "fmt" + + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/pkg/errors" +) + +// UpdateNotificationJob update notification job +func UpdateNotificationJob(job *models.NotificationJob, props ...string) (int64, error) { + if job == nil { + return 0, errors.New("nil job") + } + + if job.ID == 0 { + return 0, fmt.Errorf("notification job ID is empty") + } + + o := dao.GetOrmer() + return o.Update(job, props...) +} + +// AddNotificationJob insert new notification job to DB +func AddNotificationJob(job *models.NotificationJob) (int64, error) { + if job == nil { + return 0, errors.New("nil job") + } + o := dao.GetOrmer() + if len(job.Status) == 0 { + job.Status = models.JobPending + } + return o.Insert(job) +} + +// GetNotificationJob ... +func GetNotificationJob(id int64) (*models.NotificationJob, error) { + o := dao.GetOrmer() + j := &models.NotificationJob{ + ID: id, + } + err := o.Read(j) + if err == orm.ErrNoRows { + return nil, nil + } + return j, nil +} + +// GetTotalCountOfNotificationJobs ... +func GetTotalCountOfNotificationJobs(query ...*models.NotificationJobQuery) (int64, error) { + qs := notificationJobQueryConditions(query...) + return qs.Count() +} + +// GetNotificationJobs ... +func GetNotificationJobs(query ...*models.NotificationJobQuery) ([]*models.NotificationJob, error) { + var jobs []*models.NotificationJob + + qs := notificationJobQueryConditions(query...) + if len(query) > 0 && query[0] != nil { + qs = dao.PaginateForQuerySetter(qs, query[0].Page, query[0].Size) + } + + qs = qs.OrderBy("-UpdateTime") + + _, err := qs.All(&jobs) + return jobs, err +} + +// GetLastTriggerJobsGroupByEventType get notification jobs info of policy, including event type and last trigger time +func GetLastTriggerJobsGroupByEventType(policyID int64) ([]*models.NotificationJob, error) { + o := dao.GetOrmer() + // get jobs last triggered(created) group by event_type. postgres group by usage reference: + // https://stackoverflow.com/questions/13325583/postgresql-max-and-group-by + sql := `select distinct on (event_type) event_type, id, creation_time, status, notify_type, job_uuid, update_time, + creation_time, job_detail from notification_job where policy_id = ? + order by event_type, id desc, creation_time, status, notify_type, job_uuid, update_time, creation_time, job_detail` + + jobs := []*models.NotificationJob{} + _, err := o.Raw(sql, policyID).QueryRows(&jobs) + if err != nil { + log.Errorf("query last trigger info group by event type failed: %v", err) + return nil, err + } + + return jobs, nil +} + +// DeleteNotificationJob ... +func DeleteNotificationJob(id int64) error { + o := dao.GetOrmer() + _, err := o.Delete(&models.NotificationJob{ID: id}) + return err +} + +// DeleteAllNotificationJobsByPolicyID ... +func DeleteAllNotificationJobsByPolicyID(policyID int64) (int64, error) { + o := dao.GetOrmer() + return o.Delete(&models.NotificationJob{PolicyID: policyID}, "policy_id") +} + +func notificationJobQueryConditions(query ...*models.NotificationJobQuery) orm.QuerySeter { + qs := dao.GetOrmer().QueryTable(&models.NotificationJob{}) + if len(query) == 0 || query[0] == nil { + return qs + } + + q := query[0] + if q.PolicyID != 0 { + qs = qs.Filter("PolicyID", q.PolicyID) + } + if len(q.Statuses) > 0 { + qs = qs.Filter("Status__in", q.Statuses) + } + if len(q.EventTypes) > 0 { + qs = qs.Filter("EventType__in", q.EventTypes) + } + return qs +} diff --git a/src/common/dao/notification/notification_job_test.go b/src/common/dao/notification/notification_job_test.go new file mode 100644 index 000000000..0f7b97750 --- /dev/null +++ b/src/common/dao/notification/notification_job_test.go @@ -0,0 +1,263 @@ +package notification + +import ( + "testing" + + "github.com/goharbor/harbor/src/common/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + testJob1 = &models.NotificationJob{ + PolicyID: 1111, + EventType: "pushImage", + NotifyType: "http", + Status: "pending", + JobDetail: "{\"type\":\"pushImage\",\"occur_at\":1563536782,\"event_data\":{\"resources\":[{\"digest\":\"sha256:bf1684a6e3676389ec861c602e97f27b03f14178e5bc3f70dce198f9f160cce9\",\"tag\":\"v1.0\",\"resource_url\":\"10.194.32.23/myproj/alpine:v1.0\"}],\"repository\":{\"date_created\":1563505587,\"name\":\"alpine\",\"namespace\":\"myproj\",\"repo_full_name\":\"myproj/alpine\",\"repo_type\":\"private\"}},\"operator\":\"admin\"}", + UUID: "00000000", + } + testJob2 = &models.NotificationJob{ + PolicyID: 111, + EventType: "pullImage", + NotifyType: "http", + Status: "", + JobDetail: "{\"type\":\"pushImage\",\"occur_at\":1563537782,\"event_data\":{\"resources\":[{\"digest\":\"sha256:bf1684a6e3676389ec861c602e97f27b03f14178e5bc3f70dce198f9f160cce9\",\"tag\":\"v1.0\",\"resource_url\":\"10.194.32.23/myproj/alpine:v1.0\"}],\"repository\":{\"date_created\":1563505587,\"name\":\"alpine\",\"namespace\":\"myproj\",\"repo_full_name\":\"myproj/alpine\",\"repo_type\":\"private\"}},\"operator\":\"admin\"}", + UUID: "00000000", + } + testJob3 = &models.NotificationJob{ + PolicyID: 111, + EventType: "deleteImage", + NotifyType: "http", + Status: "pending", + JobDetail: "{\"type\":\"pushImage\",\"occur_at\":1563538782,\"event_data\":{\"resources\":[{\"digest\":\"sha256:bf1684a6e3676389ec861c602e97f27b03f14178e5bc3f70dce198f9f160cce9\",\"tag\":\"v1.0\",\"resource_url\":\"10.194.32.23/myproj/alpine:v1.0\"}],\"repository\":{\"date_created\":1563505587,\"name\":\"alpine\",\"namespace\":\"myproj\",\"repo_full_name\":\"myproj/alpine\",\"repo_type\":\"private\"}},\"operator\":\"admin\"}", + UUID: "00000000", + } +) + +func TestAddNotificationJob(t *testing.T) { + tests := []struct { + name string + job *models.NotificationJob + want int64 + wantErr bool + }{ + {name: "AddNotificationJob nil", job: nil, wantErr: true}, + {name: "AddNotificationJob 1", job: testJob1, want: 1}, + {name: "AddNotificationJob 2", job: testJob2, want: 2}, + {name: "AddNotificationJob 3", job: testJob3, want: 3}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := AddNotificationJob(tt.job) + if tt.wantErr { + require.NotNil(t, err, "wantErr: %s", err) + return + } + require.Nil(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestGetTotalCountOfNotificationJobs(t *testing.T) { + type args struct { + query *models.NotificationJobQuery + } + tests := []struct { + name string + args args + want int64 + wantErr bool + }{ + { + name: "GetTotalCountOfNotificationJobs 1", + args: args{ + query: &models.NotificationJobQuery{ + PolicyID: 111, + }, + }, + want: 2, + }, + { + name: "GetTotalCountOfNotificationJobs 2", + args: args{}, + want: 3, + }, + { + name: "GetTotalCountOfNotificationJobs 3", + args: args{ + query: &models.NotificationJobQuery{ + Statuses: []string{"pending"}, + }, + }, + want: 3, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetTotalCountOfNotificationJobs(tt.args.query) + if tt.wantErr { + require.NotNil(t, err, "wantErr: %s", err) + return + } + require.Nil(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestGetLastTriggerJobsGroupByEventType(t *testing.T) { + type args struct { + policyID int64 + } + tests := []struct { + name string + args args + want []*models.NotificationJob + wantErr bool + }{ + { + name: "GetLastTriggerJobsGroupByEventType", + args: args{ + policyID: 111, + }, + want: []*models.NotificationJob{ + testJob2, + testJob3, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetLastTriggerJobsGroupByEventType(tt.args.policyID) + if tt.wantErr { + require.NotNil(t, err, "wantErr: %s", err) + return + } + require.Nil(t, err) + assert.Equal(t, len(tt.want), len(got)) + }) + } + +} + +func TestUpdateNotificationJob(t *testing.T) { + type args struct { + job *models.NotificationJob + props []string + } + tests := []struct { + name string + args args + want int64 + wantErr bool + }{ + {name: "UpdateNotificationJob Want Error 1", args: args{job: nil}, wantErr: true}, + {name: "UpdateNotificationJob Want Error 2", args: args{job: &models.NotificationJob{ID: 0}}, wantErr: true}, + { + name: "UpdateNotificationJob 1", + args: args{ + job: &models.NotificationJob{ID: 1, UUID: "111111111111111"}, + props: []string{"UUID"}, + }, + }, + { + name: "UpdateNotificationJob 2", + args: args{ + job: &models.NotificationJob{ID: 2, UUID: "222222222222222"}, + props: []string{"UUID"}, + }, + }, + { + name: "UpdateNotificationJob 3", + args: args{ + job: &models.NotificationJob{ID: 3, UUID: "333333333333333"}, + props: []string{"UUID"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := UpdateNotificationJob(tt.args.job, tt.args.props...) + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + + require.Nil(t, err) + gotJob, err := GetNotificationJob(tt.args.job.ID) + + require.Nil(t, err) + assert.Equal(t, tt.args.job.UUID, gotJob.UUID) + }) + } +} + +func TestDeleteNotificationJob(t *testing.T) { + type args struct { + id int64 + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "DeleteNotificationJob 1", args: args{id: 1}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := DeleteNotificationJob(tt.args.id) + + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + + require.Nil(t, err) + job, err := GetNotificationJob(tt.args.id) + + require.Nil(t, err) + assert.Nil(t, job) + }) + } +} + +func TestDeleteAllNotificationJobs(t *testing.T) { + type args struct { + policyID int64 + query []*models.NotificationJobQuery + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "DeleteAllNotificationJobs 1", + args: args{ + policyID: 111, + query: []*models.NotificationJobQuery{ + {PolicyID: 111}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := DeleteAllNotificationJobsByPolicyID(tt.args.policyID) + + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + + require.Nil(t, err) + jobs, err := GetNotificationJobs(tt.args.query...) + + require.Nil(t, err) + assert.Equal(t, 0, len(jobs)) + }) + } +} diff --git a/src/common/dao/notification/notification_policy.go b/src/common/dao/notification/notification_policy.go new file mode 100755 index 000000000..58bf8a52c --- /dev/null +++ b/src/common/dao/notification/notification_policy.go @@ -0,0 +1,69 @@ +package notification + +import ( + "github.com/astaxie/beego/orm" + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/pkg/errors" +) + +// GetNotificationPolicy return notification policy by id +func GetNotificationPolicy(id int64) (*models.NotificationPolicy, error) { + policy := new(models.NotificationPolicy) + o := dao.GetOrmer() + err := o.QueryTable(policy).Filter("id", id).One(policy) + if err == orm.ErrNoRows { + return nil, nil + } + return policy, err +} + +// GetNotificationPolicyByName return notification policy by name +func GetNotificationPolicyByName(name string, projectID int64) (*models.NotificationPolicy, error) { + policy := new(models.NotificationPolicy) + o := dao.GetOrmer() + err := o.QueryTable(policy).Filter("name", name).Filter("projectID", projectID).One(policy) + if err == orm.ErrNoRows { + return nil, nil + } + return policy, err +} + +// GetNotificationPolicies returns all notification policy in project +func GetNotificationPolicies(projectID int64) ([]*models.NotificationPolicy, error) { + var policies []*models.NotificationPolicy + qs := dao.GetOrmer().QueryTable(new(models.NotificationPolicy)).Filter("ProjectID", projectID) + + _, err := qs.All(&policies) + if err != nil { + return nil, err + } + return policies, nil + +} + +// AddNotificationPolicy insert new notification policy to DB +func AddNotificationPolicy(policy *models.NotificationPolicy) (int64, error) { + if policy == nil { + return 0, errors.New("nil policy") + } + o := dao.GetOrmer() + return o.Insert(policy) +} + +// UpdateNotificationPolicy update t specified notification policy +func UpdateNotificationPolicy(policy *models.NotificationPolicy) error { + if policy == nil { + return errors.New("nil policy") + } + o := dao.GetOrmer() + _, err := o.Update(policy) + return err +} + +// DeleteNotificationPolicy delete notification policy by id +func DeleteNotificationPolicy(id int64) error { + o := dao.GetOrmer() + _, err := o.Delete(&models.NotificationPolicy{ID: id}) + return err +} diff --git a/src/common/dao/notification/notification_policy_test.go b/src/common/dao/notification/notification_policy_test.go new file mode 100644 index 000000000..756a01c7d --- /dev/null +++ b/src/common/dao/notification/notification_policy_test.go @@ -0,0 +1,291 @@ +package notification + +import ( + "testing" + "time" + + "github.com/goharbor/harbor/src/common/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + testPly1 = &models.NotificationPolicy{ + Name: "webhook test policy1", + Description: "webhook test policy1 description", + ProjectID: 111, + TargetsDB: "[{\"type\":\"http\",\"address\":\"http://10.173.32.58:9009\",\"token\":\"xxxxxxxxx\",\"skip_cert_verify\":true}]", + EventTypesDB: "[\"pushImage\",\"pullImage\",\"deleteImage\",\"uploadChart\",\"deleteChart\",\"downloadChart\",\"scanningFailed\",\"scanningCompleted\"]", + Creator: "no one", + CreationTime: time.Now(), + UpdateTime: time.Now(), + Enabled: true, + } +) + +var ( + testPly2 = &models.NotificationPolicy{ + Name: "webhook test policy2", + Description: "webhook test policy2 description", + ProjectID: 222, + TargetsDB: "[{\"type\":\"http\",\"address\":\"http://10.173.32.58:9009\",\"token\":\"xxxxxxxxx\",\"skip_cert_verify\":true}]", + EventTypesDB: "[\"pushImage\",\"pullImage\",\"deleteImage\",\"uploadChart\",\"deleteChart\",\"downloadChart\",\"scanningFailed\",\"scanningCompleted\"]", + Creator: "no one", + CreationTime: time.Now(), + UpdateTime: time.Now(), + Enabled: true, + } +) + +var ( + testPly3 = &models.NotificationPolicy{ + Name: "webhook test policy3", + Description: "webhook test policy3 description", + ProjectID: 333, + TargetsDB: "[{\"type\":\"http\",\"address\":\"http://10.173.32.58:9009\",\"token\":\"xxxxxxxxx\",\"skip_cert_verify\":true}]", + EventTypesDB: "[\"pushImage\",\"pullImage\",\"deleteImage\",\"uploadChart\",\"deleteChart\",\"downloadChart\",\"scanningFailed\",\"scanningCompleted\"]", + Creator: "no one", + CreationTime: time.Now(), + UpdateTime: time.Now(), + Enabled: true, + } +) + +func TestAddNotificationPolicy(t *testing.T) { + tests := []struct { + name string + policy *models.NotificationPolicy + want int64 + wantErr bool + }{ + {name: "AddNotificationPolicy nil", policy: nil, wantErr: true}, + {name: "AddNotificationPolicy 1", policy: testPly1, want: 1}, + {name: "AddNotificationPolicy 2", policy: testPly2, want: 2}, + {name: "AddNotificationPolicy 3", policy: testPly3, want: 3}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := AddNotificationPolicy(tt.policy) + + if tt.wantErr { + require.NotNil(t, err, "wantErr: %s", err) + return + } + require.Nil(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestGetNotificationPolicies(t *testing.T) { + tests := []struct { + name string + projectID int64 + wantPolicies []*models.NotificationPolicy + wantErr bool + }{ + {name: "GetNotificationPolicies nil", projectID: 0, wantPolicies: []*models.NotificationPolicy{}}, + {name: "GetNotificationPolicies 1", projectID: 111, wantPolicies: []*models.NotificationPolicy{testPly1}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPolicies, err := GetNotificationPolicies(tt.projectID) + if tt.wantErr { + require.NotNil(t, err, "wantErr: %s", err) + return + } + + require.Nil(t, err) + for i, gotPolicy := range gotPolicies { + assert.Equal(t, tt.wantPolicies[i].Name, gotPolicy.Name) + assert.Equal(t, tt.wantPolicies[i].ID, gotPolicy.ID) + assert.Equal(t, tt.wantPolicies[i].EventTypesDB, gotPolicy.EventTypesDB) + assert.Equal(t, tt.wantPolicies[i].TargetsDB, gotPolicy.TargetsDB) + assert.Equal(t, tt.wantPolicies[i].Creator, gotPolicy.Creator) + assert.Equal(t, tt.wantPolicies[i].Enabled, gotPolicy.Enabled) + assert.Equal(t, tt.wantPolicies[i].Description, gotPolicy.Description) + } + }) + } +} + +func TestGetNotificationPolicy(t *testing.T) { + tests := []struct { + name string + id int64 + wantPolicy *models.NotificationPolicy + wantErr bool + }{ + {name: "GetRepPolicy 1", id: 1, wantPolicy: testPly1}, + {name: "GetRepPolicy 2", id: 2, wantPolicy: testPly2}, + {name: "GetRepPolicy 3", id: 3, wantPolicy: testPly3}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPolicy, err := GetNotificationPolicy(tt.id) + if tt.wantErr { + require.NotNil(t, err, "wantErr: %s", err) + return + } + require.Nil(t, err) + assert.Equal(t, tt.wantPolicy.Name, gotPolicy.Name) + assert.Equal(t, tt.wantPolicy.ID, gotPolicy.ID) + assert.Equal(t, tt.wantPolicy.EventTypesDB, gotPolicy.EventTypesDB) + assert.Equal(t, tt.wantPolicy.TargetsDB, gotPolicy.TargetsDB) + assert.Equal(t, tt.wantPolicy.Creator, gotPolicy.Creator) + assert.Equal(t, tt.wantPolicy.Enabled, gotPolicy.Enabled) + assert.Equal(t, tt.wantPolicy.Description, gotPolicy.Description) + }) + } +} + +func TestGetNotificationPolicyByName(t *testing.T) { + type args struct { + name string + projectID int64 + } + tests := []struct { + name string + args args + wantPolicy *models.NotificationPolicy + wantErr bool + }{ + {name: "GetNotificationPolicyByName 1", args: args{name: testPly1.Name, projectID: testPly1.ProjectID}, wantPolicy: testPly1}, + {name: "GetNotificationPolicyByName 2", args: args{name: testPly2.Name, projectID: testPly2.ProjectID}, wantPolicy: testPly2}, + {name: "GetNotificationPolicyByName 3", args: args{name: testPly3.Name, projectID: testPly3.ProjectID}, wantPolicy: testPly3}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPolicy, err := GetNotificationPolicyByName(tt.args.name, tt.args.projectID) + if tt.wantErr { + require.NotNil(t, err, "wantErr: %s", err) + return + } + require.Nil(t, err) + assert.Equal(t, tt.wantPolicy.Name, gotPolicy.Name) + assert.Equal(t, tt.wantPolicy.ID, gotPolicy.ID) + assert.Equal(t, tt.wantPolicy.EventTypesDB, gotPolicy.EventTypesDB) + assert.Equal(t, tt.wantPolicy.TargetsDB, gotPolicy.TargetsDB) + assert.Equal(t, tt.wantPolicy.Creator, gotPolicy.Creator) + assert.Equal(t, tt.wantPolicy.Enabled, gotPolicy.Enabled) + assert.Equal(t, tt.wantPolicy.Description, gotPolicy.Description) + }) + } + +} + +func TestUpdateNotificationPolicy(t *testing.T) { + type args struct { + policy *models.NotificationPolicy + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "UpdateNotificationPolicy nil", + args: args{ + policy: nil, + }, + wantErr: true, + }, + + { + name: "UpdateNotificationPolicy 1", + args: args{ + policy: &models.NotificationPolicy{ + ID: 1, + Name: "webhook test policy1 new", + Description: "webhook test policy1 description new", + ProjectID: 111, + TargetsDB: "[{\"type\":\"http\",\"address\":\"http://10.173.32.58:9009\",\"token\":\"xxxxxxxxx\",\"skip_cert_verify\":true}]", + EventTypesDB: "[\"pushImage\",\"pullImage\",\"deleteImage\",\"uploadChart\",\"deleteChart\",\"downloadChart\",\"scanningFailed\",\"scanningCompleted\"]", + Creator: "no one", + CreationTime: time.Now(), + UpdateTime: time.Now(), + Enabled: true, + }, + }, + }, + { + name: "UpdateNotificationPolicy 2", + args: args{ + policy: &models.NotificationPolicy{ + ID: 2, + Name: "webhook test policy2 new", + Description: "webhook test policy2 description new", + ProjectID: 222, + TargetsDB: "[{\"type\":\"http\",\"address\":\"http://10.173.32.58:9009\",\"token\":\"xxxxxxxxx\",\"skip_cert_verify\":true}]", + EventTypesDB: "[\"pushImage\",\"pullImage\",\"deleteImage\",\"uploadChart\",\"deleteChart\",\"downloadChart\",\"scanningFailed\",\"scanningCompleted\"]", + Creator: "no one", + CreationTime: time.Now(), + UpdateTime: time.Now(), + Enabled: true, + }, + }, + }, + { + name: "UpdateNotificationPolicy 3", + args: args{ + policy: &models.NotificationPolicy{ + ID: 3, + Name: "webhook test policy3 new", + Description: "webhook test policy3 description new", + ProjectID: 333, + TargetsDB: "[{\"type\":\"http\",\"address\":\"http://10.173.32.58:9009\",\"token\":\"xxxxxxxxx\",\"skip_cert_verify\":true}]", + EventTypesDB: "[\"pushImage\",\"pullImage\",\"deleteImage\",\"uploadChart\",\"deleteChart\",\"downloadChart\",\"scanningFailed\",\"scanningCompleted\"]", + Creator: "no one", + CreationTime: time.Now(), + UpdateTime: time.Now(), + Enabled: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := UpdateNotificationPolicy(tt.args.policy) + + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + + require.Nil(t, err) + gotPolicy, err := GetNotificationPolicy(tt.args.policy.ID) + + require.Nil(t, err) + assert.Equal(t, tt.args.policy.Description, gotPolicy.Description) + assert.Equal(t, tt.args.policy.Name, gotPolicy.Name) + }) + } + +} + +func TestDeleteNotificationPolicy(t *testing.T) { + tests := []struct { + name string + id int64 + wantErr bool + }{ + {name: "DeleteNotificationPolicy 1", id: 1, wantErr: false}, + {name: "DeleteNotificationPolicy 2", id: 2, wantErr: false}, + {name: "DeleteNotificationPolicy 3", id: 3, wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := DeleteNotificationPolicy(tt.id) + if tt.wantErr { + require.NotNil(t, err, "wantErr: %s", err) + return + } + require.Nil(t, err) + policy, err := GetNotificationPolicy(tt.id) + require.Nil(t, err) + assert.Nil(t, policy) + }) + } +} diff --git a/src/common/dao/notification/notification_test.go b/src/common/dao/notification/notification_test.go new file mode 100644 index 000000000..2912e75f9 --- /dev/null +++ b/src/common/dao/notification/notification_test.go @@ -0,0 +1,13 @@ +package notification + +import ( + "os" + "testing" + + "github.com/goharbor/harbor/src/common/dao" +) + +func TestMain(m *testing.M) { + dao.PrepareTestForPostgresSQL() + os.Exit(m.Run()) +} diff --git a/src/common/http/client.go b/src/common/http/client.go index 533212dc0..7699e33f2 100644 --- a/src/common/http/client.go +++ b/src/common/http/client.go @@ -16,6 +16,7 @@ package http import ( "bytes" + "crypto/tls" "encoding/json" "errors" "io" @@ -35,6 +36,36 @@ type Client struct { client *http.Client } +var defaultHTTPTransport, secureHTTPTransport, insecureHTTPTransport *http.Transport + +func init() { + defaultHTTPTransport = &http.Transport{} + + secureHTTPTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: false, + }, + } + insecureHTTPTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } +} + +// GetHTTPTransport returns HttpTransport based on insecure configuration +func GetHTTPTransport(insecure ...bool) *http.Transport { + if len(insecure) == 0 { + return defaultHTTPTransport + } + if insecure[0] { + return insecureHTTPTransport + } + return secureHTTPTransport +} + // NewClient creates an instance of Client. // Use net/http.Client as the default value if c is nil. // Modifiers modify the request before sending it. diff --git a/src/common/http/client_test.go b/src/common/http/client_test.go new file mode 100644 index 000000000..09f576c97 --- /dev/null +++ b/src/common/http/client_test.go @@ -0,0 +1,14 @@ +package http + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetHTTPTransport(t *testing.T) { + transport := GetHTTPTransport(true) + assert.True(t, transport.TLSClientConfig.InsecureSkipVerify) + transport = GetHTTPTransport(false) + assert.False(t, transport.TLSClientConfig.InsecureSkipVerify) +} diff --git a/src/common/models/base.go b/src/common/models/base.go index 7ecee503c..3dcb5869c 100644 --- a/src/common/models/base.go +++ b/src/common/models/base.go @@ -37,6 +37,8 @@ func init() { new(JobLog), new(Robot), new(OIDCUser), + new(NotificationPolicy), + new(NotificationJob), new(Blob), new(Artifact), new(ArtifactAndBlob), diff --git a/src/common/models/hook_notification.go b/src/common/models/hook_notification.go new file mode 100755 index 000000000..60c667afd --- /dev/null +++ b/src/common/models/hook_notification.go @@ -0,0 +1,111 @@ +package models + +import ( + "encoding/json" + "time" +) + +const ( + // NotificationPolicyTable is table name for notification policies + NotificationPolicyTable = "notification_policy" + // NotificationJobTable is table name for notification job + NotificationJobTable = "notification_job" +) + +// NotificationPolicy is the model for a notification policy. +type NotificationPolicy struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + Name string `orm:"column(name)" json:"name"` + Description string `orm:"column(description)" json:"description"` + ProjectID int64 `orm:"column(project_id)" json:"project_id"` + TargetsDB string `orm:"column(targets)" json:"-"` + Targets []EventTarget `orm:"-" json:"targets"` + EventTypesDB string `orm:"column(event_types)" json:"-"` + EventTypes []string `orm:"-" json:"event_types"` + Creator string `orm:"column(creator)" json:"creator"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now_add" json:"update_time"` + Enabled bool `orm:"column(enabled)" json:"enabled"` +} + +// TableName set table name for ORM. +func (w *NotificationPolicy) TableName() string { + return NotificationPolicyTable +} + +// ConvertToDBModel convert struct data in notification policy to DB model data +func (w *NotificationPolicy) ConvertToDBModel() error { + if len(w.Targets) != 0 { + targets, err := json.Marshal(w.Targets) + if err != nil { + return err + } + w.TargetsDB = string(targets) + } + if len(w.EventTypes) != 0 { + eventTypes, err := json.Marshal(w.EventTypes) + if err != nil { + return err + } + w.EventTypesDB = string(eventTypes) + } + + return nil +} + +// ConvertFromDBModel convert from DB model data to struct data +func (w *NotificationPolicy) ConvertFromDBModel() error { + targets := []EventTarget{} + if len(w.TargetsDB) != 0 { + err := json.Unmarshal([]byte(w.TargetsDB), &targets) + if err != nil { + return err + } + } + w.Targets = targets + + types := []string{} + if len(w.EventTypesDB) != 0 { + err := json.Unmarshal([]byte(w.EventTypesDB), &types) + if err != nil { + return err + } + } + w.EventTypes = types + + return nil +} + +// NotificationJob is the model for a notification job +type NotificationJob struct { + ID int64 `orm:"pk;auto;column(id)" json:"id"` + PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` + EventType string `orm:"column(event_type)" json:"event_type"` + NotifyType string `orm:"column(notify_type)" json:"notify_type"` + Status string `orm:"column(status)" json:"status"` + JobDetail string `orm:"column(job_detail)" json:"job_detail"` + UUID string `orm:"column(job_uuid)" json:"-"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +// TableName set table name for ORM. +func (w *NotificationJob) TableName() string { + return NotificationJobTable +} + +// NotificationJobQuery holds query conditions for notification job +type NotificationJobQuery struct { + PolicyID int64 + Statuses []string + EventTypes []string + Pagination +} + +// EventTarget defines the structure of target a notification send to +type EventTarget struct { + Type string `json:"type"` + Address string `json:"address"` + AuthHeader string `json:"auth_header,omitempty"` + SkipCertVerify bool `json:"skip_cert_verify"` +} diff --git a/src/common/models/hook_notification_test.go b/src/common/models/hook_notification_test.go new file mode 100644 index 000000000..31c18c8b6 --- /dev/null +++ b/src/common/models/hook_notification_test.go @@ -0,0 +1,114 @@ +package models + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNotificationPolicy_ConvertFromDBModel(t *testing.T) { + tests := []struct { + name string + policy *NotificationPolicy + want *NotificationPolicy + wantErr bool + }{ + { + name: "ConvertFromDBModel want error 1", + policy: &NotificationPolicy{ + TargetsDB: "[{{\"type\":\"http\",\"address\":\"http://10.173.32.58:9009\"}]", + }, + wantErr: true, + }, + { + name: "ConvertFromDBModel want error 2", + policy: &NotificationPolicy{ + EventTypesDB: "[{\"pushImage\",\"pullImage\",\"deleteImage\"]", + }, + wantErr: true, + }, + { + name: "ConvertFromDBModel 1", + policy: &NotificationPolicy{ + TargetsDB: "[{\"type\":\"http\",\"address\":\"http://10.173.32.58:9009\"}]", + EventTypesDB: "[\"pushImage\",\"pullImage\",\"deleteImage\"]", + }, + want: &NotificationPolicy{ + Targets: []EventTarget{ + { + Type: "http", + Address: "http://10.173.32.58:9009", + }, + }, + EventTypes: []string{"pushImage", "pullImage", "deleteImage"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.policy.ConvertFromDBModel() + if tt.wantErr { + require.NotNil(t, err, "wantErr: %s", err) + return + } + require.Nil(t, err) + assert.Equal(t, tt.want.Targets, tt.policy.Targets) + assert.Equal(t, tt.want.EventTypes, tt.policy.EventTypes) + }) + } +} + +func TestNotificationPolicy_ConvertToDBModel(t *testing.T) { + tests := []struct { + name string + policy *NotificationPolicy + want *NotificationPolicy + wantErr bool + }{ + { + name: "ConvertToDBModel 1", + policy: &NotificationPolicy{ + Targets: []EventTarget{ + { + Type: "http", + Address: "http://127.0.0.1", + SkipCertVerify: false, + }, + }, + EventTypes: []string{"pushImage", "pullImage", "deleteImage"}, + }, + want: &NotificationPolicy{ + TargetsDB: "[{\"type\":\"http\",\"address\":\"http://127.0.0.1\",\"skip_cert_verify\":false}]", + EventTypesDB: "[\"pushImage\",\"pullImage\",\"deleteImage\"]", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.policy.ConvertToDBModel() + if tt.wantErr { + require.NotNil(t, err, "wantErr: %s", err) + return + } + require.Nil(t, err) + assert.Equal(t, tt.want.TargetsDB, tt.policy.TargetsDB) + assert.Equal(t, tt.want.EventTypesDB, tt.policy.EventTypesDB) + }) + } +} + +func TestNotificationJob_TableName(t *testing.T) { + job := &NotificationJob{} + got := job.TableName() + assert.Equal(t, NotificationJobTable, got) +} + +func TestNotificationPolicy_TableName(t *testing.T) { + policy := &NotificationPolicy{} + got := policy.TableName() + assert.Equal(t, NotificationPolicyTable, got) + +} diff --git a/src/common/models/project.go b/src/common/models/project.go index e7f888ae1..1b56284a3 100644 --- a/src/common/models/project.go +++ b/src/common/models/project.go @@ -21,8 +21,14 @@ import ( "github.com/goharbor/harbor/src/pkg/types" ) -// ProjectTable is the table name for project -const ProjectTable = "project" +const ( + // ProjectTable is the table name for project + ProjectTable = "project" + // ProjectPublic means project is public + ProjectPublic = "public" + // ProjectPrivate means project is private + ProjectPrivate = "private" +) // Project holds the details of a project. type Project struct { diff --git a/src/common/rbac/const.go b/src/common/rbac/const.go old mode 100644 new mode 100755 index fa7da634e..6cadbddef --- a/src/common/rbac/const.go +++ b/src/common/rbac/const.go @@ -56,5 +56,6 @@ const ( ResourceRepositoryTagScanJob = Resource("repository-tag-scan-job") ResourceRepositoryTagVulnerability = Resource("repository-tag-vulnerability") ResourceRobot = Resource("robot") + ResourceNotificationPolicy = Resource("notification-policy") ResourceSelf = Resource("") // subresource for self ) diff --git a/src/common/rbac/project/util.go b/src/common/rbac/project/util.go index 5e2dd3769..3de3f5810 100644 --- a/src/common/rbac/project/util.go +++ b/src/common/rbac/project/util.go @@ -151,6 +151,12 @@ var ( {Resource: rbac.ResourceRobot, Action: rbac.ActionUpdate}, {Resource: rbac.ResourceRobot, Action: rbac.ActionDelete}, {Resource: rbac.ResourceRobot, Action: rbac.ActionList}, + + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionCreate}, + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionUpdate}, + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionDelete}, + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList}, + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionRead}, } ) diff --git a/src/common/rbac/project/visitor_role.go b/src/common/rbac/project/visitor_role.go old mode 100644 new mode 100755 index ed221502f..36202a602 --- a/src/common/rbac/project/visitor_role.go +++ b/src/common/rbac/project/visitor_role.go @@ -108,6 +108,12 @@ var ( {Resource: rbac.ResourceRobot, Action: rbac.ActionUpdate}, {Resource: rbac.ResourceRobot, Action: rbac.ActionDelete}, {Resource: rbac.ResourceRobot, Action: rbac.ActionList}, + + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionCreate}, + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionUpdate}, + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionDelete}, + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList}, + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionRead}, }, "master": { @@ -183,6 +189,8 @@ var ( {Resource: rbac.ResourceRobot, Action: rbac.ActionRead}, {Resource: rbac.ResourceRobot, Action: rbac.ActionList}, + + {Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList}, }, "developer": { diff --git a/src/core/api/chart_repository.go b/src/core/api/chart_repository.go old mode 100644 new mode 100755 index b327c783b..dd9be934f --- a/src/core/api/chart_repository.go +++ b/src/core/api/chart_repository.go @@ -18,6 +18,7 @@ import ( hlog "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/label" + "github.com/goharbor/harbor/src/core/middlewares" rep_event "github.com/goharbor/harbor/src/replication/event" "github.com/goharbor/harbor/src/replication/model" diff --git a/src/core/api/harborapi_test.go b/src/core/api/harborapi_test.go index 574092ee9..5357a6579 100644 --- a/src/core/api/harborapi_test.go +++ b/src/core/api/harborapi_test.go @@ -39,6 +39,7 @@ import ( _ "github.com/goharbor/harbor/src/core/auth/ldap" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/filter" + "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/testing/apitests/apilib" ) @@ -170,6 +171,12 @@ func init() { beego.Router("/api/retentions/:id/executions/:eid/tasks", &RetentionAPI{}, "get:ListRetentionExecTasks") beego.Router("/api/retentions/:id/executions/:eid/tasks/:tid", &RetentionAPI{}, "get:GetRetentionExecTaskLog") + beego.Router("/api/projects/:pid([0-9]+)/webhook/policies", &NotificationPolicyAPI{}, "get:List;post:Post") + beego.Router("/api/projects/:pid([0-9]+)/webhook/policies/:id([0-9]+)", &NotificationPolicyAPI{}) + beego.Router("/api/projects/:pid([0-9]+)/webhook/policies/test", &NotificationPolicyAPI{}, "post:Test") + beego.Router("/api/projects/:pid([0-9]+)/webhook/lasttrigger", &NotificationPolicyAPI{}, "get:ListGroupByEventType") + beego.Router("/api/projects/:pid([0-9]+)/webhook/jobs/", &NotificationJobAPI{}, "get:List") + // Charts are controlled under projects chartRepositoryAPIType := &ChartRepositoryAPI{} beego.Router("/api/chartrepo/health", chartRepositoryAPIType, "get:GetHealthStatus") @@ -205,6 +212,9 @@ func init() { unknownUsr = &usrInfo{"unknown", "unknown"} testUser = &usrInfo{TestUserName, TestUserPwd} + // Init notification related check map + notification.Init() + // Init mock jobservice mockServer := test.NewJobServiceServer() defer mockServer.Close() diff --git a/src/core/api/notification_job.go b/src/core/api/notification_job.go new file mode 100755 index 000000000..775c9fc9f --- /dev/null +++ b/src/core/api/notification_job.go @@ -0,0 +1,108 @@ +package api + +import ( + "errors" + "fmt" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/pkg/notification" +) + +// NotificationJobAPI ... +type NotificationJobAPI struct { + BaseController + project *models.Project +} + +// Prepare ... +func (w *NotificationJobAPI) Prepare() { + w.BaseController.Prepare() + if !w.SecurityCtx.IsAuthenticated() { + w.SendUnAuthorizedError(errors.New("UnAuthorized")) + return + } + + pid, err := w.GetInt64FromPath(":pid") + if err != nil { + w.SendBadRequestError(fmt.Errorf("failed to get project ID: %v", err)) + return + } + if pid <= 0 { + w.SendBadRequestError(fmt.Errorf("invalid project ID: %d", pid)) + return + } + + project, err := w.ProjectMgr.Get(pid) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to get project %d: %v", pid, err)) + return + } + if project == nil { + w.SendNotFoundError(fmt.Errorf("project %d not found", pid)) + return + } + w.project = project +} + +// List ... +func (w *NotificationJobAPI) List() { + if !w.validateRBAC(rbac.ActionList, w.project.ProjectID) { + return + } + + policyID, err := w.GetInt64("policy_id") + if err != nil || policyID <= 0 { + w.SendBadRequestError(fmt.Errorf("invalid policy_id: %s", w.GetString("policy_id"))) + return + } + + policy, err := notification.PolicyMgr.Get(policyID) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to get policy %d: %v", policyID, err)) + return + } + if policy == nil { + w.SendBadRequestError(fmt.Errorf("policy %d not found", policyID)) + return + } + + query := &models.NotificationJobQuery{ + PolicyID: policyID, + } + + query.Statuses = w.GetStrings("status") + + query.Page, query.Size, err = w.GetPaginationParams() + if err != nil { + w.SendBadRequestError(err) + return + } + + total, jobs, err := notification.JobMgr.List(query) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to list notification jobs: %v", err)) + return + } + w.SetPaginationHeader(total, query.Page, query.Size) + w.WriteJSONData(jobs) +} + +func (w *NotificationJobAPI) validateRBAC(action rbac.Action, projectID int64) bool { + if w.SecurityCtx.IsSysAdmin() { + return true + } + + project, err := w.ProjectMgr.Get(projectID) + if err != nil { + w.ParseAndHandleError(fmt.Sprintf("failed to get project %d", projectID), err) + return false + } + + resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceNotificationPolicy) + if !w.SecurityCtx.Can(action, resource) { + w.SendForbiddenError(errors.New(w.SecurityCtx.GetUsername())) + return false + } + return true +} diff --git a/src/core/api/notification_job_test.go b/src/core/api/notification_job_test.go new file mode 100644 index 000000000..d6a9ac099 --- /dev/null +++ b/src/core/api/notification_job_test.go @@ -0,0 +1,107 @@ +package api + +import ( + "net/http" + "testing" + "time" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/pkg/notification" + "github.com/goharbor/harbor/src/pkg/notification/model" +) + +type fakedNotificationJobMgr struct { +} + +func (f *fakedNotificationJobMgr) Create(job *models.NotificationJob) (int64, error) { + return 1, nil +} + +func (f *fakedNotificationJobMgr) List(...*models.NotificationJobQuery) (int64, []*models.NotificationJob, error) { + return 0, nil, nil +} + +func (f *fakedNotificationJobMgr) Update(job *models.NotificationJob, props ...string) error { + return nil +} + +func (f *fakedNotificationJobMgr) ListJobsGroupByEventType(policyID int64) ([]*models.NotificationJob, error) { + return []*models.NotificationJob{ + { + EventType: model.EventTypePullImage, + CreationTime: time.Now(), + }, + { + EventType: model.EventTypeDeleteImage, + CreationTime: time.Now(), + }, + }, nil +} + +func TestNotificationJobAPI_List(t *testing.T) { + policyMgr := notification.PolicyMgr + jobMgr := notification.JobMgr + defer func() { + notification.PolicyMgr = policyMgr + notification.JobMgr = jobMgr + }() + notification.PolicyMgr = &fakedNotificationPlyMgr{} + notification.JobMgr = &fakedNotificationJobMgr{} + + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/jobs?policy_id=1", + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/jobs?policy_id=1", + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + // 400 policyID invalid + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/jobs?policy_id=0", + credential: sysAdmin, + }, + code: http.StatusBadRequest, + }, + // 400 policyID not found + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/jobs?policy_id=123", + credential: sysAdmin, + }, + code: http.StatusBadRequest, + }, + // 404 project not found + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/123/webhook/jobs?policy_id=1", + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 200 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/jobs?policy_id=1", + credential: sysAdmin, + }, + code: http.StatusOK, + }, + } + runCodeCheckingCases(t, cases...) +} diff --git a/src/core/api/notification_policy.go b/src/core/api/notification_policy.go new file mode 100755 index 000000000..c7acdbea2 --- /dev/null +++ b/src/core/api/notification_policy.go @@ -0,0 +1,384 @@ +package api + +import ( + "errors" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/rbac" + "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/pkg/notification" +) + +// NotificationPolicyAPI ... +type NotificationPolicyAPI struct { + BaseController + project *models.Project +} + +// notificationPolicyForUI defines the structure of notification policy info display in UI +type notificationPolicyForUI struct { + EventType string `json:"event_type"` + Enabled bool `json:"enabled"` + CreationTime *time.Time `json:"creation_time"` + LastTriggerTime *time.Time `json:"last_trigger_time,omitempty"` +} + +// Prepare ... +func (w *NotificationPolicyAPI) Prepare() { + w.BaseController.Prepare() + if !w.SecurityCtx.IsAuthenticated() { + w.SendUnAuthorizedError(errors.New("UnAuthorized")) + return + } + + pid, err := w.GetInt64FromPath(":pid") + if err != nil { + w.SendBadRequestError(fmt.Errorf("failed to get project ID: %v", err)) + return + } + if pid <= 0 { + w.SendBadRequestError(fmt.Errorf("invalid project ID: %d", pid)) + return + } + + project, err := w.ProjectMgr.Get(pid) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to get project %d: %v", pid, err)) + return + } + if project == nil { + w.SendNotFoundError(fmt.Errorf("project %d not found", pid)) + return + } + w.project = project +} + +// Get ... +func (w *NotificationPolicyAPI) Get() { + if !w.validateRBAC(rbac.ActionRead, w.project.ProjectID) { + return + } + + id, err := w.GetIDFromURL() + if err != nil { + w.SendBadRequestError(err) + return + } + + policy, err := notification.PolicyMgr.Get(id) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to get the notification policy %d: %v", id, err)) + return + } + if policy == nil { + w.SendNotFoundError(fmt.Errorf("notification policy %d not found", id)) + return + } + + if w.project.ProjectID != policy.ProjectID { + w.SendBadRequestError(fmt.Errorf("notification policy %d with projectID %d not belong to project %d in URL", id, policy.ProjectID, w.project.ProjectID)) + return + } + + w.WriteJSONData(policy) +} + +// Post ... +func (w *NotificationPolicyAPI) Post() { + if !w.validateRBAC(rbac.ActionCreate, w.project.ProjectID) { + return + } + + policy := &models.NotificationPolicy{} + isValid, err := w.DecodeJSONReqAndValidate(policy) + if !isValid { + w.SendBadRequestError(err) + return + } + + if !w.validateTargets(policy) { + return + } + + if !w.validateEventTypes(policy) { + return + } + + if policy.ID != 0 { + w.SendBadRequestError(fmt.Errorf("cannot accept policy creating request with ID: %d", policy.ID)) + return + } + + policy.Creator = w.SecurityCtx.GetUsername() + policy.ProjectID = w.project.ProjectID + + id, err := notification.PolicyMgr.Create(policy) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to create the notification policy: %v", err)) + return + } + w.Redirect(http.StatusCreated, strconv.FormatInt(id, 10)) +} + +// Put ... +func (w *NotificationPolicyAPI) Put() { + if !w.validateRBAC(rbac.ActionUpdate, w.project.ProjectID) { + return + } + + id, err := w.GetIDFromURL() + if id < 0 || err != nil { + w.SendBadRequestError(errors.New("invalid notification policy ID")) + return + } + + oriPolicy, err := notification.PolicyMgr.Get(id) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to get the notification policy %d: %v", id, err)) + return + } + if oriPolicy == nil { + w.SendNotFoundError(fmt.Errorf("notification policy %d not found", id)) + return + } + + policy := &models.NotificationPolicy{} + isValid, err := w.DecodeJSONReqAndValidate(policy) + if !isValid { + w.SendBadRequestError(err) + return + } + + if !w.validateTargets(policy) { + return + } + + if !w.validateEventTypes(policy) { + return + } + + if w.project.ProjectID != oriPolicy.ProjectID { + w.SendBadRequestError(fmt.Errorf("notification policy %d with projectID %d not belong to project %d in URL", id, oriPolicy.ProjectID, w.project.ProjectID)) + return + } + + policy.ID = id + policy.ProjectID = w.project.ProjectID + + if err = notification.PolicyMgr.Update(policy); err != nil { + w.SendInternalServerError(fmt.Errorf("failed to update the notification policy: %v", err)) + return + } +} + +// List ... +func (w *NotificationPolicyAPI) List() { + projectID := w.project.ProjectID + if !w.validateRBAC(rbac.ActionList, projectID) { + return + } + + res, err := notification.PolicyMgr.List(projectID) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to list notification policies by projectID %d: %v", projectID, err)) + return + } + + policies := []*models.NotificationPolicy{} + if res != nil { + for _, policy := range res { + policies = append(policies, policy) + } + } + + w.WriteJSONData(policies) +} + +// ListGroupByEventType lists notification policy trigger info grouped by event type for UI, +// displays event type, status(enabled/disabled), create time, last trigger time +func (w *NotificationPolicyAPI) ListGroupByEventType() { + projectID := w.project.ProjectID + if !w.validateRBAC(rbac.ActionList, projectID) { + return + } + + res, err := notification.PolicyMgr.List(projectID) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to list notification policies by projectID %d: %v", projectID, err)) + return + } + + policies, err := constructPolicyWithTriggerTime(res) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to list the notification policy trigger information: %v", err)) + return + } + w.WriteJSONData(policies) +} + +// Delete ... +func (w *NotificationPolicyAPI) Delete() { + projectID := w.project.ProjectID + if !w.validateRBAC(rbac.ActionDelete, projectID) { + return + } + + id, err := w.GetIDFromURL() + if id < 0 || err != nil { + w.SendBadRequestError(errors.New("invalid notification policy ID")) + return + } + + policy, err := notification.PolicyMgr.Get(id) + if err != nil { + w.SendInternalServerError(fmt.Errorf("failed to get the notification policy %d: %v", id, err)) + return + } + if policy == nil { + w.SendNotFoundError(fmt.Errorf("notification policy %d not found", id)) + return + } + + if projectID != policy.ProjectID { + w.SendBadRequestError(fmt.Errorf("notification policy %d with projectID %d not belong to project %d in URL", id, policy.ProjectID, projectID)) + return + } + + if err = notification.PolicyMgr.Delete(id); err != nil { + w.SendInternalServerError(fmt.Errorf("failed to delete notification policy %d: %v", id, err)) + return + } +} + +// Test ... +func (w *NotificationPolicyAPI) Test() { + projectID := w.project.ProjectID + if !w.validateRBAC(rbac.ActionCreate, projectID) { + return + } + + policy := &models.NotificationPolicy{} + isValid, err := w.DecodeJSONReqAndValidate(policy) + if !isValid { + w.SendBadRequestError(err) + return + } + + if !w.validateTargets(policy) { + return + } + + if err := notification.PolicyMgr.Test(policy); err != nil { + w.SendBadRequestError(fmt.Errorf("notification policy %s test failed: %v", policy.Name, err)) + return + } +} + +func (w *NotificationPolicyAPI) validateRBAC(action rbac.Action, projectID int64) bool { + if w.SecurityCtx.IsSysAdmin() { + return true + } + + project, err := w.ProjectMgr.Get(projectID) + if err != nil { + w.ParseAndHandleError(fmt.Sprintf("failed to get project %d", projectID), err) + return false + } + + resource := rbac.NewProjectNamespace(project.ProjectID).Resource(rbac.ResourceNotificationPolicy) + if !w.SecurityCtx.Can(action, resource) { + w.SendForbiddenError(errors.New(w.SecurityCtx.GetUsername())) + return false + } + return true +} + +func (w *NotificationPolicyAPI) validateTargets(policy *models.NotificationPolicy) bool { + if len(policy.Targets) == 0 { + w.SendBadRequestError(fmt.Errorf("empty notification target with policy %s", policy.Name)) + return false + } + + for _, target := range policy.Targets { + url, err := utils.ParseEndpoint(target.Address) + if err != nil { + w.SendBadRequestError(err) + return false + } + // Prevent SSRF security issue #3755 + target.Address = url.Scheme + "://" + url.Host + url.Path + + _, ok := notification.SupportedNotifyTypes[target.Type] + if !ok { + w.SendBadRequestError(fmt.Errorf("unsupport target type %s with policy %s", target.Type, policy.Name)) + return false + } + } + + return true +} + +func (w *NotificationPolicyAPI) validateEventTypes(policy *models.NotificationPolicy) bool { + if len(policy.EventTypes) == 0 { + w.SendBadRequestError(errors.New("empty event type")) + return false + } + + for _, eventType := range policy.EventTypes { + _, ok := notification.SupportedEventTypes[eventType] + if !ok { + w.SendBadRequestError(fmt.Errorf("unsupport event type %s", eventType)) + return false + } + } + + return true +} + +func getLastTriggerTimeGroupByEventType(eventType string, policyID int64) (time.Time, error) { + jobs, err := notification.JobMgr.ListJobsGroupByEventType(policyID) + if err != nil { + return time.Time{}, err + } + + for _, job := range jobs { + if eventType == job.EventType { + return job.CreationTime, nil + } + } + return time.Time{}, nil +} + +// constructPolicyWithTriggerTime construct notification policy information displayed in UI +// including event type, enabled, creation time, last trigger time +func constructPolicyWithTriggerTime(policies []*models.NotificationPolicy) ([]*notificationPolicyForUI, error) { + res := []*notificationPolicyForUI{} + if policies != nil { + for _, policy := range policies { + for _, t := range policy.EventTypes { + ply := ¬ificationPolicyForUI{ + EventType: t, + Enabled: policy.Enabled, + CreationTime: &policy.CreationTime, + } + if !policy.CreationTime.IsZero() { + ply.CreationTime = &policy.CreationTime + } + + ltTime, err := getLastTriggerTimeGroupByEventType(t, policy.ID) + if err != nil { + return nil, err + } + if !ltTime.IsZero() { + ply.LastTriggerTime = <Time + } + res = append(res, ply) + } + } + } + return res, nil +} diff --git a/src/core/api/notification_policy_test.go b/src/core/api/notification_policy_test.go new file mode 100644 index 000000000..a63f6b72e --- /dev/null +++ b/src/core/api/notification_policy_test.go @@ -0,0 +1,637 @@ +package api + +import ( + "net/http" + "testing" + + "github.com/pkg/errors" + + "github.com/goharbor/harbor/src/pkg/notification/model" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/pkg/notification" +) + +type fakedNotificationPlyMgr struct { +} + +func (f *fakedNotificationPlyMgr) Create(*models.NotificationPolicy) (int64, error) { + return 0, nil +} + +func (f *fakedNotificationPlyMgr) List(id int64) ([]*models.NotificationPolicy, error) { + return []*models.NotificationPolicy{ + { + ID: 1, + EventTypes: []string{ + model.EventTypePullImage, + model.EventTypePushImage, + }, + }, + }, nil +} + +func (f *fakedNotificationPlyMgr) Get(id int64) (*models.NotificationPolicy, error) { + switch id { + case 1: + return &models.NotificationPolicy{ID: 1, ProjectID: 1}, nil + case 2: + return &models.NotificationPolicy{ID: 2, ProjectID: 222}, nil + case 3: + return nil, errors.New("") + default: + return nil, nil + } +} + +func (f *fakedNotificationPlyMgr) GetByNameAndProjectID(string, int64) (*models.NotificationPolicy, error) { + return nil, nil +} + +func (f *fakedNotificationPlyMgr) Update(*models.NotificationPolicy) error { + + return nil +} + +func (f *fakedNotificationPlyMgr) Delete(int64) error { + return nil +} + +func (f *fakedNotificationPlyMgr) Test(*models.NotificationPolicy) error { + return nil +} + +func (f *fakedNotificationPlyMgr) GetRelatedPolices(int64, string) ([]*models.NotificationPolicy, error) { + return nil, nil +} + +func TestNotificationPolicyAPI_List(t *testing.T) { + policyCtl := notification.PolicyMgr + defer func() { + notification.PolicyMgr = policyCtl + }() + + notification.PolicyMgr = &fakedNotificationPlyMgr{} + + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/policies", + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/policies", + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + // 404 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/123/webhook/policies", + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 200 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/policies", + credential: sysAdmin, + }, + code: http.StatusOK, + }, + } + runCodeCheckingCases(t, cases...) + +} + +func TestNotificationPolicyAPI_Post(t *testing.T) { + policyCtl := notification.PolicyMgr + defer func() { + notification.PolicyMgr = policyCtl + }() + + notification.PolicyMgr = &fakedNotificationPlyMgr{} + + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies", + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies", + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + // 400 invalid json body + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies", + credential: sysAdmin, + bodyJSON: "invalid json body", + }, + code: http.StatusBadRequest, + }, + // 400 empty targets + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + Targets: []models.EventTarget{}, + }}, + code: http.StatusBadRequest, + }, + // 400 invalid event target address + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + EventTypes: []string{"pullImage", "pushImage", "deleteImage"}, + Targets: []models.EventTarget{ + { + Address: "tcp://127.0.0.1:8080", + }, + }, + }}, + code: http.StatusBadRequest, + }, + // 400 invalid event target type + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + EventTypes: []string{"pullImage", "pushImage", "deleteImage"}, + Targets: []models.EventTarget{ + { + Type: "smn", + Address: "http://127.0.0.1:8080", + }, + }, + }}, + code: http.StatusBadRequest, + }, + // 400 invalid event type + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + EventTypes: []string{"invalidType"}, + Targets: []models.EventTarget{ + { + Address: "tcp://127.0.0.1:8080", + }, + }, + }}, + code: http.StatusBadRequest, + }, + // 400 policy ID != 0 + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + ID: 111, + EventTypes: []string{"pullImage", "pushImage", "deleteImage"}, + Targets: []models.EventTarget{ + { + Type: "http", + Address: "http://10.173.32.58:9009", + AuthHeader: "xxxxxxxxx", + SkipCertVerify: true, + }, + }, + }, + }, + code: http.StatusBadRequest, + }, + // 201 + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + EventTypes: []string{"pullImage", "pushImage", "deleteImage"}, + Targets: []models.EventTarget{ + { + Type: "http", + Address: "http://10.173.32.58:9009", + AuthHeader: "xxxxxxxxx", + SkipCertVerify: true, + }, + }, + }, + }, + code: http.StatusCreated, + }, + } + runCodeCheckingCases(t, cases...) +} + +func TestNotificationPolicyAPI_Get(t *testing.T) { + policyCtl := notification.PolicyMgr + defer func() { + notification.PolicyMgr = policyCtl + }() + + notification.PolicyMgr = &fakedNotificationPlyMgr{} + + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/policies/111", + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/policies/111", + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + // 404 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/policies/1234", + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 400 projectID not match with projectID in URL + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/policies/2", + credential: sysAdmin, + }, + code: http.StatusBadRequest, + }, + // 500 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/policies/3", + credential: sysAdmin, + }, + code: http.StatusInternalServerError, + }, + // 200 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/policies/1", + credential: sysAdmin, + }, + code: http.StatusOK, + }, + } + runCodeCheckingCases(t, cases...) +} + +func TestNotificationPolicyAPI_Put(t *testing.T) { + policyCtl := notification.PolicyMgr + defer func() { + notification.PolicyMgr = policyCtl + }() + + notification.PolicyMgr = &fakedNotificationPlyMgr{} + + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/projects/1/webhook/policies/111", + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/projects/1/webhook/policies/111", + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + // 404 + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/projects/1/webhook/policies/1234", + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 400 invalid json body + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/projects/1/webhook/policies/1", + credential: sysAdmin, + bodyJSON: "invalidJSONBody", + }, + code: http.StatusBadRequest, + }, + // 400 empty targets + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/projects/1/webhook/policies/1", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + EventTypes: []string{"pullImage", "pushImage", "deleteImage"}, + Targets: []models.EventTarget{}, + }}, + code: http.StatusBadRequest, + }, + // 400 invalid event target address + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/projects/1/webhook/policies/1", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + EventTypes: []string{"pullImage", "pushImage", "deleteImage"}, + Targets: []models.EventTarget{ + { + Address: "tcp://127.0.0.1:8080", + }, + }, + }}, + code: http.StatusBadRequest, + }, + // 400 invalid event target type + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/projects/1/webhook/policies/1", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + EventTypes: []string{"pullImage", "pushImage", "deleteImage"}, + Targets: []models.EventTarget{ + { + Type: "smn", + Address: "http://127.0.0.1:8080", + }, + }, + }}, + code: http.StatusBadRequest, + }, + // 400 invalid event type + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/projects/1/webhook/policies/1", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + EventTypes: []string{"invalidType"}, + Targets: []models.EventTarget{ + { + Address: "tcp://127.0.0.1:8080", + }, + }, + }}, + code: http.StatusBadRequest, + }, + // 200 + { + request: &testingRequest{ + method: http.MethodPut, + url: "/api/projects/1/webhook/policies/1", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + Name: "imagePolicyTest", + EventTypes: []string{"pullImage", "pushImage", "deleteImage"}, + Targets: []models.EventTarget{ + { + Type: "http", + Address: "http://10.173.32.58:9009", + AuthHeader: "xxxxxxxxx", + SkipCertVerify: true, + }, + }, + }, + }, + code: http.StatusOK, + }, + } + runCodeCheckingCases(t, cases...) +} + +func TestNotificationPolicyAPI_Test(t *testing.T) { + policyCtl := notification.PolicyMgr + defer func() { + notification.PolicyMgr = policyCtl + }() + + notification.PolicyMgr = &fakedNotificationPlyMgr{} + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies/test", + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies/test", + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + // 404 + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/123/webhook/policies/test", + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 400 invalid json body + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies/test", + credential: sysAdmin, + bodyJSON: 1234125, + }, + code: http.StatusBadRequest, + }, + // 200 + { + request: &testingRequest{ + method: http.MethodPost, + url: "/api/projects/1/webhook/policies/test", + credential: sysAdmin, + bodyJSON: &models.NotificationPolicy{ + Targets: []models.EventTarget{ + { + Type: "http", + Address: "http://10.173.32.58:9009", + AuthHeader: "xxxxxxxxx", + SkipCertVerify: true, + }, + }, + }, + }, + code: http.StatusOK, + }, + } + runCodeCheckingCases(t, cases...) +} + +func TestNotificationPolicyAPI_ListGroupByEventType(t *testing.T) { + policyCtl := notification.PolicyMgr + jobMgr := notification.JobMgr + defer func() { + notification.PolicyMgr = policyCtl + notification.JobMgr = jobMgr + }() + + notification.PolicyMgr = &fakedNotificationPlyMgr{} + notification.JobMgr = &fakedNotificationJobMgr{} + + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/lasttrigger", + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/lasttrigger", + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + // 404 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/123/webhook/lasttrigger", + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 200 + { + request: &testingRequest{ + method: http.MethodGet, + url: "/api/projects/1/webhook/lasttrigger", + credential: sysAdmin, + }, + code: http.StatusOK, + }, + } + runCodeCheckingCases(t, cases...) +} + +func TestNotificationPolicyAPI_Delete(t *testing.T) { + policyCtl := notification.PolicyMgr + defer func() { + notification.PolicyMgr = policyCtl + }() + + notification.PolicyMgr = &fakedNotificationPlyMgr{} + + cases := []*codeCheckingCase{ + // 401 + { + request: &testingRequest{ + method: http.MethodDelete, + url: "/api/projects/1/webhook/policies/111", + }, + code: http.StatusUnauthorized, + }, + // 403 + { + request: &testingRequest{ + method: http.MethodDelete, + url: "/api/projects/1/webhook/policies/111", + credential: nonSysAdmin, + }, + code: http.StatusForbidden, + }, + // 404 + { + request: &testingRequest{ + method: http.MethodDelete, + url: "/api/projects/1/webhook/policies/1234", + credential: sysAdmin, + }, + code: http.StatusNotFound, + }, + // 400 projectID not match + { + request: &testingRequest{ + method: http.MethodDelete, + url: "/api/projects/1/webhook/policies/2", + credential: sysAdmin, + }, + code: http.StatusBadRequest, + }, + // 500 failed to get policy + { + request: &testingRequest{ + method: http.MethodDelete, + url: "/api/projects/1/webhook/policies/3", + credential: sysAdmin, + }, + code: http.StatusInternalServerError, + }, + // 200 + { + request: &testingRequest{ + method: http.MethodDelete, + url: "/api/projects/1/webhook/policies/1", + credential: sysAdmin, + }, + code: http.StatusOK, + }, + } + runCodeCheckingCases(t, cases...) +} diff --git a/src/core/api/repository.go b/src/core/api/repository.go old mode 100644 new mode 100755 index dd88c3ebb..5da092b10 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -38,6 +38,7 @@ import ( notarymodel "github.com/goharbor/harbor/src/common/utils/notary/model" "github.com/goharbor/harbor/src/common/utils/registry" "github.com/goharbor/harbor/src/core/config" + notifierEvt "github.com/goharbor/harbor/src/core/notifier/event" coreutils "github.com/goharbor/harbor/src/core/utils" "github.com/goharbor/harbor/src/pkg/scan" "github.com/goharbor/harbor/src/replication" @@ -339,6 +340,24 @@ func (ra *RepositoryAPI) Delete() { }(t) } + // build and publish image delete event + evt := ¬ifierEvt.Event{} + imgDelMetadata := ¬ifierEvt.ImageDelMetaData{ + Project: project, + Tags: tags, + RepoName: repoName, + OccurAt: time.Now(), + Operator: ra.SecurityCtx.GetUsername(), + } + if err := evt.Build(imgDelMetadata); err != nil { + // do not return when building event metadata failed + log.Errorf("failed to build image delete event metadata: %v", err) + } + if err := evt.Publish(); err != nil { + // do not return when publishing event failed + log.Errorf("failed to publish image delete event: %v", err) + } + exist, err := repositoryExist(repoName, rc) if err != nil { log.Errorf("failed to check the existence of repository %s: %v", repoName, err) diff --git a/src/core/api/systeminfo.go b/src/core/api/systeminfo.go index 140a688df..a0929d545 100644 --- a/src/core/api/systeminfo.go +++ b/src/core/api/systeminfo.go @@ -16,13 +16,13 @@ package api import ( "errors" + "fmt" "io/ioutil" "net/http" "os" "strings" "sync" - "fmt" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/common/dao" clairdao "github.com/goharbor/harbor/src/common/dao/clair" @@ -106,6 +106,7 @@ type GeneralInfo struct { RegistryStorageProviderName string `json:"registry_storage_provider_name"` ReadOnly bool `json:"read_only"` WithChartMuseum bool `json:"with_chartmuseum"` + NotificationEnable bool `json:"notification_enable"` } // GetVolumeInfo gets specific volume storage info. @@ -188,6 +189,7 @@ func (sia *SystemInfoAPI) GetGeneralInfo() { RegistryStorageProviderName: utils.SafeCastString(cfg[common.RegistryStorageProviderName]), ReadOnly: config.ReadOnly(), WithChartMuseum: config.WithChartMuseum(), + NotificationEnable: utils.SafeCastBool(cfg[common.NotificationEnable]), } if info.WithClair { info.ClairVulnStatus = getClairVulnStatus() diff --git a/src/core/config/config.go b/src/core/config/config.go old mode 100644 new mode 100755 index 94e6651ab..57c02bad1 --- a/src/core/config/config.go +++ b/src/core/config/config.go @@ -515,6 +515,11 @@ func OIDCSetting() (*models.OIDCSetting, error) { }, nil } +// NotificationEnable returns a bool to indicates if notification enabled in harbor +func NotificationEnable() bool { + return cfgMgr.Get(common.NotificationEnable).GetBool() +} + // QuotaSetting returns the setting of quota. func QuotaSetting() (*models.QuotaSetting, error) { if err := cfgMgr.Load(); err != nil { diff --git a/src/core/config/config_test.go b/src/core/config/config_test.go index b8d73cb9c..ae31c04bc 100644 --- a/src/core/config/config_test.go +++ b/src/core/config/config_test.go @@ -211,6 +211,7 @@ func TestConfig(t *testing.T) { localCoreURL := LocalCoreURL() assert.Equal("http://127.0.0.1:8080", localCoreURL) + assert.True(NotificationEnable()) } func currPath() string { diff --git a/src/core/main.go b/src/core/main.go old mode 100644 new mode 100755 index 1b00d8941..6ea199757 --- a/src/core/main.go +++ b/src/core/main.go @@ -22,11 +22,10 @@ import ( "strconv" "syscall" - "github.com/goharbor/harbor/src/common/job" - "github.com/astaxie/beego" _ "github.com/astaxie/beego/session/redis" "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/job" "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils/log" @@ -38,7 +37,9 @@ import ( "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/filter" "github.com/goharbor/harbor/src/core/middlewares" + _ "github.com/goharbor/harbor/src/core/notifier/topic" "github.com/goharbor/harbor/src/core/service/token" + "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/scheduler" "github.com/goharbor/harbor/src/replication" ) @@ -143,6 +144,9 @@ func main() { log.Fatalf("failed to init for replication: %v", err) } + log.Info("initializing notification...") + notification.Init() + filter.Init() beego.InsertFilter("/*", beego.BeforeRouter, filter.SecurityFilter) beego.InsertFilter("/*", beego.BeforeRouter, filter.ReadonlyFilter) diff --git a/src/core/notifier/event/event.go b/src/core/notifier/event/event.go new file mode 100644 index 000000000..088a8d2e7 --- /dev/null +++ b/src/core/notifier/event/event.go @@ -0,0 +1,154 @@ +package event + +import ( + "time" + + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/notifier" + "github.com/goharbor/harbor/src/core/notifier/model" + notifyModel "github.com/goharbor/harbor/src/pkg/notification/model" + "github.com/pkg/errors" +) + +// Event to publish +type Event struct { + Topic string + Data interface{} +} + +// Metadata is the event raw data to be processed +type Metadata interface { + Resolve(event *Event) error +} + +// ImageDelMetaData defines images deleting related event data +type ImageDelMetaData struct { + Project *models.Project + Tags []string + OccurAt time.Time + Operator string + RepoName string +} + +// Resolve image deleting metadata into common image event +func (i *ImageDelMetaData) Resolve(evt *Event) error { + data := &model.ImageEvent{ + EventType: notifyModel.EventTypeDeleteImage, + Project: i.Project, + OccurAt: i.OccurAt, + Operator: i.Operator, + RepoName: i.RepoName, + } + for _, t := range i.Tags { + res := &model.ImgResource{Tag: t} + data.Resource = append(data.Resource, res) + } + evt.Topic = model.DeleteImageTopic + evt.Data = data + return nil +} + +// ImagePushMetaData defines images pushing related event data +type ImagePushMetaData struct { + Project *models.Project + Tag string + Digest string + OccurAt time.Time + Operator string + RepoName string +} + +// Resolve image pushing metadata into common image event +func (i *ImagePushMetaData) Resolve(evt *Event) error { + data := &model.ImageEvent{ + EventType: notifyModel.EventTypePushImage, + Project: i.Project, + OccurAt: i.OccurAt, + Operator: i.Operator, + RepoName: i.RepoName, + Resource: []*model.ImgResource{ + { + Tag: i.Tag, + Digest: i.Digest, + }, + }, + } + + evt.Topic = model.PushImageTopic + evt.Data = data + return nil +} + +// ImagePullMetaData defines images pulling related event data +type ImagePullMetaData struct { + Project *models.Project + Tag string + Digest string + OccurAt time.Time + Operator string + RepoName string +} + +// Resolve image pulling metadata into common image event +func (i *ImagePullMetaData) Resolve(evt *Event) error { + data := &model.ImageEvent{ + EventType: notifyModel.EventTypePullImage, + Project: i.Project, + OccurAt: i.OccurAt, + Operator: i.Operator, + RepoName: i.RepoName, + Resource: []*model.ImgResource{ + { + Tag: i.Tag, + Digest: i.Digest, + }, + }, + } + + evt.Topic = model.PullImageTopic + evt.Data = data + return nil +} + +// HookMetaData defines hook notification related event data +type HookMetaData struct { + PolicyID int64 + EventType string + Target *models.EventTarget + Payload *model.Payload +} + +// Resolve hook metadata into hook event +func (h *HookMetaData) Resolve(evt *Event) error { + data := &model.HookEvent{ + PolicyID: h.PolicyID, + EventType: h.EventType, + Target: h.Target, + Payload: h.Payload, + } + + evt.Topic = h.Target.Type + evt.Data = data + return nil +} + +// Build an event by metadata +func (e *Event) Build(metadata ...Metadata) error { + for _, md := range metadata { + if err := md.Resolve(e); err != nil { + log.Debugf("failed to resolve event metadata: %v", md) + return errors.Wrap(err, "failed to resolve event metadata") + } + } + return nil +} + +// Publish an event +func (e *Event) Publish() error { + if err := notifier.Publish(e.Topic, e.Data); err != nil { + log.Debugf("failed to publish topic %s with event: %v", e.Topic, e.Data) + return errors.Wrap(err, "failed to publish event") + } + return nil +} diff --git a/src/core/notifier/event/event_test.go b/src/core/notifier/event/event_test.go new file mode 100644 index 000000000..21e0d8d23 --- /dev/null +++ b/src/core/notifier/event/event_test.go @@ -0,0 +1,212 @@ +package event + +import ( + "testing" + "time" + + "github.com/goharbor/harbor/src/common/models" + notifierModel "github.com/goharbor/harbor/src/core/notifier/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImagePushEvent_Build(t *testing.T) { + type args struct { + imgPushMetadata *ImagePushMetaData + hookMetadata *HookMetaData + } + + tests := []struct { + name string + args args + wantErr bool + want *Event + }{ + { + name: "Build Image Push Event", + args: args{ + imgPushMetadata: &ImagePushMetaData{ + Project: &models.Project{ProjectID: 1, Name: "library"}, + Tag: "v1.0", + Digest: "abcd", + OccurAt: time.Now(), + Operator: "admin", + RepoName: "library/alpine", + }, + }, + want: &Event{ + Topic: notifierModel.PushImageTopic, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := &Event{} + err := event.Build(tt.args.imgPushMetadata) + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + assert.Equal(t, tt.want.Topic, event.Topic) + }) + } +} + +func TestImagePullEvent_Build(t *testing.T) { + type args struct { + imgPullMetadata *ImagePullMetaData + } + + tests := []struct { + name string + args args + wantErr bool + want *Event + }{ + { + name: "Build Image Pull Event", + args: args{ + imgPullMetadata: &ImagePullMetaData{ + Project: &models.Project{ProjectID: 1, Name: "library"}, + Tag: "v1.0", + Digest: "abcd", + OccurAt: time.Now(), + Operator: "admin", + RepoName: "library/alpine", + }, + }, + want: &Event{ + Topic: notifierModel.PullImageTopic, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := &Event{} + err := event.Build(tt.args.imgPullMetadata) + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + assert.Equal(t, tt.want.Topic, event.Topic) + }) + } +} + +func TestImageDelEvent_Build(t *testing.T) { + type args struct { + imgDelMetadata *ImageDelMetaData + } + + tests := []struct { + name string + args args + wantErr bool + want *Event + }{ + { + name: "Build Image Delete Event", + args: args{ + imgDelMetadata: &ImageDelMetaData{ + Project: &models.Project{ProjectID: 1, Name: "library"}, + Tags: []string{"v1.0"}, + OccurAt: time.Now(), + Operator: "admin", + RepoName: "library/alpine", + }, + }, + want: &Event{ + Topic: notifierModel.DeleteImageTopic, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := &Event{} + err := event.Build(tt.args.imgDelMetadata) + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + assert.Equal(t, tt.want.Topic, event.Topic) + }) + } +} + +func TestHookEvent_Build(t *testing.T) { + type args struct { + hookMetadata *HookMetaData + } + + tests := []struct { + name string + args args + wantErr bool + want *Event + }{ + { + name: "Build HTTP Hook Event", + args: args{ + hookMetadata: &HookMetaData{ + PolicyID: 1, + EventType: "pushImage", + Target: &models.EventTarget{ + Type: "http", + Address: "http://127.0.0.1", + }, + Payload: nil, + }, + }, + want: &Event{ + Topic: notifierModel.WebhookTopic, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event := &Event{} + err := event.Build(tt.args.hookMetadata) + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + assert.Equal(t, tt.want.Topic, event.Topic) + }) + } +} + +func TestEvent_Publish(t *testing.T) { + type args struct { + event *Event + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Publish Error 1", + args: args{ + event: &Event{ + Topic: notifierModel.WebhookTopic, + Data: nil, + }, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.args.event.Publish() + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + }) + } +} diff --git a/src/core/notifier/handler/notification/http_handler.go b/src/core/notifier/handler/notification/http_handler.go new file mode 100755 index 000000000..9795a7c2b --- /dev/null +++ b/src/core/notifier/handler/notification/http_handler.go @@ -0,0 +1,59 @@ +package notification + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/goharbor/harbor/src/common/job/models" + "github.com/goharbor/harbor/src/core/notifier/model" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/pkg/notification" +) + +// HTTPHandler preprocess http event data and start the hook processing +type HTTPHandler struct { +} + +// Handle handles http event +func (h *HTTPHandler) Handle(value interface{}) error { + if value == nil { + return errors.New("HTTPHandler cannot handle nil value") + } + + event, ok := value.(*model.HookEvent) + if !ok || event == nil { + return errors.New("invalid notification http event") + } + + return h.process(event) +} + +// IsStateful ... +func (h *HTTPHandler) IsStateful() bool { + return false +} + +func (h *HTTPHandler) process(event *model.HookEvent) error { + j := &models.JobData{ + Metadata: &models.JobMetadata{ + JobKind: job.KindGeneric, + }, + } + j.Name = job.WebhookJob + + payload, err := json.Marshal(event.Payload) + if err != nil { + return fmt.Errorf("marshal from payload %v failed: %v", event.Payload, err) + } + + j.Parameters = map[string]interface{}{ + "payload": string(payload), + "address": event.Target.Address, + // Users can define a auth header in http statement in notification(webhook) policy. + // So it will be sent in header in http request. + "auth_header": event.Target.AuthHeader, + "skip_cert_verify": event.Target.SkipCertVerify, + } + return notification.HookManager.StartHook(event, j) +} diff --git a/src/core/notifier/handler/notification/http_handler_test.go b/src/core/notifier/handler/notification/http_handler_test.go new file mode 100644 index 000000000..c7d5ef3ae --- /dev/null +++ b/src/core/notifier/handler/notification/http_handler_test.go @@ -0,0 +1,97 @@ +package notification + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/goharbor/harbor/src/common/job/models" + cModels "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/core/notifier/event" + "github.com/goharbor/harbor/src/core/notifier/model" + "github.com/goharbor/harbor/src/pkg/notification" + "github.com/stretchr/testify/require" +) + +type fakedHookManager struct { +} + +func (f *fakedHookManager) StartHook(event *model.HookEvent, job *models.JobData) error { + return nil +} + +func TestHTTPHandler_Handle(t *testing.T) { + hookMgr := notification.HookManager + defer func() { + notification.HookManager = hookMgr + }() + notification.HookManager = &fakedHookManager{} + + handler := &HTTPHandler{} + + type args struct { + event *event.Event + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "HTTPHandler_Handle Want Error 1", + args: args{ + event: &event.Event{ + Topic: "http", + Data: nil, + }, + }, + wantErr: true, + }, + { + name: "HTTPHandler_Handle Want Error 2", + args: args{ + event: &event.Event{ + Topic: "http", + Data: &model.ImageEvent{}, + }, + }, + wantErr: true, + }, + { + name: "HTTPHandler_Handle 1", + args: args{ + event: &event.Event{ + Topic: "http", + Data: &model.HookEvent{ + PolicyID: 1, + EventType: "pushImage", + Target: &cModels.EventTarget{ + Type: "http", + Address: "http://127.0.0.1:8080", + }, + Payload: &model.Payload{ + OccurAt: time.Now().Unix(), + }, + }, + }, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := handler.Handle(tt.args.event.Data) + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + }) + } +} + +func TestHTTPHandler_IsStateful(t *testing.T) { + handler := &HTTPHandler{} + assert.False(t, handler.IsStateful()) +} diff --git a/src/core/notifier/handler/notification/image_handler.go b/src/core/notifier/handler/notification/image_handler.go new file mode 100644 index 000000000..f9dc12468 --- /dev/null +++ b/src/core/notifier/handler/notification/image_handler.go @@ -0,0 +1,18 @@ +package notification + +// ImagePreprocessHandler preprocess image event data +type ImagePreprocessHandler struct { +} + +// Handle preprocess image event data and then publish hook event +func (h *ImagePreprocessHandler) Handle(value interface{}) error { + if err := preprocessAndSendImageHook(value); err != nil { + return err + } + return nil +} + +// IsStateful ... +func (h *ImagePreprocessHandler) IsStateful() bool { + return false +} diff --git a/src/core/notifier/handler/notification/image_handler_test.go b/src/core/notifier/handler/notification/image_handler_test.go new file mode 100644 index 000000000..ae1696b44 --- /dev/null +++ b/src/core/notifier/handler/notification/image_handler_test.go @@ -0,0 +1,193 @@ +package notification + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/notifier/model" + "github.com/goharbor/harbor/src/pkg/notification" + notificationModel "github.com/goharbor/harbor/src/pkg/notification/model" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +type fakedNotificationPlyMgr struct { +} + +func (f *fakedNotificationPlyMgr) Create(*models.NotificationPolicy) (int64, error) { + return 0, nil +} + +func (f *fakedNotificationPlyMgr) List(id int64) ([]*models.NotificationPolicy, error) { + return nil, nil +} + +func (f *fakedNotificationPlyMgr) Get(id int64) (*models.NotificationPolicy, error) { + return nil, nil +} + +func (f *fakedNotificationPlyMgr) GetByNameAndProjectID(string, int64) (*models.NotificationPolicy, error) { + return nil, nil +} + +func (f *fakedNotificationPlyMgr) Update(*models.NotificationPolicy) error { + return nil +} + +func (f *fakedNotificationPlyMgr) Delete(int64) error { + return nil +} + +func (f *fakedNotificationPlyMgr) Test(*models.NotificationPolicy) error { + return nil +} + +func (f *fakedNotificationPlyMgr) GetRelatedPolices(id int64, eventType string) ([]*models.NotificationPolicy, error) { + if id == 1 { + return []*models.NotificationPolicy{ + { + ID: 1, + EventTypes: []string{ + notificationModel.EventTypePullImage, + notificationModel.EventTypePushImage, + }, + Targets: []models.EventTarget{ + { + Type: "http", + Address: "http://127.0.0.1:8080", + }, + }, + }, + }, nil + } + if id == 2 { + return nil, nil + } + return nil, errors.New("") +} + +func TestMain(m *testing.M) { + dao.PrepareTestForPostgresSQL() + os.Exit(m.Run()) +} + +func TestImagePreprocessHandler_Handle(t *testing.T) { + PolicyMgr := notification.PolicyMgr + defer func() { + notification.PolicyMgr = PolicyMgr + }() + notification.PolicyMgr = &fakedNotificationPlyMgr{} + + handler := &ImagePreprocessHandler{} + config.Init() + + type args struct { + data interface{} + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "ImagePreprocessHandler Want Error 1", + args: args{ + data: nil, + }, + wantErr: true, + }, + { + name: "ImagePreprocessHandler Want Error 2", + args: args{ + data: &model.ImageEvent{}, + }, + wantErr: true, + }, + { + name: "ImagePreprocessHandler Want Error 3", + args: args{ + data: &model.ImageEvent{ + Resource: []*model.ImgResource{ + { + Tag: "v1.0", + }, + }, + Project: &models.Project{ + ProjectID: 3, + }, + }, + }, + wantErr: true, + }, + { + name: "ImagePreprocessHandler Want Error 4", + args: args{ + data: &model.ImageEvent{ + Resource: []*model.ImgResource{ + { + Tag: "v1.0", + }, + }, + Project: &models.Project{ + ProjectID: 1, + }, + }, + }, + wantErr: true, + }, + // No handlers registered for handling topic http + { + name: "ImagePreprocessHandler Want Error 5", + args: args{ + data: &model.ImageEvent{ + RepoName: "test/alpine", + Resource: []*model.ImgResource{ + { + Tag: "v1.0", + }, + }, + Project: &models.Project{ + ProjectID: 1, + }, + }, + }, + wantErr: true, + }, + { + name: "ImagePreprocessHandler 2", + args: args{ + data: &model.ImageEvent{ + Resource: []*model.ImgResource{ + { + Tag: "v1.0", + }, + }, + Project: &models.Project{ + ProjectID: 2, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := handler.Handle(tt.args.data) + if tt.wantErr { + require.NotNil(t, err, "Error: %s", err) + return + } + assert.Nil(t, err) + }) + } +} + +func TestImagePreprocessHandler_IsStateful(t *testing.T) { + handler := &ImagePreprocessHandler{} + assert.False(t, handler.IsStateful()) +} diff --git a/src/core/notifier/handler/notification/processor.go b/src/core/notifier/handler/notification/processor.go new file mode 100644 index 000000000..513640fd2 --- /dev/null +++ b/src/core/notifier/handler/notification/processor.go @@ -0,0 +1,174 @@ +package notification + +import ( + "errors" + "fmt" + "strings" + + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/notifier/event" + notifyModel "github.com/goharbor/harbor/src/core/notifier/model" + "github.com/goharbor/harbor/src/pkg/notification" +) + +// getNameFromImgRepoFullName gets image name from repo full name with format `repoName/imageName` +func getNameFromImgRepoFullName(repo string) string { + idx := strings.Index(repo, "/") + return repo[idx+1:] +} + +func buildImageResourceURL(extURL, repoName, tag string) (string, error) { + resURL := fmt.Sprintf("%s/%s:%s", extURL, repoName, tag) + return resURL, nil +} + +func constructImagePayload(event *notifyModel.ImageEvent) (*notifyModel.Payload, error) { + repoName := event.RepoName + if repoName == "" { + return nil, fmt.Errorf("invalid %s event with empty repo name", event.EventType) + } + + repoType := models.ProjectPrivate + if event.Project.IsPublic() { + repoType = models.ProjectPublic + } + + imageName := getNameFromImgRepoFullName(repoName) + + payload := ¬ifyModel.Payload{ + Type: event.EventType, + OccurAt: event.OccurAt.Unix(), + EventData: ¬ifyModel.EventData{ + Repository: ¬ifyModel.Repository{ + Name: imageName, + Namespace: event.Project.Name, + RepoFullName: repoName, + RepoType: repoType, + }, + }, + Operator: event.Operator, + } + + repoRecord, err := dao.GetRepositoryByName(repoName) + if err != nil { + log.Errorf("failed to get repository with name %s: %v", repoName, err) + return nil, err + } + // once repo has been delete, cannot ensure to get repo record + if repoRecord == nil { + log.Debugf("cannot find repository info with repo %s", repoName) + } else { + payload.EventData.Repository.DateCreated = repoRecord.CreationTime.Unix() + } + + extURL, err := config.ExtURL() + if err != nil { + return nil, fmt.Errorf("get external endpoint failed: %v", err) + } + + for _, res := range event.Resource { + tag := res.Tag + digest := res.Digest + + if tag == "" { + log.Errorf("invalid notification event with empty tag: %v", event) + continue + } + + resURL, err := buildImageResourceURL(extURL, event.RepoName, tag) + if err != nil { + log.Errorf("get resource URL failed: %v", err) + continue + } + + resource := ¬ifyModel.Resource{ + Tag: tag, + Digest: digest, + ResourceURL: resURL, + } + payload.EventData.Resources = append(payload.EventData.Resources, resource) + } + + return payload, nil +} + +// send hook by publishing topic of specified target type(notify type) +func sendHookWithPolicies(policies []*models.NotificationPolicy, payload *notifyModel.Payload, eventType string) error { + for _, ply := range policies { + targets := ply.Targets + for _, target := range targets { + evt := &event.Event{} + hookMetadata := &event.HookMetaData{ + EventType: eventType, + PolicyID: ply.ID, + Payload: payload, + Target: &target, + } + if err := evt.Build(hookMetadata); err != nil { + log.Errorf("failed to build hook notify event metadata: %v", err) + return err + } + if err := evt.Publish(); err != nil { + log.Errorf("failed to publish hook notify event: %v", err) + return err + } + + log.Debugf("published image event %s by topic %s", payload.Type, target.Type) + } + } + return nil +} + +func resolveImageEventData(value interface{}) (*notifyModel.ImageEvent, error) { + imgEvent, ok := value.(*notifyModel.ImageEvent) + if !ok || imgEvent == nil { + return nil, errors.New("invalid image event") + } + + if len(imgEvent.Resource) == 0 { + return nil, fmt.Errorf("empty event resouece data in image event: %v", imgEvent) + } + + return imgEvent, nil +} + +// preprocessAndSendImageHook preprocess image event data and send hook by notification policy target +func preprocessAndSendImageHook(value interface{}) error { + // if global notification configured disabled, return directly + if !config.NotificationEnable() { + log.Debug("notification feature is not enabled") + return nil + } + + imgEvent, err := resolveImageEventData(value) + if err != nil { + return err + } + + policies, err := notification.PolicyMgr.GetRelatedPolices(imgEvent.Project.ProjectID, imgEvent.EventType) + if err != nil { + log.Errorf("failed to find policy for %s event: %v", imgEvent.EventType, err) + return err + } + // if cannot find policy including event type in project, return directly + if len(policies) == 0 { + log.Debugf("cannot find policy for %s event: %v", imgEvent.EventType, imgEvent) + return nil + } + + payload, err := constructImagePayload(imgEvent) + if err != nil { + return err + } + + err = sendHookWithPolicies(policies, payload, imgEvent.EventType) + if err != nil { + return err + } + + return nil + +} diff --git a/src/core/notifier/model/event.go b/src/core/notifier/model/event.go new file mode 100755 index 000000000..67889e751 --- /dev/null +++ b/src/core/notifier/model/event.go @@ -0,0 +1,61 @@ +package model + +import ( + "time" + + "github.com/goharbor/harbor/src/common/models" +) + +// ImageEvent is image related event data to publish +type ImageEvent struct { + EventType string + Project *models.Project + Resource []*ImgResource + OccurAt time.Time + Operator string + RepoName string +} + +// ImgResource include image digest and tag +type ImgResource struct { + Digest string + Tag string +} + +// HookEvent is hook related event data to publish +type HookEvent struct { + PolicyID int64 + EventType string + Target *models.EventTarget + Payload *Payload +} + +// Payload of notification event +type Payload struct { + Type string `json:"type"` + OccurAt int64 `json:"occur_at"` + EventData *EventData `json:"event_data,omitempty"` + Operator string `json:"operator"` +} + +// EventData of notification event payload +type EventData struct { + Resources []*Resource `json:"resources"` + Repository *Repository `json:"repository"` +} + +// Resource describe infos of resource triggered notification +type Resource struct { + Digest string `json:"digest,omitempty"` + Tag string `json:"tag"` + ResourceURL string `json:"resource_url,omitempty"` +} + +// Repository info of notification event +type Repository struct { + DateCreated int64 `json:"date_created,omitempty"` + Name string `json:"name"` + Namespace string `json:"namespace"` + RepoFullName string `json:"repo_full_name"` + RepoType string `json:"repo_type"` +} diff --git a/src/core/notifier/model/topic.go b/src/core/notifier/model/topic.go new file mode 100644 index 000000000..7278858b8 --- /dev/null +++ b/src/core/notifier/model/topic.go @@ -0,0 +1,26 @@ +package model + +// Define global topic names +const ( + // PushImageTopic is topic for push image event + PushImageTopic = "OnPushImage" + // PullImageTopic is topic for pull image event + PullImageTopic = "OnPullImage" + // DeleteImageTopic is topic for delete image event + DeleteImageTopic = "OnDeleteImage" + // UploadChartTopic is topic for upload chart event + UploadChartTopic = "OnUploadChart" + // DownloadChartTopic is topic for download chart event + DownloadChartTopic = "OnDownloadChart" + // DeleteChartTopic is topic for delete chart event + DeleteChartTopic = "OnDeleteChart" + // ScanningFailedTopic is topic for scanning failed event + ScanningFailedTopic = "OnScanningFailed" + // ScanningCompletedTopic is topic for scanning completed event + ScanningCompletedTopic = "OnScanningCompleted" + + // WebhookTopic is topic for sending webhook payload + WebhookTopic = "http" + // EmailTopic is topic for sending email payload + EmailTopic = "email" +) diff --git a/src/core/notifier/topic/topics.go b/src/core/notifier/topic/topics.go new file mode 100644 index 000000000..2762da259 --- /dev/null +++ b/src/core/notifier/topic/topics.go @@ -0,0 +1,28 @@ +package topic + +import ( + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/notifier" + "github.com/goharbor/harbor/src/core/notifier/handler/notification" + "github.com/goharbor/harbor/src/core/notifier/model" +) + +// Subscribe topics +func init() { + handlersMap := map[string][]notifier.NotificationHandler{ + model.PushImageTopic: {¬ification.ImagePreprocessHandler{}}, + model.PullImageTopic: {¬ification.ImagePreprocessHandler{}}, + model.DeleteImageTopic: {¬ification.ImagePreprocessHandler{}}, + model.WebhookTopic: {¬ification.HTTPHandler{}}, + } + + for t, handlers := range handlersMap { + for _, handler := range handlers { + if err := notifier.Subscribe(t, handler); err != nil { + log.Errorf("failed to subscribe topic %s: %v", t, err) + continue + } + log.Debugf("topic %s is subscribed", t) + } + } +} diff --git a/src/core/notifier/topics.go b/src/core/notifier/topics.go deleted file mode 100644 index 23aca94cf..000000000 --- a/src/core/notifier/topics.go +++ /dev/null @@ -1,11 +0,0 @@ -package notifier - -import ( - "github.com/goharbor/harbor/src/common" -) - -// Define global topic names -const ( - // ScanAllPolicyTopic is for notifying the change of scanning all policy. - ScanAllPolicyTopic = common.ScanAllPolicy -) diff --git a/src/core/router.go b/src/core/router.go old mode 100644 new mode 100755 index d474229bb..04fd1a173 --- a/src/core/router.go +++ b/src/core/router.go @@ -15,6 +15,7 @@ package main import ( + "github.com/astaxie/beego" "github.com/goharbor/harbor/src/common" "github.com/goharbor/harbor/src/core/api" "github.com/goharbor/harbor/src/core/config" @@ -24,8 +25,6 @@ import ( "github.com/goharbor/harbor/src/core/service/notifications/registry" "github.com/goharbor/harbor/src/core/service/notifications/scheduler" "github.com/goharbor/harbor/src/core/service/token" - - "github.com/astaxie/beego" ) func initRouters() { @@ -114,6 +113,14 @@ func initRouters() { beego.Router("/api/replication/policies", &api.ReplicationPolicyAPI{}, "get:List;post:Create") beego.Router("/api/replication/policies/:id([0-9]+)", &api.ReplicationPolicyAPI{}, "get:Get;put:Update;delete:Delete") + beego.Router("/api/projects/:pid([0-9]+)/webhook/policies", &api.NotificationPolicyAPI{}, "get:List;post:Post") + beego.Router("/api/projects/:pid([0-9]+)/webhook/policies/:id([0-9]+)", &api.NotificationPolicyAPI{}) + beego.Router("/api/projects/:pid([0-9]+)/webhook/policies/test", &api.NotificationPolicyAPI{}, "post:Test") + + beego.Router("/api/projects/:pid([0-9]+)/webhook/lasttrigger", &api.NotificationPolicyAPI{}, "get:ListGroupByEventType") + + beego.Router("/api/projects/:pid([0-9]+)/webhook/jobs/", &api.NotificationJobAPI{}, "get:List") + beego.Router("/api/internal/configurations", &api.ConfigAPI{}, "get:GetInternalConfig;put:Put") beego.Router("/api/configurations", &api.ConfigAPI{}, "get:Get;put:Put") beego.Router("/api/statistics", &api.StatisticAPI{}) @@ -134,6 +141,7 @@ func initRouters() { beego.Router("/service/notifications/jobs/adminjob/:id([0-9]+)", &admin.Handler{}, "post:HandleAdminJob") beego.Router("/service/notifications/jobs/replication/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationScheduleJob") beego.Router("/service/notifications/jobs/replication/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleReplicationTask") + beego.Router("/service/notifications/jobs/webhook/:id([0-9]+)", &jobs.Handler{}, "post:HandleNotificationJob") beego.Router("/service/notifications/jobs/retention/task/:id([0-9]+)", &jobs.Handler{}, "post:HandleRetentionTask") beego.Router("/service/notifications/schedules/:id([0-9]+)", &scheduler.Handler{}, "post:Handle") beego.Router("/service/token", &token.Handler{}) diff --git a/src/core/service/notifications/jobs/handler.go b/src/core/service/notifications/jobs/handler.go old mode 100644 new mode 100755 index d35147f9d..a5b923aba --- a/src/core/service/notifications/jobs/handler.go +++ b/src/core/service/notifications/jobs/handler.go @@ -24,6 +24,7 @@ import ( "github.com/goharbor/harbor/src/common/models" "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/api" + "github.com/goharbor/harbor/src/pkg/notification" "github.com/goharbor/harbor/src/pkg/retention" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/replication/operation/hook" @@ -146,3 +147,17 @@ func (h *Handler) HandleRetentionTask() { return } } + +// HandleNotificationJob handles the hook of notification job +func (h *Handler) HandleNotificationJob() { + log.Debugf("received notification job status update event: job-%d, status-%s", h.id, h.status) + if err := notification.JobMgr.Update(&models.NotificationJob{ + ID: h.id, + Status: h.status, + UpdateTime: time.Now(), + }, "Status", "UpdateTime"); err != nil { + log.Errorf("Failed to update notification job status, id: %d, status: %s", h.id, h.status) + h.SendInternalServerError(err) + return + } +} diff --git a/src/core/service/notifications/registry/handler.go b/src/core/service/notifications/registry/handler.go old mode 100644 new mode 100755 index 0afd5233c..351938a45 --- a/src/core/service/notifications/registry/handler.go +++ b/src/core/service/notifications/registry/handler.go @@ -27,6 +27,7 @@ import ( "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/api" "github.com/goharbor/harbor/src/core/config" + notifierEvt "github.com/goharbor/harbor/src/core/notifier/event" coreutils "github.com/goharbor/harbor/src/core/utils" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/replication/adapter" @@ -116,6 +117,25 @@ func (n *NotificationHandler) Post() { return } + // build and publish image push event + evt := ¬ifierEvt.Event{} + imgPushMetadata := ¬ifierEvt.ImagePushMetaData{ + Project: pro, + Tag: tag, + Digest: event.Target.Digest, + RepoName: event.Target.Repository, + OccurAt: time.Now(), + Operator: event.Actor.Name, + } + if err := evt.Build(imgPushMetadata); err != nil { + // do not return when building event metadata failed + log.Errorf("failed to build image push event metadata: %v", err) + } + if err := evt.Publish(); err != nil { + // do not return when publishing event failed + log.Errorf("failed to publish image push event: %v", err) + } + // TODO: handle image delete event and chart event go func() { e := &rep_event.Event{ @@ -148,6 +168,25 @@ func (n *NotificationHandler) Post() { } } if action == "pull" { + // build and publish image pull event + evt := ¬ifierEvt.Event{} + imgPullMetadata := ¬ifierEvt.ImagePullMetaData{ + Project: pro, + Tag: tag, + Digest: event.Target.Digest, + RepoName: event.Target.Repository, + OccurAt: time.Now(), + Operator: event.Actor.Name, + } + if err := evt.Build(imgPullMetadata); err != nil { + // do not return when building event metadata failed + log.Errorf("failed to build image push event metadata: %v", err) + } + if err := evt.Publish(); err != nil { + // do not return when publishing event failed + log.Errorf("failed to publish image pull event: %v", err) + } + go func() { log.Debugf("Increase the repository %s pull count.", repository) if err := dao.IncreasePullCount(repository); err != nil { diff --git a/src/pkg/notification/hook/hook.go b/src/pkg/notification/hook/hook.go new file mode 100755 index 000000000..8524a0e0e --- /dev/null +++ b/src/pkg/notification/hook/hook.go @@ -0,0 +1,85 @@ +package hook + +import ( + "encoding/json" + "fmt" + "time" + + cJob "github.com/goharbor/harbor/src/common/job" + "github.com/goharbor/harbor/src/common/job/models" + cModels "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/config" + "github.com/goharbor/harbor/src/core/notifier/model" + "github.com/goharbor/harbor/src/core/utils" + "github.com/goharbor/harbor/src/pkg/notification/job" + "github.com/goharbor/harbor/src/pkg/notification/job/manager" +) + +// Manager send hook +type Manager interface { + StartHook(*model.HookEvent, *models.JobData) error +} + +// DefaultManager ... +type DefaultManager struct { + jobMgr job.Manager + client cJob.Client +} + +// NewHookManager ... +func NewHookManager() *DefaultManager { + return &DefaultManager{ + jobMgr: manager.NewDefaultManager(), + client: utils.GetJobServiceClient(), + } +} + +// StartHook create a notification job record in database, and submit it to jobservice +func (hm *DefaultManager) StartHook(event *model.HookEvent, data *models.JobData) error { + payload, err := json.Marshal(event.Payload) + if err != nil { + return err + } + + t := time.Now() + id, err := hm.jobMgr.Create(&cModels.NotificationJob{ + PolicyID: event.PolicyID, + EventType: event.EventType, + NotifyType: event.Target.Type, + Status: cModels.JobPending, + CreationTime: t, + UpdateTime: t, + JobDetail: string(payload), + }) + if err != nil { + return fmt.Errorf("failed to create the job record for notification based on policy %d: %v", event.PolicyID, err) + } + statusHookURL := fmt.Sprintf("%s/service/notifications/jobs/webhook/%d", config.InternalCoreURL(), id) + data.StatusHook = statusHookURL + + log.Debugf("created a notification job %d for the policy %d", id, event.PolicyID) + + // submit hook job to jobservice + jobUUID, err := hm.client.SubmitJob(data) + if err != nil { + log.Errorf("failed to submit job with notification event: %v", err) + e := hm.jobMgr.Update(&cModels.NotificationJob{ + ID: id, + Status: cModels.JobError, + }, "Status") + if e != nil { + log.Errorf("failed to update the notification job status %d: %v", id, e) + } + return err + } + + if err = hm.jobMgr.Update(&cModels.NotificationJob{ + ID: id, + UUID: jobUUID, + }, "UUID"); err != nil { + log.Errorf("failed to update the notification job %d: %v", id, err) + return err + } + return nil +} diff --git a/src/pkg/notification/job/manager.go b/src/pkg/notification/job/manager.go new file mode 100755 index 000000000..da8ac8027 --- /dev/null +++ b/src/pkg/notification/job/manager.go @@ -0,0 +1,20 @@ +package job + +import ( + "github.com/goharbor/harbor/src/common/models" +) + +// Manager manages notification jobs recorded in database +type Manager interface { + // Create create a notification job + Create(job *models.NotificationJob) (int64, error) + + // List list notification jobs + List(...*models.NotificationJobQuery) (int64, []*models.NotificationJob, error) + + // Update update notification job + Update(job *models.NotificationJob, props ...string) error + + // ListJobsGroupByEventType lists last triggered jobs group by event type + ListJobsGroupByEventType(policyID int64) ([]*models.NotificationJob, error) +} diff --git a/src/pkg/notification/job/manager/manager.go b/src/pkg/notification/job/manager/manager.go new file mode 100755 index 000000000..8db3aecd6 --- /dev/null +++ b/src/pkg/notification/job/manager/manager.go @@ -0,0 +1,55 @@ +package manager + +import ( + "fmt" + + "github.com/goharbor/harbor/src/common/dao/notification" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/pkg/notification/job" +) + +// DefaultManager .. +type DefaultManager struct { +} + +// NewDefaultManager ... +func NewDefaultManager() job.Manager { + return &DefaultManager{} +} + +// Create ... +func (d *DefaultManager) Create(job *models.NotificationJob) (int64, error) { + return notification.AddNotificationJob(job) +} + +// List ... +func (d *DefaultManager) List(query ...*models.NotificationJobQuery) (int64, []*models.NotificationJob, error) { + total, err := notification.GetTotalCountOfNotificationJobs(query...) + if err != nil { + return 0, nil, err + } + + executions, err := notification.GetNotificationJobs(query...) + if err != nil { + return 0, nil, err + } + return total, executions, nil +} + +// Update ... +func (d *DefaultManager) Update(job *models.NotificationJob, props ...string) error { + n, err := notification.UpdateNotificationJob(job, props...) + if err != nil { + return err + } + + if n == 0 { + return fmt.Errorf("execution %d not found", job.ID) + } + return nil +} + +// ListJobsGroupByEventType lists last triggered jobs group by event type +func (d *DefaultManager) ListJobsGroupByEventType(policyID int64) ([]*models.NotificationJob, error) { + return notification.GetLastTriggerJobsGroupByEventType(policyID) +} diff --git a/src/pkg/notification/job/manager/manager_test.go b/src/pkg/notification/job/manager/manager_test.go new file mode 100644 index 000000000..a373f618b --- /dev/null +++ b/src/pkg/notification/job/manager/manager_test.go @@ -0,0 +1,22 @@ +package manager + +import ( + "reflect" + "testing" +) + +func TestNewDefaultManger(t *testing.T) { + tests := []struct { + name string + want *DefaultManager + }{ + {want: &DefaultManager{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewDefaultManager(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewDefaultManager() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/pkg/notification/model/const.go b/src/pkg/notification/model/const.go new file mode 100644 index 000000000..51b8288ee --- /dev/null +++ b/src/pkg/notification/model/const.go @@ -0,0 +1,16 @@ +package model + +// const definitions +const ( + EventTypePushImage = "pushImage" + EventTypePullImage = "pullImage" + EventTypeDeleteImage = "deleteImage" + EventTypeUploadChart = "uploadChart" + EventTypeDeleteChart = "deleteChart" + EventTypeDownloadChart = "downloadChart" + EventTypeScanningCompleted = "scanningCompleted" + EventTypeScanningFailed = "scanningFailed" + EventTypeTestEndpoint = "testEndpoint" + + NotifyTypeHTTP = "http" +) diff --git a/src/pkg/notification/notification.go b/src/pkg/notification/notification.go new file mode 100755 index 000000000..4de7479d1 --- /dev/null +++ b/src/pkg/notification/notification.go @@ -0,0 +1,63 @@ +package notification + +import ( + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/pkg/notification/hook" + "github.com/goharbor/harbor/src/pkg/notification/job" + jobMgr "github.com/goharbor/harbor/src/pkg/notification/job/manager" + "github.com/goharbor/harbor/src/pkg/notification/model" + "github.com/goharbor/harbor/src/pkg/notification/policy" + "github.com/goharbor/harbor/src/pkg/notification/policy/manager" +) + +var ( + // PolicyMgr is a global notification policy manager + PolicyMgr policy.Manager + + // JobMgr is a notification job controller + JobMgr job.Manager + + // HookManager is a hook manager + HookManager hook.Manager + + // SupportedEventTypes is a map to store supported event type, eg. pushImage, pullImage etc + SupportedEventTypes map[string]struct{} + + // SupportedNotifyTypes is a map to store notification type, eg. HTTP, Email etc + SupportedNotifyTypes map[string]struct{} +) + +// Init ... +func Init() { + // init notification policy manager + PolicyMgr = manager.NewDefaultManger() + // init hook manager + HookManager = hook.NewHookManager() + // init notification job manager + JobMgr = jobMgr.NewDefaultManager() + + SupportedEventTypes = make(map[string]struct{}) + SupportedNotifyTypes = make(map[string]struct{}) + + initSupportedEventType( + model.EventTypePushImage, model.EventTypePullImage, model.EventTypeDeleteImage, + model.EventTypeUploadChart, model.EventTypeDeleteChart, model.EventTypeDownloadChart, + model.EventTypeScanningCompleted, model.EventTypeScanningFailed, + ) + + initSupportedNotifyType(model.NotifyTypeHTTP) + + log.Info("notification initialization completed") +} + +func initSupportedEventType(eventTypes ...string) { + for _, eventType := range eventTypes { + SupportedEventTypes[eventType] = struct{}{} + } +} + +func initSupportedNotifyType(notifyTypes ...string) { + for _, notifyType := range notifyTypes { + SupportedNotifyTypes[notifyType] = struct{}{} + } +} diff --git a/src/pkg/notification/policy/manager.go b/src/pkg/notification/policy/manager.go new file mode 100755 index 000000000..d08ffc3bd --- /dev/null +++ b/src/pkg/notification/policy/manager.go @@ -0,0 +1,25 @@ +package policy + +import ( + "github.com/goharbor/harbor/src/common/models" +) + +// Manager manages the notification policies +type Manager interface { + // Create new policy + Create(*models.NotificationPolicy) (int64, error) + // List the policies, returns the policy list and error + List(int64) ([]*models.NotificationPolicy, error) + // Get policy with specified ID + Get(int64) (*models.NotificationPolicy, error) + // GetByNameAndProjectID get policy by the name and projectID + GetByNameAndProjectID(string, int64) (*models.NotificationPolicy, error) + // Update the specified policy + Update(*models.NotificationPolicy) error + // Delete the specified policy + Delete(int64) error + // Test the specified policy + Test(*models.NotificationPolicy) error + // GetRelatedPolices get event type related policies in project + GetRelatedPolices(int64, string) ([]*models.NotificationPolicy, error) +} diff --git a/src/pkg/notification/policy/manager/manager.go b/src/pkg/notification/policy/manager/manager.go new file mode 100755 index 000000000..c4f6681c2 --- /dev/null +++ b/src/pkg/notification/policy/manager/manager.go @@ -0,0 +1,159 @@ +package manager + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/goharbor/harbor/src/common/dao/notification" + commonhttp "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/common/models" + "github.com/goharbor/harbor/src/common/utils/log" + notifierModel "github.com/goharbor/harbor/src/core/notifier/model" + "github.com/goharbor/harbor/src/pkg/notification/model" +) + +// DefaultManager ... +type DefaultManager struct { +} + +// NewDefaultManger ... +func NewDefaultManger() *DefaultManager { + return &DefaultManager{} +} + +// Create notification policy +func (m *DefaultManager) Create(policy *models.NotificationPolicy) (int64, error) { + t := time.Now() + policy.CreationTime = t + policy.UpdateTime = t + + err := policy.ConvertToDBModel() + if err != nil { + return 0, err + } + return notification.AddNotificationPolicy(policy) +} + +// List the notification policies, returns the policy list and error +func (m *DefaultManager) List(projectID int64) ([]*models.NotificationPolicy, error) { + policies := []*models.NotificationPolicy{} + persisPolicies, err := notification.GetNotificationPolicies(projectID) + if err != nil { + return nil, err + } + + for _, policy := range persisPolicies { + err := policy.ConvertFromDBModel() + if err != nil { + return nil, err + } + policies = append(policies, policy) + } + + return policies, nil +} + +// Get notification policy with specified ID +func (m *DefaultManager) Get(id int64) (*models.NotificationPolicy, error) { + policy, err := notification.GetNotificationPolicy(id) + if err != nil { + return nil, err + } + if policy == nil { + return nil, nil + } + err = policy.ConvertFromDBModel() + return policy, err +} + +// GetByNameAndProjectID notification policy by the name and projectID +func (m *DefaultManager) GetByNameAndProjectID(name string, projectID int64) (*models.NotificationPolicy, error) { + policy, err := notification.GetNotificationPolicyByName(name, projectID) + if err != nil { + return nil, err + } + err = policy.ConvertFromDBModel() + return policy, err +} + +// Update the specified notification policy +func (m *DefaultManager) Update(policy *models.NotificationPolicy) error { + policy.UpdateTime = time.Now() + err := policy.ConvertToDBModel() + if err != nil { + return err + } + return notification.UpdateNotificationPolicy(policy) +} + +// Delete the specified notification policy +func (m *DefaultManager) Delete(policyID int64) error { + return notification.DeleteNotificationPolicy(policyID) +} + +// Test the specified notification policy, just test for network connection without request body +func (m *DefaultManager) Test(policy *models.NotificationPolicy) error { + p, err := json.Marshal(notifierModel.Payload{ + Type: model.EventTypeTestEndpoint, + }) + if err != nil { + return err + } + + for _, target := range policy.Targets { + switch target.Type { + case "http": + return m.policyHTTPTest(target.Address, target.SkipCertVerify, p) + default: + return fmt.Errorf("invalid policy target type: %s", target.Type) + } + } + return nil +} + +func (m *DefaultManager) policyHTTPTest(address string, skipCertVerify bool, p []byte) error { + req, err := http.NewRequest(http.MethodPost, address, nil) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + + client := http.Client{ + Transport: commonhttp.GetHTTPTransport(skipCertVerify), + } + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + log.Debugf("policy test success with address %s, skip cert verify :%v", address, skipCertVerify) + + return nil +} + +// GetRelatedPolices get policies including event type in project +func (m *DefaultManager) GetRelatedPolices(projectID int64, eventType string) ([]*models.NotificationPolicy, error) { + policies, err := m.List(projectID) + if err != nil { + return nil, fmt.Errorf("failed to get notification policies with projectID %d: %v", projectID, err) + } + + var result []*models.NotificationPolicy + + for _, ply := range policies { + if !ply.Enabled { + continue + } + for _, t := range ply.EventTypes { + if t != eventType { + continue + } + result = append(result, ply) + } + } + return result, nil +} diff --git a/src/pkg/notification/policy/manager/manager_test.go b/src/pkg/notification/policy/manager/manager_test.go new file mode 100644 index 000000000..9dfd6970f --- /dev/null +++ b/src/pkg/notification/policy/manager/manager_test.go @@ -0,0 +1,22 @@ +package manager + +import ( + "reflect" + "testing" +) + +func TestNewDefaultManger(t *testing.T) { + tests := []struct { + name string + want *DefaultManager + }{ + {want: &DefaultManager{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := NewDefaultManger(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewDefaultManger() = %v, want %v", got, tt.want) + } + }) + } +} From 94d4f9c6b60483dad2637e98de559146e1019485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=B7=BB?= Date: Wed, 7 Aug 2019 20:56:31 +0800 Subject: [PATCH 2/3] add webhook job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 王添 --- make/harbor.yml | 4 + .../prepare/templates/jobservice/env.jinja | 1 + make/photon/prepare/utils/configs.py | 3 + .../job/impl/notification/webhook_job.go | 99 +++++++++++++++++++ .../job/impl/notification/webhook_job_test.go | 75 ++++++++++++++ src/jobservice/job/known_jobs.go | 2 + src/jobservice/runtime/bootstrap.go | 2 + 7 files changed, 186 insertions(+) create mode 100644 src/jobservice/job/impl/notification/webhook_job.go create mode 100644 src/jobservice/job/impl/notification/webhook_job_test.go diff --git a/make/harbor.yml b/make/harbor.yml index 515ac72c5..347ef0c8c 100644 --- a/make/harbor.yml +++ b/make/harbor.yml @@ -64,6 +64,10 @@ jobservice: # Maximum number of job workers in job service max_job_workers: 10 +notification: + # Maximum retry count for webhook job + webhook_job_max_retry: 10 + chart: # Change the value of absolute_url to enabled can enable absolute url in chart absolute_url: disabled diff --git a/make/photon/prepare/templates/jobservice/env.jinja b/make/photon/prepare/templates/jobservice/env.jinja index 2f4923248..d9e32c521 100644 --- a/make/photon/prepare/templates/jobservice/env.jinja +++ b/make/photon/prepare/templates/jobservice/env.jinja @@ -1,3 +1,4 @@ CORE_SECRET={{core_secret}} JOBSERVICE_SECRET={{jobservice_secret}} CORE_URL={{core_url}} +JOBSERVICE_WEBHOOK_JOB_MAX_RETRY={{notification_webhook_job_max_retry}} diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index c2e8b41fc..c57856845 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -188,6 +188,9 @@ def parse_yaml_config(config_file_path): config_dict['max_job_workers'] = js_config["max_job_workers"] config_dict['jobservice_secret'] = generate_random_string(16) + # notification config + notification_config = configs.get('notification') or {} + config_dict['notification_webhook_job_max_retry'] = notification_config["webhook_job_max_retry"] # Log configs allowed_levels = ['debug', 'info', 'warning', 'error', 'fatal'] diff --git a/src/jobservice/job/impl/notification/webhook_job.go b/src/jobservice/job/impl/notification/webhook_job.go new file mode 100644 index 000000000..b8c56966b --- /dev/null +++ b/src/jobservice/job/impl/notification/webhook_job.go @@ -0,0 +1,99 @@ +package notification + +import ( + "bytes" + "fmt" + commonhttp "github.com/goharbor/harbor/src/common/http" + "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/jobservice/logger" + "net/http" + "os" + "strconv" +) + +// Max retry has the same meaning as max fails. +const maxFails = "JOBSERVICE_WEBHOOK_JOB_MAX_RETRY" + +// WebhookJob implements the job interface, which send notification by http or https. +type WebhookJob struct { + client *http.Client + logger logger.Interface + ctx job.Context +} + +// MaxFails returns that how many times this job can fail, get this value from ctx. +func (wj *WebhookJob) MaxFails() uint { + if maxFails, exist := os.LookupEnv(maxFails); exist { + result, err := strconv.ParseUint(maxFails, 10, 32) + // Unable to log error message because the logger isn't initialized when calling this function. + if err == nil { + return uint(result) + } + } + + // Default max fails count is 10, and its max retry interval is around 3h + // Large enough to ensure most situations can notify successfully + return 10 +} + +// ShouldRetry ... +func (wj *WebhookJob) ShouldRetry() bool { + return true +} + +// Validate implements the interface in job/Interface +func (wj *WebhookJob) Validate(params job.Parameters) error { + return nil +} + +// Run implements the interface in job/Interface +func (wj *WebhookJob) Run(ctx job.Context, params job.Parameters) error { + if err := wj.init(ctx, params); err != nil { + return err + } + + return wj.execute(ctx, params) +} + +// init webhook job +func (wj *WebhookJob) init(ctx job.Context, params map[string]interface{}) error { + wj.logger = ctx.GetLogger() + wj.ctx = ctx + + // default insecureSkipVerify is false + insecureSkipVerify := false + if v, ok := params["skip_cert_verify"]; ok { + insecureSkipVerify = v.(bool) + } + wj.client = &http.Client{ + Transport: commonhttp.GetHTTPTransport(insecureSkipVerify), + } + + return nil +} + +// execute webhook job +func (wj *WebhookJob) execute(ctx job.Context, params map[string]interface{}) error { + payload := params["payload"].(string) + address := params["address"].(string) + + req, err := http.NewRequest(http.MethodPost, address, bytes.NewReader([]byte(payload))) + if err != nil { + return err + } + if v, ok := params["auth_header"]; ok && len(v.(string)) > 0 { + req.Header.Set("Authorization", v.(string)) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := wj.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("webhook job(target: %s) response code is %d", address, resp.StatusCode) + } + + return nil +} diff --git a/src/jobservice/job/impl/notification/webhook_job_test.go b/src/jobservice/job/impl/notification/webhook_job_test.go new file mode 100644 index 000000000..d5a1db69a --- /dev/null +++ b/src/jobservice/job/impl/notification/webhook_job_test.go @@ -0,0 +1,75 @@ +package notification + +import ( + "github.com/goharbor/harbor/src/jobservice/job/impl" + "github.com/stretchr/testify/assert" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "testing" +) + +func TestMaxFails(t *testing.T) { + rep := &WebhookJob{} + // test default max fails + assert.Equal(t, uint(10), rep.MaxFails()) + + // test user defined max fails + _ = os.Setenv(maxFails, "15") + assert.Equal(t, uint(15), rep.MaxFails()) + + // test user defined wrong max fails + _ = os.Setenv(maxFails, "abc") + assert.Equal(t, uint(10), rep.MaxFails()) +} + +func TestShouldRetry(t *testing.T) { + rep := &WebhookJob{} + assert.True(t, rep.ShouldRetry()) +} + +func TestValidate(t *testing.T) { + rep := &WebhookJob{} + assert.Nil(t, rep.Validate(nil)) +} + +func TestRun(t *testing.T) { + rep := &WebhookJob{} + + // test webhook request + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := ioutil.ReadAll(r.Body) + + // test request method + assert.Equal(t, http.MethodPost, r.Method) + // test request header + assert.Equal(t, "auth_test", r.Header.Get("Authorization")) + // test request body + assert.Equal(t, string(body), `{"key": "value"}`) + })) + defer ts.Close() + params := map[string]interface{}{ + "skip_cert_verify": true, + "payload": `{"key": "value"}`, + "address": ts.URL, + "auth_header": "auth_test", + } + // test correct webhook response + assert.Nil(t, rep.Run(&impl.Context{}, params)) + + tsWrong := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + })) + defer tsWrong.Close() + paramsWrong := map[string]interface{}{ + "skip_cert_verify": true, + "payload": `{"key": "value"}`, + "address": tsWrong.URL, + "auth_header": "auth_test", + } + // test incorrect webhook response + assert.NotNil(t, rep.Run(&impl.Context{}, paramsWrong)) +} diff --git a/src/jobservice/job/known_jobs.go b/src/jobservice/job/known_jobs.go index 60baa4ff9..307141e2d 100644 --- a/src/jobservice/job/known_jobs.go +++ b/src/jobservice/job/known_jobs.go @@ -30,6 +30,8 @@ const ( Replication = "REPLICATION" // ReplicationScheduler : the name of the replication scheduler job in job service ReplicationScheduler = "IMAGE_REPLICATE" + // WebhookJob : the name of the webhook job in job service + WebhookJob = "WEBHOOK" // Retention : the name of the retention job Retention = "RETENTION" ) diff --git a/src/jobservice/runtime/bootstrap.go b/src/jobservice/runtime/bootstrap.go index eb645c623..88dac6081 100644 --- a/src/jobservice/runtime/bootstrap.go +++ b/src/jobservice/runtime/bootstrap.go @@ -33,6 +33,7 @@ import ( "github.com/goharbor/harbor/src/jobservice/hook" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/job/impl/gc" + "github.com/goharbor/harbor/src/jobservice/job/impl/notification" "github.com/goharbor/harbor/src/jobservice/job/impl/replication" "github.com/goharbor/harbor/src/jobservice/job/impl/sample" "github.com/goharbor/harbor/src/jobservice/job/impl/scan" @@ -248,6 +249,7 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool( job.ReplicationScheduler: (*replication.Scheduler)(nil), job.Retention: (*retention.Job)(nil), scheduler.JobNameScheduler: (*scheduler.PeriodicJob)(nil), + job.WebhookJob: (*notification.WebhookJob)(nil), }); err != nil { // exit return nil, err From 7e40e335b75add0f0f3bb9176cbd203698254697 Mon Sep 17 00:00:00 2001 From: guanxiatao Date: Wed, 7 Aug 2019 21:33:42 +0800 Subject: [PATCH 3/3] add sql schema Signed-off-by: guanxiatao Signed-off-by: guanxiatao --- .../postgresql/0010_1.9.0_schema.up.sql | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/make/migrations/postgresql/0010_1.9.0_schema.up.sql b/make/migrations/postgresql/0010_1.9.0_schema.up.sql index 7ec76adeb..9303fa79b 100644 --- a/make/migrations/postgresql/0010_1.9.0_schema.up.sql +++ b/make/migrations/postgresql/0010_1.9.0_schema.up.sql @@ -139,3 +139,34 @@ create table schedule PRIMARY KEY (id) ); +/*add notification policy table*/ +create table notification_policy ( + id SERIAL NOT NULL, + name varchar(256), + project_id int NOT NULL, + enabled boolean NOT NULL DEFAULT true, + description text, + targets text, + event_types text, + creator varchar(256), + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP, + PRIMARY KEY (id), + CONSTRAINT unique_project_id UNIQUE (project_id) + ); + +/*add notification job table*/ + CREATE TABLE notification_job ( + id SERIAL NOT NULL, + policy_id int NOT NULL, + status varchar(32), + /* event_type is the type of trigger event, eg. pushImage, pullImage, uploadChart... */ + event_type varchar(256), + /* notify_type is the type to notify event to user, eg. HTTP, Email... */ + notify_type varchar(256), + job_detail text, + job_uuid varchar(64), + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP, + PRIMARY KEY (id) + );