Use new query model to get audit logs (#11113)

* Use new query model to get audit logs

leverage the query builder to build query, remove the old style query string

Signed-off-by: wang yan <wangyan@vmware.com>

* Switch to new API  for  project log page

Signed-off-by: AllForNothing <sshijun@vmware.com>

Co-authored-by: AllForNothing <sshijun@vmware.com>
This commit is contained in:
Wang Yan 2020-03-18 13:46:49 +08:00 committed by GitHub
parent b0e87b46e4
commit 050967f95f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 201 additions and 276 deletions

View File

@ -546,38 +546,9 @@ paths:
operationId: listAuditLogs
parameters:
- $ref: '#/parameters/requestId'
- $ref: '#/parameters/query'
- $ref: '#/parameters/page'
- $ref: '#/parameters/pageSize'
- name: username
in: query
type: string
required: false
description: Username of the operator.
- name: resource
in: query
type: string
required: false
description: The identity of resource
- name: resource_type
in: query
type: string
required: false
description: The type of resource, artifact/tag/repository
- name: operation
in: query
type: string
required: false
description: The operation, create/delete
- name: begin_timestamp
in: query
type: string
required: false
description: The begin timestamp
- name: end_timestamp
in: query
type: string
required: false
description: The end timestamp
responses:
'200':
description: Success

View File

@ -32,14 +32,17 @@ type PushArtifactEventMetadata struct {
// Resolve to the event from the metadata
func (p *PushArtifactEventMetadata) Resolve(event *event.Event) error {
ae := &event2.ArtifactEvent{
EventType: event2.TopicPushArtifact,
Repository: p.Artifact.RepositoryName,
Artifact: p.Artifact,
OccurAt: time.Now(),
}
if p.Tag != "" {
ae.Tags = []string{p.Tag}
}
data := &event2.PushArtifactEvent{
ArtifactEvent: &event2.ArtifactEvent{
EventType: event2.TopicPushArtifact,
Repository: p.Artifact.RepositoryName,
Artifact: p.Artifact,
Tags: []string{p.Tag},
OccurAt: time.Now(),
},
ArtifactEvent: ae,
}
ctx, exist := security.FromContext(p.Ctx)
if exist {
@ -59,14 +62,17 @@ type PullArtifactEventMetadata struct {
// Resolve to the event from the metadata
func (p *PullArtifactEventMetadata) Resolve(event *event.Event) error {
ae := &event2.ArtifactEvent{
EventType: event2.TopicPullArtifact,
Repository: p.Artifact.RepositoryName,
Artifact: p.Artifact,
OccurAt: time.Now(),
}
if p.Tag != "" {
ae.Tags = []string{p.Tag}
}
data := &event2.PullArtifactEvent{
ArtifactEvent: &event2.ArtifactEvent{
EventType: event2.TopicPullArtifact,
Repository: p.Artifact.RepositoryName,
Artifact: p.Artifact,
Tags: []string{p.Tag},
OccurAt: time.Now(),
},
ArtifactEvent: ae,
}
ctx, exist := security.FromContext(p.Ctx)
if exist {

View File

@ -30,16 +30,16 @@
</div>
</div>
<div class="col-lg-12 col-md-12 col-sm-12 col-xs-12 datagrid-margin-top ">
<clr-datagrid (clrDgRefresh)="retrievePage()">
<clr-datagrid [clrDgLoading]="loading" (clrDgRefresh)="retrieve()">
<clr-dg-column>{{'AUDIT_LOG.USERNAME' | translate}}</clr-dg-column>
<clr-dg-column>{{'AUDIT_LOG.REPOSITORY_NAME' | translate}}</clr-dg-column>
<clr-dg-column>{{'AUDIT_LOG.TAGS' | translate}}</clr-dg-column>
<clr-dg-column>{{'AUDIT_LOG.RESOURCE' | translate}}</clr-dg-column>
<clr-dg-column>{{'AUDIT_LOG.RESOURCE_TYPE' | translate}}</clr-dg-column>
<clr-dg-column>{{'AUDIT_LOG.OPERATION' | translate}}</clr-dg-column>
<clr-dg-column>{{'AUDIT_LOG.TIMESTAMP' | translate}}</clr-dg-column>
<clr-dg-row *ngFor="let l of auditLogs">
<clr-dg-cell>{{l.username}}</clr-dg-cell>
<clr-dg-cell>{{l.repo_name}}</clr-dg-cell>
<clr-dg-cell>{{l.repo_tag}}</clr-dg-cell>
<clr-dg-cell>{{l.resource}}</clr-dg-cell>
<clr-dg-cell>{{l.resource_type}}</clr-dg-cell>
<clr-dg-cell>{{l.operation}}</clr-dg-cell>
<clr-dg-cell>{{l.op_time | date: 'short'}}</clr-dg-cell>
</clr-dg-row>

View File

@ -2,16 +2,19 @@ import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { TranslateModule, TranslateService } from '@ngx-translate/core';
import { AuditLogComponent } from './audit-log.component';
import { ClarityModule } from '@clr/angular';
import { FormsModule } from '@angular/forms';
import { AuditLogService } from './audit-log.service';
import { MessageHandlerService } from '../shared/message-handler/message-handler.service';
import { ActivatedRoute, Router } from '@angular/router';
import { of } from 'rxjs';
import { CUSTOM_ELEMENTS_SCHEMA } from '@angular/core';
import { CUSTOM_ELEMENTS_SCHEMA, DebugElement } from '@angular/core';
import { BrowserAnimationsModule, NoopAnimationsModule } from '@angular/platform-browser/animations';
import { RouterTestingModule } from '@angular/router/testing';
import { delay } from 'rxjs/operators';
import { HarborLibraryModule } from "../../lib/harbor-library.module";
import { AuditLog } from "../../../ng-swagger-gen/models/audit-log";
import { HttpHeaders, HttpResponse } from "@angular/common/http";
import { ProjectService } from "../../../ng-swagger-gen/services/project.service";
import { click } from "../../lib/utils/utils";
import { FormsModule } from "@angular/forms";
describe('AuditLogComponent', () => {
let component: AuditLogComponent;
@ -19,27 +22,59 @@ describe('AuditLogComponent', () => {
const mockMessageHandlerService = {
handleError: () => {}
};
const mockAuditLogService = {
listAuditLogs: () => {
return of({
headers: new Map().set('x-total-count', 0),
body: []
}).pipe(delay(0));
},
};
const mockActivatedRoute = {
parent: {
snapshot: {
data: null
}
},
snapshot: {
data: null
},
data: of({
auditLogResolver: ""
}).pipe(delay(0)),
snapshot: {
parent: {
params: {
id: 1
}).pipe(delay(0))
};
const mockRouter = null;
const mockedAuditLogs: AuditLog [] = [];
for (let i = 0; i < 18; i++) {
let item: AuditLog = {
id: 234 + i,
resource: "myProject/Demo" + i,
resource_type: "N/A",
operation: "create",
op_time: "2017-04-11T10:26:22Z",
username: "user91" + i
};
mockedAuditLogs.push(item);
}
const fakedAuditlogService = {
getLogsResponse(params: ProjectService.GetLogsParams) {
if (params.q && params.q.indexOf('Demo0') !== -1) {
return of(new HttpResponse({
body: mockedAuditLogs.slice(0, 1),
headers: new HttpHeaders({
"x-total-count": "18"
})
})).pipe(delay(0));
}
if (params.page <= 1) {
return of(new HttpResponse({
body: mockedAuditLogs.slice(0, 15),
headers: new HttpHeaders({
"x-total-count": "18"
})
})).pipe(delay(0));
} else {
return of(new HttpResponse({
body: mockedAuditLogs.slice(15),
headers: new HttpHeaders({
"x-total-count": "18"
})
})).pipe(delay(0));
}
}
};
const mockRouter = null;
beforeEach(async(() => {
TestBed.configureTestingModule({
@ -60,7 +95,7 @@ describe('AuditLogComponent', () => {
TranslateService,
{ provide: ActivatedRoute, useValue: mockActivatedRoute },
{ provide: Router, useValue: mockRouter },
{ provide: AuditLogService, useValue: mockAuditLogService },
{ provide: ProjectService, useValue: fakedAuditlogService },
{ provide: MessageHandlerService, useValue: mockMessageHandlerService },
]
@ -76,4 +111,57 @@ describe('AuditLogComponent', () => {
it('should create', () => {
expect(component).toBeTruthy();
});
it('should get data from AccessLogService', async(() => {
fixture.detectChanges();
fixture.whenStable().then(() => { // wait for async getRecentLogs
fixture.detectChanges();
expect(component.auditLogs).toBeTruthy();
expect(component.auditLogs.length).toEqual(15);
});
}));
it('should render data to view', async(() => {
fixture.detectChanges();
fixture.whenStable().then(() => {
fixture.detectChanges();
let de: DebugElement = fixture.debugElement.query(del => del.classes['datagrid-cell']);
expect(de).toBeTruthy();
let el: HTMLElement = de.nativeElement;
expect(el).toBeTruthy();
expect(el.textContent.trim()).toEqual('user910');
});
}));
it('should support pagination', async () => {
fixture.autoDetectChanges(true);
await fixture.whenStable();
let el: HTMLButtonElement = fixture.nativeElement.querySelector('.pagination-next');
expect(el).toBeTruthy();
el.click();
fixture.detectChanges();
await fixture.whenStable();
expect(component.currentPage).toEqual(2);
expect(component.auditLogs.length).toEqual(3);
});
it('should support filtering list by keywords', async(() => {
fixture.detectChanges();
let el: HTMLElement = fixture.nativeElement.querySelector('.search-btn');
expect(el).toBeTruthy("Not found search icon");
click(el);
fixture.detectChanges();
let el2: HTMLInputElement = fixture.nativeElement.querySelector('input');
expect(el2).toBeTruthy("Not found input");
fixture.detectChanges();
fixture.whenStable().then(() => {
fixture.detectChanges();
component.doSearchAuditLogs("Demo0");
fixture.detectChanges();
fixture.whenStable().then(() => {
fixture.detectChanges();
expect(component.auditLogs).toBeTruthy();
expect(component.auditLogs.length).toEqual(1);
});
});
}));
});

View File

@ -11,17 +11,14 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { Component, OnInit, ViewChild } from '@angular/core';
import { NgModel } from '@angular/forms';
import { Component, OnInit } from '@angular/core';
import { ActivatedRoute, Router } from '@angular/router';
import { AuditLog } from './audit-log';
import { SessionUser } from '../shared/session-user';
import { AuditLogService } from './audit-log.service';
import { MessageHandlerService } from '../shared/message-handler/message-handler.service';
import { State } from '../../lib/services/interface';
import { ProjectService } from "../../../ng-swagger-gen/services/project.service";
import { AuditLog } from "../../../ng-swagger-gen/models/audit-log";
import { Project } from "../project/project";
import { finalize } from "rxjs/operators";
const optionalSearch: {} = { 0: 'AUDIT_LOG.ADVANCED', 1: 'AUDIT_LOG.SIMPLE' };
@ -55,8 +52,13 @@ export class AuditLogComponent implements OnInit {
search: SearchOption = new SearchOption();
currentUser: SessionUser;
projectId: number;
queryParam: AuditLog = new AuditLog();
projectName: string;
queryUsername: string;
queryStartTime: string;
queryEndTime: string;
queryOperation: string[] = [];
auditLogs: AuditLog[];
loading: boolean = true;
toggleName = optionalSearch;
currentOption: number = 0;
@ -82,25 +84,60 @@ export class AuditLogComponent implements OnInit {
constructor(
private route: ActivatedRoute,
private router: Router,
private auditLogService: AuditLogService,
private auditLogService: ProjectService,
private messageHandlerService: MessageHandlerService) {
// Get current user from registered resolver.
this.route.data.subscribe(data => this.currentUser = <SessionUser>data['auditLogResolver']);
}
ngOnInit(): void {
this.projectId = +this.route.snapshot.parent.params['id'];
this.queryParam.project_id = this.projectId;
this.queryParam.page_size = this.pageSize;
const resolverData = this.route.parent.snapshot.data;
if (resolverData) {
const pro: Project = <Project>resolverData['projectResolver'];
this.projectName = pro.name;
}
}
private retrieve(): void {
retrieve() {
const arr: string[] = [];
if (this.queryUsername) {
arr.push(`username=~${this.queryUsername}`);
}
if (this.queryStartTime && this.queryEndTime) {
arr.push(`op_time=[${this.queryStartTime}~${this.queryEndTime}]`);
} else {
if (this.queryStartTime) {
arr.push(`op_time=[${this.queryStartTime}~]`);
}
if (this.queryEndTime) {
arr.push(`op_time=[~${this.queryEndTime}]`);
}
}
if (this.queryOperation && this.queryOperation.length > 0) {
arr.push(`operation={${this.queryOperation.join(' ')}}`);
}
const param: ProjectService.GetLogsParams = {
projectName: this.projectName,
pageSize: this.pageSize,
page: this.currentPage,
};
if (arr && arr.length > 0) {
param.q = encodeURIComponent(arr.join(','));
}
this.loading = true;
this.auditLogService
.listAuditLogs(this.queryParam)
.getLogsResponse(param)
.pipe(finalize(() => this.loading = false))
.subscribe(
response => {
this.totalRecordCount = Number.parseInt(response.headers.get('x-total-count'));
// Get total count
if (response.headers) {
let xHeader: string = response.headers.get("x-total-count");
if (xHeader) {
this.totalRecordCount = Number.parseInt(xHeader);
}
}
this.auditLogs = response.body;
},
error => {
@ -108,24 +145,18 @@ export class AuditLogComponent implements OnInit {
}
);
}
retrievePage() {
this.queryParam.page = this.currentPage;
this.retrieve();
}
doSearchAuditLogs(searchUsername: string): void {
this.queryParam.username = searchUsername;
this.queryUsername = searchUsername;
this.retrieve();
}
doSearchByStartTime(fromTimestamp: string): void {
this.queryParam.begin_timestamp = fromTimestamp;
this.queryStartTime = fromTimestamp;
this.retrieve();
}
doSearchByEndTime(toTimestamp: string): void {
this.queryParam.end_timestamp = toTimestamp;
this.queryEndTime = toTimestamp;
this.retrieve();
}
@ -134,7 +165,7 @@ export class AuditLogComponent implements OnInit {
let operationFilter: string[] = [];
for (let filterOption of this.filterOptions) {
if (filterOption.checked) {
operationFilter.push('operation=' + filterOption.key);
operationFilter.push(filterOption.key);
} else {
selectAll = false;
}
@ -142,7 +173,7 @@ export class AuditLogComponent implements OnInit {
if (selectAll) {
operationFilter = [];
}
this.queryParam.keywords = operationFilter.join('&');
this.queryOperation = operationFilter;
this.retrieve();
}

View File

@ -1,18 +0,0 @@
import { TestBed, inject } from '@angular/core/testing';
import { HttpClientTestingModule } from '@angular/common/http/testing';
import { AuditLogService } from './audit-log.service';
describe('AuditLogService', () => {
beforeEach(() => {
TestBed.configureTestingModule({
imports: [
HttpClientTestingModule
],
providers: [AuditLogService]
});
});
it('should be created', inject([AuditLogService], (service: AuditLogService) => {
expect(service).toBeTruthy();
}));
});

View File

@ -1,65 +0,0 @@
import {throwError as observableThrowError, Observable } from "rxjs";
import {map, catchError} from 'rxjs/operators';
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { Injectable } from '@angular/core';
import { HttpClient, HttpParams, HttpResponse } from '@angular/common/http';
import { AuditLog } from './audit-log';
import {
buildHttpRequestOptions,
buildHttpRequestOptionsWithObserveResponse,
CURRENT_BASE_HREF
} from "../../lib/utils/utils";
import { RequestQueryParams } from "../../lib/services";
export const logEndpoint = CURRENT_BASE_HREF + '/logs';
@Injectable()
export class AuditLogService {
constructor(private http: HttpClient) {}
listAuditLogs(queryParam: AuditLog): Observable<any> {
let params: HttpParams = new HttpParams({fromString: queryParam.keywords});
if (queryParam.begin_timestamp) {
params = params.set('begin_timestamp', <string>queryParam.begin_timestamp);
}
if (queryParam.end_timestamp) {
params = params.set('end_timestamp', <string>queryParam.end_timestamp);
}
if (queryParam.username) {
params = params.set('username', queryParam.username);
}
if (queryParam.page) {
params = params.set('page', <string>queryParam.page);
}
if (queryParam.page_size) {
params = params.set('page_size', <string>queryParam.page_size);
}
return this.http
.get<HttpResponse<AuditLog[]>>(`${ CURRENT_BASE_HREF }/projects/${queryParam.project_id}/logs`
, buildHttpRequestOptionsWithObserveResponse(params)).pipe(
catchError(error => observableThrowError(error)), );
}
getRecentLogs(lines: number): Observable<AuditLog[]> {
let params: RequestQueryParams = new RequestQueryParams();
params = params.set('page_size', '' + lines);
return this.http.get(logEndpoint, buildHttpRequestOptions(params)).pipe(
map(response => response as AuditLog[]),
catchError(error => observableThrowError(error)), );
}
}

View File

@ -1,47 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
{
"log_id": 3,
"user_id": 0,
"project_id": 0,
"repo_name": "library/mysql",
"repo_tag": "5.6",
"guid": "",
"operation": "push",
"op_time": "2017-02-14T09:22:58Z",
"username": "admin",
"keywords": "",
"BeginTime": "0001-01-01T00:00:00Z",
"begin_timestamp": 0,
"EndTime": "0001-01-01T00:00:00Z",
"end_timestamp": 0
}
*/
export class AuditLog {
log_id: number | string;
project_id: number | string;
username: string;
repo_name: string;
repo_tag: string;
operation: string;
op_time: Date;
begin_timestamp: number | string;
end_timestamp: number | string;
keywords: string;
page: number | string;
page_size: number | string;
fromTime: string;
toTime: string;
}

View File

@ -14,7 +14,6 @@
import { NgModule } from '@angular/core';
import { AuditLogComponent } from './audit-log.component';
import { SharedModule } from '../shared/shared.module';
import { AuditLogService } from './audit-log.service';
import { LogPageComponent } from './log-page.component';
@NgModule({
@ -23,7 +22,6 @@ import { LogPageComponent } from './log-page.component';
AuditLogComponent,
LogPageComponent
],
providers: [AuditLogService],
exports: [
AuditLogComponent,
LogPageComponent]

View File

@ -33,29 +33,10 @@ export class DatePickerComponent implements OnChanges {
false
);
}
convertDate(strDate: string): string {
if (
/^(0[1-9]|[12][0-9]|3[01])[- /.](0[1-9]|1[012])[- /.](19|20)\d\d$/.test(
strDate
)
) {
let parts = strDate.split(/[-\/]/);
strDate =
parts[2] /*Year*/ + "-" + parts[0] /*Month*/+ "-" + parts[1] /*Date*/;
}
return strDate;
}
doSearch() {
let searchTerm: string = "";
if (this.searchTime.valid && this.dateInput) {
let timestamp: number =
new Date(this.convertDate(this.searchTime.value)).getTime() / 1000;
if (this.oneDayOffset) {
timestamp += 3600 * 24;
}
searchTerm = timestamp.toString();
searchTerm = this.searchTime.value;
}
this.search.emit(searchTerm);
}

View File

@ -7,7 +7,7 @@
<select id="selectKey" (change)="selectFilterKey($event)">
<option value="username">{{"AUDIT_LOG.USERNAME" | translate | lowercase}}</option>
<option value="resource">{{"AUDIT_LOG.RESOURCE" | translate | lowercase}}</option>
<option value="resourceType">{{"AUDIT_LOG.RESOURCE_TYPE" | translate | lowercase}}</option>
<option value="resource_type">{{"AUDIT_LOG.RESOURCE_TYPE" | translate | lowercase}}</option>
<option value="operation">{{"AUDIT_LOG.OPERATION" | translate | lowercase}}</option>
</select>
</div>

View File

@ -10,7 +10,6 @@ import { of } from 'rxjs';
import { AuditLog } from "../../../../ng-swagger-gen/models/audit-log";
import { AuditlogService } from "../../../../ng-swagger-gen/services/auditlog.service";
import { HttpHeaders, HttpResponse } from "@angular/common/http";
import ListAuditLogsParams = AuditlogService.ListAuditLogsParams;
import { delay } from "rxjs/operators";
describe('RecentLogComponent (inline template)', () => {
@ -39,9 +38,9 @@ describe('RecentLogComponent (inline template)', () => {
mockedAuditLogs.push(item);
}
const fakedAuditlogService = {
listAuditLogsResponse(params: ListAuditLogsParams) {
if (params && params.username) {
if (params.username === 'demo0') {
listAuditLogsResponse(params: AuditlogService.ListAuditLogsParams) {
if (params && params.q) {
if (params.q.indexOf('demo0') !== -1) {
return of(new HttpResponse({
body: mockedAuditLogs.slice(0, 1),
headers: new HttpHeaders({

View File

@ -77,7 +77,7 @@ export class RecentLogComponent implements OnInit {
pageSize: this.pageSize
};
if (this.currentTerm && this.currentTerm !== "") {
params[this.defaultFilter] = this.currentTerm;
params.q = encodeURIComponent(`${this.defaultFilter}=~${this.currentTerm}`);
}
this.loading = true;
this.logService.listAuditLogsResponse(params).pipe(finalize(() => (this.loading = false)))

View File

@ -4,7 +4,6 @@ import (
"context"
"github.com/go-openapi/runtime/middleware"
"github.com/goharbor/harbor/src/pkg/audit"
"github.com/goharbor/harbor/src/pkg/q"
"github.com/goharbor/harbor/src/server/v2.0/models"
"github.com/goharbor/harbor/src/server/v2.0/restapi/operations/auditlog"
operation "github.com/goharbor/harbor/src/server/v2.0/restapi/operations/auditlog"
@ -26,28 +25,9 @@ func (a *auditlogAPI) ListAuditLogs(ctx context.Context, params auditlog.ListAud
// if !a.HasPermission(ctx, rbac.ActionList, rbac.ResourceLog) {
// return a.SendError(ctx, ierror.ForbiddenError(nil))
// }
keywords := make(map[string]interface{})
query := &q.Query{
Keywords: keywords,
}
// TODO support fuzzy match and start end time
if params.Username != nil {
query.Keywords["Username"] = *(params.Username)
}
if params.Operation != nil {
query.Keywords["Operation"] = *(params.Operation)
}
if params.Resource != nil {
query.Keywords["Resource"] = *(params.Resource)
}
if params.ResourceType != nil {
query.Keywords["ResourceType"] = *(params.ResourceType)
}
if params.Page != nil {
query.PageNumber = *(params.Page)
}
if params.PageSize != nil {
query.PageSize = *(params.PageSize)
query, err := a.BuildQuery(ctx, params.Q, params.Page, params.PageSize)
if err != nil {
return a.SendError(ctx, err)
}
total, err := a.auditMgr.Count(ctx, query)
if err != nil {

View File

@ -148,6 +148,7 @@ func (r *repositoryAPI) DeleteRepository(ctx context.Context, params operation.D
notification.AddEvent(ctx, &metadata.DeleteRepositoryEventMetadata{
Ctx: ctx,
Repository: repository.Name,
ProjectID: repository.ProjectID,
})
return operation.NewDeleteRepositoryOK()