Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-24 17:47:46 +01:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into 170724_registry

Browse Source 
Conflicts:
	src/ui/utils/utils.go
This commit is contained in:
Wenkai Yin 2017-07-26 18:46:41 +08:00
commit 71e4c3c447
61 changed files with 837 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -81,7 +81,7 @@ REGISTRYSERVER=
REGISTRYPROJECTNAME=vmware
DEVFLAG=true
NOTARYFLAG=false
REGISTRYVERSION=2.6.1-photon
REGISTRYVERSION=2.6.2-photon
NGINXVERSION=1.11.13
PHOTONVERSION=1.0
NOTARYVERSION=server-0.5.0

55
docs/clair_offlinedata_guide.md Normal file
View File

@ -0,0 +1,55 @@
## Guild for update Clair offline data
### Summary
In some case when user install harbor in an environment without internet access. Then Clair will not be able to fetch the latest vulnerability database. In this circumstance user need manfully update the Clair database.
This document is a step by step instruction on update Clair vulnerability database in Harbor v1.2.
### Preparation
A. User need to install Clair 2.0.1 ( if you have a harbor1.2 instance with internet access will also works.)
B. Check the Clair already update the vulnerability to the latest.
a. 'docker ps' to list the Clair container. Get the Clair container id .
b. Check the log of the Clair container.
c. If you are using harbor you can find the latest Clair log under /var/log/harbor/2017--xx-xx/clair.log
d. You will find some logs like follow:
```
Jul 3 20:40:45 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:45.890364","updater name":"rhel"}
Jul 3 20:40:46 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:46.768924","updater name":"alpine"}
Jul 3 20:40:47 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:40:47.190982","updater name":"oracle"}
Jul 3 20:41:07 172.18.0.1 clair[3516]: {"Event":"Debian buster is not mapped to any version number (eg. Jessie-\u003e8). Please update me.","Level":"warning","Location":"debian.go:128","Time":"2017-07-04 03:41:07.833720"}
Jul 3 20:41:07 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 03:41:07.833975","updater name":"debian"}
Jul 4 00:26:17 172.18.0.1 clair[3516]: {"Event":"finished fetching","Level":"info","Location":"updater.go:227","Time":"2017-07-04 07:26:17.596986","updater name":"ubuntu"}
Jul 4 00:26:18 172.18.0.1 clair[3516]: {"Event":"adding metadata to vulnerabilities","Level":"info","Location":"updater.go:253","Time":"2017-07-04 07:26:18.060810"}
Jul 4 00:38:05 172.18.0.1 clair[3516]: {"Event":"update finished","Level":"info","Location":"updater.go:198","Time":"2017-07-04 07:38:05.251580"}
```
e. The update finished indicate that Clair has finished an vulnerability update round. You need to check that logs above it to make sure all the endpoints are update correctly.
### Data dump
A. Login into the host where Clair db is running.
B. Dump the Clair vulnerability database by the follow command
* $>docker exec clair-db /bin/bash -c "pg_dump -U postgres -a -t feature -t keyvalue -t namespace -t schema_migrations -t vulnerability -t vulnerability_fixedin_feature" > vulnerability.sql
* $>docker exec clair-db /bin/bash -c "pg_dump -U postgres -c -s" > clear.sql
### Back Up Clair DB
A. Before update the offline data, user are strongly suggested to backup their Clair db.
* docker exec clair-db /bin/bash -c "pg_dump -U postgres -c" > all.sql
### Update Clair DB
A. Copy the vulnerability.sql and clear.sql to the host where clair-db container which you want to update is running on.
C. $>docker exec -i clair-db psql -U postgres < clear.sql
D. $>docker exec -i clair-db psql –U postgres < vulnerability.sql
### Rescan
After update the offline data, user need to trigger the "rescan all" functionality to scan all the images and Harbor reflect the new changes automatically after the scan finished.(Otherwise the vulnerability detail will not show up)

2
make/docker-compose.tpl
View File

@ -11,7 +11,7 @@ services:
networks:
- harbor
registry:
image: vmware/registry:2.6.1-photon
image: vmware/registry:2.6.2-photon
container_name: registry
restart: always
volumes:

15
make/photon/registry/Dockerfile Normal file
View File

@ -0,0 +1,15 @@
FROM library/photon:1.0
MAINTAINER wangyan@vmware.com
# The original script in the docker offical registry image.
COPY entrypoint.sh /
RUN chmod u+x /entrypoint.sh
COPY registry /usr/bin
RUN chmod u+x /usr/bin/registry
VOLUME ["/var/lib/registry"]
EXPOSE 5000
ENTRYPOINT ["/entrypoint.sh"]
CMD ["/etc/docker/registry/config.yml"]

11
src/common/dao/repository.go
View File

@ -48,9 +48,9 @@ func GetRepositoryByName(name string) (*models.RepoRecord, error) {
}
// GetAllRepositories ...
func GetAllRepositories() ([]models.RepoRecord, error) {
func GetAllRepositories() ([]*models.RepoRecord, error) {
o := GetOrmer()
var repos []models.RepoRecord
var repos []*models.RepoRecord
_, err := o.QueryTable("repository").
OrderBy("Name").All(&repos)
return repos, err
@ -160,9 +160,10 @@ func GetRepositoriesByProject(projectID int64, name string,
if len(name) != 0 {
qs = qs.Filter("Name__contains", name)
}
_, err := qs.Limit(limit).
Offset(offset).All(&repositories)
if limit > 0 {
qs = qs.Limit(limit).Offset(offset)
}
_, err := qs.All(&repositories)
return repositories, err
}

7
src/common/notifier/config_watcher.go
View File

@ -2,7 +2,6 @@ package notifier
import (
"errors"
"reflect"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils"
@ -27,8 +26,10 @@ func WatchConfigChanges(cfg map[string]interface{}) error {
}
if t, yes := policyCfg.Parm["daily_time"]; yes {
if reflect.TypeOf(t).Kind() == reflect.Int {
policyNotification.DailyTime = (int64)(t.(int))
if dt, success := t.(float64); success {
policyNotification.DailyTime = (int64)(dt)
} else {
return errors.New("Invalid daily_time type")
}
}

2
src/common/notifier/notifier.go
View File

@ -200,6 +200,8 @@ func (nw *NotificationWatcher) Notify(notification Notification) error {
if err := hd.Handle(notification.Value); err != nil {
//Currently, we just log the error
log.Errorf("Error occurred when triggerring handler %s of topic %s: %s\n", reflect.TypeOf(hd).String(), notification.Topic, err.Error())
} else {
log.Infof("Handle notification with topic '%s': %#v\n", notification.Topic, notification.Value)
}
}()
}(h, handlerChan)

2
src/common/scheduler/policy/alternate_policy.go
View File

@ -5,6 +5,7 @@ import (
"time"
"github.com/vmware/harbor/src/common/scheduler/task"
"github.com/vmware/harbor/src/common/utils/log"
)
//AlternatePolicyConfiguration store the related configurations for alternate policy.
@ -125,6 +126,7 @@ func (alp *AlternatePolicy) Evaluate() (<-chan bool, error) {
}
if diff > 0 {
//Wait for a while.
log.Infof("Waiting for %d seconds after comparing offset %d and utc time %d\n", diff, alp.config.OffsetTime, utcTime)
select {
case <-time.After(time.Duration(diff) * time.Second):
case <-alp.terminator:

1
src/common/scheduler/watcher.go
View File

@ -70,6 +70,7 @@ func (wc *Watcher) Start() {
select {
case <-evalChan:
{
log.Infof("Receive evaluation signal from policy '%s'\n", pl.Name())
//Start to run the attached tasks.
for _, t := range pl.Tasks() {
go func(tk task.Task) {

15
src/jobservice/job/workerpool.go
View File

@ -16,6 +16,7 @@ package job
import (
"fmt"
"sync"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
@ -33,6 +34,9 @@ type workerPool struct {
// WorkerPools is a map contains workerpools for different types of jobs.
var WorkerPools map[Type]*workerPool
// For WorkerPools initialization.
var once sync.Once
//TODO: remove the hard code?
const maxScanWorker = 3
@ -118,16 +122,15 @@ func NewWorker(id int, t Type, wp *workerPool) *Worker {
// InitWorkerPools create worker pools for different types of jobs.
func InitWorkerPools() error {
if len(WorkerPools) > 0 {
return fmt.Errorf("The WorkerPool map has been initialised")
}
maxRepWorker, err := config.MaxJobWorkers()
if err != nil {
return err
}
WorkerPools = make(map[Type]*workerPool)
WorkerPools[ReplicationType] = createWorkerPool(maxRepWorker, ReplicationType)
WorkerPools[ScanType] = createWorkerPool(maxScanWorker, ScanType)
once.Do(func() {
WorkerPools = make(map[Type]*workerPool)
WorkerPools[ReplicationType] = createWorkerPool(maxRepWorker, ReplicationType)
WorkerPools[ScanType] = createWorkerPool(maxScanWorker, ScanType)
})
return nil
}

8
src/jobservice/main.go
View File

@ -42,7 +42,9 @@ func main() {
}
initRouters()
job.InitWorkerPools()
if err := job.InitWorkerPools(); err != nil {
log.Fatalf("Failed to initialize worker pools, error: %v", err)
}
go job.Dispatch()
resumeJobs()
beego.Run()
@ -71,6 +73,8 @@ func init() {
configPath := os.Getenv("CONFIG_PATH")
if len(configPath) != 0 {
log.Infof("Config path: %s", configPath)
beego.LoadAppConfig("ini", configPath)
if err := beego.LoadAppConfig("ini", configPath); err != nil {
log.Fatalf("Failed to load config file: %s, error: %v", configPath, err)
}
}
}

48
src/ui/api/repository.go
View File

@ -19,6 +19,7 @@ import (
"fmt"
"io/ioutil"
"net/http"
"strconv"
"time"
"github.com/docker/distribution/manifest/schema1"
@ -731,23 +732,40 @@ func (ra *RepositoryAPI) ScanAll() {
ra.HandleUnauthorized()
return
}
if !ra.SecurityCtx.IsSysAdmin() {
ra.HandleForbidden(ra.SecurityCtx.GetUsername())
return
}
projectIDStr := ra.GetString("project_id")
if len(projectIDStr) > 0 { //scan images under the project only.
pid, err := strconv.ParseInt(projectIDStr, 10, 64)
if err != nil || pid <= 0 {
ra.HandleBadRequest(fmt.Sprintf("Invalid project_id %s", projectIDStr))
return
}
if !ra.SecurityCtx.HasAllPerm(pid) {
ra.HandleForbidden(ra.SecurityCtx.GetUsername())
return
}
if err := uiutils.ScanImagesByProjectID(pid); err != nil {
log.Errorf("Failed triggering scan images in project: %d, error: %v", pid, err)
ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))
return
}
} else { //scan all images in Harbor
if !ra.SecurityCtx.IsSysAdmin() {
ra.HandleForbidden(ra.SecurityCtx.GetUsername())
return
}
if !utils.ScanAllMarker().Check() {
log.Warningf("There is a scan all scheduled at: %v, the request will not be processed.", utils.ScanAllMarker().Next())
ra.RenderError(http.StatusPreconditionFailed, "Unable handle frequent scan all requests")
return
}
if !utils.ScanAllMarker().Check() {
log.Warningf("There is a scan all scheduled at: %v, the request will not be processed.", utils.ScanAllMarker().Next())
ra.RenderError(http.StatusPreconditionFailed, "Unable handle frequent scan all requests")
return
if err := uiutils.ScanAllImages(); err != nil {
log.Errorf("Failed triggering scan all images, error: %v", err)
ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))
return
}
utils.ScanAllMarker().Mark()
}
if err := uiutils.ScanAllImages(); err != nil {
log.Errorf("Failed triggering scan all images, error: %v", err)
ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))
return
}
utils.ScanAllMarker().Mark()
ra.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted)
}

2
src/ui/api/utils.go
View File

@ -176,7 +176,7 @@ func SyncRegistry(pm projectmanager.ProjectManager) error {
return err
}
var repoRecordsInDB []models.RepoRecord
var repoRecordsInDB []*models.RepoRecord
repoRecordsInDB, err = dao.GetAllRepositories()
if err != nil {
log.Errorf("error occurred while getting all registories. %v", err)

64
src/ui/utils/utils.go
View File

@ -31,40 +31,54 @@ import (
"net/http"
)
// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move one if failed to subit any job of a single image.
// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move on if failed to submit any job of a single image.
func ScanAllImages() error {
repos, err := dao.GetAllRepositories()
if err != nil {
log.Errorf("Failed to list all repositories, error: %v", err)
return err
}
log.Infof("Rescanning all images.")
log.Infof("Scanning all images on Harbor.")
go func() {
var repoClient *registry.Repository
var err error
var tags []string
for _, r := range repos {
repoClient, err = NewRepositoryClientForUI("harbor-ui", r.Name)
if err != nil {
log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err)
continue
}
tags, err = repoClient.ListTag()
if err != nil {
log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err)
continue
}
for _, t := range tags {
if err = TriggerImageScan(r.Name, t); err != nil {
log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err)
} else {
log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t)
}
go scanRepos(repos)
return nil
}
// ScanImagesByProjectID scans all images under a projet, the whole process will move on if failed to submit any job of a single image.
func ScanImagesByProjectID(id int64) error {
repos, err := dao.GetRepositoriesByProject(id, "", 0, 0)
if err != nil {
log.Errorf("Failed list repositories in project %d, error: %v", id, err)
return err
}
log.Infof("Scanning all images in project: %d ", id)
go scanRepos(repos)
return nil
}
func scanRepos(repos []*models.RepoRecord) {
var repoClient *registry.Repository
var err error
var tags []string
for _, r := range repos {
repoClient, err = NewRepositoryClientForUI("harbor-ui", r.Name)
if err != nil {
log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err)
continue
}
tags, err = repoClient.ListTag()
if err != nil {
log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err)
continue
}
for _, t := range tags {
if err = TriggerImageScan(r.Name, t); err != nil {
log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err)
} else {
log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t)
}
}
}()
return nil
}
}
// RequestAsUI is a shortcut to make a request attach UI secret and send the request.

2
src/ui_ng/lib/src/config/registry-config.component.html.ts
View File

@ -2,7 +2,7 @@ export const REGISTRY_CONFIG_HTML: string = `
<div>
<replication-config #replicationConfig [(replicationConfig)]="config" [showSubTitle]="true"></replication-config>
<system-settings #systemSettings [(systemSettings)]="config" [showSubTitle]="true" [hasAdminRole]="hasAdminRole" [hasCAFile]="hasCAFile"></system-settings>
<vulnerability-config *ngIf="withClair" #vulnerabilityConfig [(vulnerabilityConfig)]="config" [showSubTitle]="true" [clairDBStatus]="clairDB"></vulnerability-config>
<vulnerability-config *ngIf="withClair" #vulnerabilityConfig [(vulnerabilityConfig)]="config" [showSubTitle]="true"></vulnerability-config>
<div>
<button type="button" class="btn btn-primary" (click)="save()" [disabled]="shouldDisable">{{'BUTTON.SAVE' | translate}}</button>
<button type="button" class="btn btn-outline" (click)="cancel()" [disabled]="shouldDisable">{{'BUTTON.CANCEL' | translate}}</button>

5
src/ui_ng/lib/src/config/registry-config.component.spec.ts
View File

@ -52,7 +52,8 @@ describe('RegistryConfigComponent (inline template)', () => {
"project_creation_restriction": "everyone",
"self_registration": true,
"has_ca_root": true,
"harbor_version": "v1.1.1-rc1-160-g565110d"
"harbor_version": "v1.1.1-rc1-160-g565110d",
"next_scan_all": 0
};
beforeEach(async(() => {
@ -85,7 +86,7 @@ describe('RegistryConfigComponent (inline template)', () => {
systemInfoService = fixture.debugElement.injector.get(SystemInfoService);
spy = spyOn(cfgService, 'getConfigurations').and.returnValue(Promise.resolve(mockConfig));
saveSpy = spyOn(cfgService, 'saveConfigurations').and.returnValue(Promise.resolve(true));
spySystemInfo = spyOn(systemInfoService, 'getSystemInfo').and.returnValues(Promise.resolve(mockSystemInfo));
spySystemInfo = spyOn(systemInfoService, 'getSystemInfo').and.returnValue(Promise.resolve(mockSystemInfo));
fixture.detectChanges();
});

26
src/ui_ng/lib/src/config/registry-config.component.ts
View File

@ -54,17 +54,8 @@ export class RegistryConfigComponent implements OnInit {
return this.systemInfo && this.systemInfo.with_clair;
}
get clairDB(): ClairDBStatus {
return this.systemInfo && this.systemInfo.clair_vulnerability_status ?
this.systemInfo.clair_vulnerability_status : null;
}
ngOnInit(): void {
//Get system info
toPromise<SystemInfo>(this.systemInfoService.getSystemInfo())
.then((info: SystemInfo) => this.systemInfo = info)
.catch(error => this.errorHandler.error(error));
this.loadSystemInfo();
//Initialize
this.load();
}
@ -82,20 +73,25 @@ export class RegistryConfigComponent implements OnInit {
return !this._isEmptyObject(this.getChanges());
}
//Get system info
loadSystemInfo(): void {
toPromise<SystemInfo>(this.systemInfoService.getSystemInfo())
.then((info: SystemInfo) => this.systemInfo = info)
.catch(error => this.errorHandler.error(error));
}
//Load configurations
load(): void {
this.onGoing = true;
toPromise<Configuration>(this.configService.getConfigurations())
.then((config: Configuration) => {
this.onGoing = false;
this.configCopy = this._clone(config);
this.config = config;
this.onGoing = false;
})
.catch(error => {
this.onGoing = false;
this.errorHandler.error(error);
this.onGoing = false;
});
}
@ -118,6 +114,8 @@ export class RegistryConfigComponent implements OnInit {
});
//Reload to fetch all the updates
this.load();
//Reload all system info
//this.loadSystemInfo();
})
.catch(error => {
this.onGoing = false;

7
src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.template.ts
View File

@ -12,13 +12,13 @@ export const VULNERABILITY_CONFIG_HTML: string = `
</clr-tooltip>
<clr-dropdown *ngIf="isClairDBFullyReady" [clrMenuPosition]="'bottom-right'" style="margin-top:-8px;" class="clr-dropdown-override">
<button class="btn btn-link btn-font" clrDropdownToggle>
{{ updatedTimestamp }}
{{ updatedTimestamp | date:'MM/dd/y HH:mm:ss' }}
<clr-icon shape="caret down"></clr-icon>
</button>
<div class="dropdown-menu" style="min-width:300px;">
<div *ngFor="let nt of namespaceTimestamps" class="namespace">
<span class="label label-info">{{nt.namespace}}</span>
<span>{{convertToLocalTime(nt.last_update*1000)}}</span>
<span>{{ convertToLocalTime(nt.last_update) | date:'MM/dd/y HH:mm:ss'}}</span>
</div>
</div>
</clr-dropdown>
@ -38,7 +38,8 @@ export const VULNERABILITY_CONFIG_HTML: string = `
</a>
</div>
<div class="form-group form-group-override">
<button class="btn btn-primary btn-sm" style="width:160px;" (click)="scanNow()">{{ 'CONFIG.SCANNING.SCAN_NOW' | translate }}</button>
<button class="btn btn-primary btn-sm" style="width:160px;" (click)="scanNow()" [disabled]="!scanAvailable">{{ 'CONFIG.SCANNING.SCAN_NOW' | translate }}</button>
<span style="margin-top: 12px;" *ngIf="!scanAvailable">{{ 'CONFIG.SCANNING.NEXT_SCAN' | translate }} {{ nextScanTimestamp | date:'HH:mm' }}</span>
</div>
</section>
</form>

103
src/ui_ng/lib/src/config/vulnerability/vulnerability-config.component.ts
View File

@ -1,9 +1,13 @@
import { Component, Input, Output, EventEmitter, ViewChild } from '@angular/core';
import { Component, Input, Output, EventEmitter, ViewChild, OnInit } from '@angular/core';
import { NgForm } from '@angular/forms';
import { Configuration } from '../config';
import { VULNERABILITY_CONFIG_HTML, VULNERABILITY_CONFIG_STYLES } from './vulnerability-config.component.template';
import { ScanningResultService } from '../../service/scanning.service';
import {
ScanningResultService,
SystemInfo,
SystemInfoService
} from '../../service/index';
import { ErrorHandler } from '../../error-handler';
import { toPromise } from '../../utils';
import { TranslateService } from '@ngx-translate/core';
@ -19,9 +23,10 @@ const ONE_DAY_SECONDS: number = 24 * ONE_HOUR_SECONDS;
template: VULNERABILITY_CONFIG_HTML,
styles: [VULNERABILITY_CONFIG_STYLES, REGISTRY_CONFIG_STYLES]
})
export class VulnerabilityConfigComponent {
export class VulnerabilityConfigComponent implements OnInit {
_localTime: Date = new Date();
onSubmitting: boolean = false;
config: Configuration;
openState: boolean = false;
@Output() configChange: EventEmitter<Configuration> = new EventEmitter<Configuration>();
@ -46,21 +51,37 @@ export class VulnerabilityConfigComponent {
}
@Input() showSubTitle: boolean = false;
@Input() clairDBStatus: ClairDBStatus;
systemInfo: SystemInfo;
get updatedTimestamp(): string {
if (this.clairDBStatus && this.clairDBStatus.overall_last_update > 0) {
return this.convertToLocalTime(this.clairDBStatus.overall_last_update*1000);
get scanAvailable(): boolean {
let dt: Date = new Date();
return !this.onSubmitting && (this.nextScanTime <= 0 || dt.getTime() > ((this.nextScanTime + 300) * 1000));
}
get nextScanTimestamp(): Date {
return this.nextScanTime > 0 ? this.convertToLocalTime(this.nextScanTime) : null;
}
get nextScanTime(): number {
return this.systemInfo && this.systemInfo.next_scan_all ? this.systemInfo.next_scan_all : 0;
}
get updatedTimestamp(): Date {
if (this.systemInfo &&
this.systemInfo.clair_vulnerability_status &&
this.systemInfo.clair_vulnerability_status.overall_last_update > 0) {
return this.convertToLocalTime(this.systemInfo.clair_vulnerability_status.overall_last_update);
}
return "--";
return null;
}
get namespaceTimestamps(): ClairDetail[] {
if (this.clairDBStatus &&
this.clairDBStatus.details &&
this.clairDBStatus.details.length > 0) {
return this.clairDBStatus.details;
if (this.systemInfo &&
this.systemInfo.clair_vulnerability_status &&
this.systemInfo.clair_vulnerability_status.details &&
this.systemInfo.clair_vulnerability_status.details.length > 0) {
return this.systemInfo.clair_vulnerability_status.details;
}
return [];
@ -207,31 +228,67 @@ export class VulnerabilityConfigComponent {
}
get isClairDBFullyReady(): boolean {
return this.clairDBStatus && this.clairDBStatus.overall_last_update > 0;
return this.systemInfo &&
this.systemInfo.clair_vulnerability_status &&
this.systemInfo.clair_vulnerability_status.overall_last_update > 0;
}
constructor(
private scanningService: ScanningResultService,
private errorHandler: ErrorHandler,
private translate: TranslateService) { }
private translate: TranslateService,
private systemInfoService: SystemInfoService
) { }
convertToLocalTime(utcTime: number): string {
let offset: number = this._localTime.getTimezoneOffset() * 60;
let timeWithLocal: number = utcTime - offset;
let dt = new Date();
dt.setTime(timeWithLocal);
return dt.toLocaleString();
ngOnInit(): void {
this.getSystemInfo();
}
convertToLocalTime(utcTime: number): Date {
let dt: Date = new Date();
dt.setTime(utcTime * 1000);
return dt;
}
scanNow(): void {
if (this.onSubmitting) {
return;//Aoid duplicated submitting
}
if(!this.scanAvailable) {
return; //Aoid page hacking
}
this.onSubmitting = true;
toPromise<any>(this.scanningService.startScanningAll())
.then(() => {
this.translate.get("CONFIG.SCANNING.TRIGGER_SCAN_ALL_SUCCESS").subscribe((res: string) => {
this.errorHandler.info(res);
});
//TODO:
//Change button disable status.
//Update system info
this.getSystemInfo().then(() => {
this.onSubmitting = false;
}).catch(() => {
this.onSubmitting = false;
});
})
.catch(error => this.errorHandler.error(error))
.catch(error => {
if (error && error.status && error.status === 412) {
this.translate.get("CONFIG.SCANNING.TRIGGER_SCAN_ALL_FAIL", { error: '' + error }).subscribe((res: string) => {
this.errorHandler.error(res);
});
} else {
this.errorHandler.error(error);
}
this.onSubmitting = false;
});
}
getSystemInfo(): Promise<SystemInfo> {
return toPromise<SystemInfo>(this.systemInfoService.getSystemInfo())
.then((info: SystemInfo) => this.systemInfo = info)
.catch(error => this.errorHandler.error(error));
}
}

10
src/ui_ng/lib/src/create-edit-rule/create-edit-rule.component.spec.ts
View File

@ -18,7 +18,12 @@ import { ReplicationRule, ReplicationJob, Endpoint } from '../service/interface'
import { ErrorHandler } from '../error-handler/error-handler';
import { SERVICE_CONFIG, IServiceConfig } from '../service.config';
import { ReplicationService, ReplicationDefaultService } from '../service/replication.service';
import {
ReplicationService,
ReplicationDefaultService,
JobLogService,
JobLogDefaultService
} from '../service/index';
import { EndpointService, EndpointDefaultService } from '../service/endpoint.service';
import { JobLogViewerComponent } from '../job-log-viewer/job-log-viewer.component';
@ -183,7 +188,8 @@ describe('CreateEditRuleComponent (inline template)', ()=>{
ErrorHandler,
{ provide: SERVICE_CONFIG, useValue: config },
{ provide: ReplicationService, useClass: ReplicationDefaultService },
{ provide: EndpointService, useClass: EndpointDefaultService }
{ provide: EndpointService, useClass: EndpointDefaultService },
{ provide: JobLogService, useClass: JobLogDefaultService }
]
});
}));

14
src/ui_ng/lib/src/harbor-library.module.ts
View File

@ -41,7 +41,9 @@ import {
ScanningResultService,
ScanningResultDefaultService,
ConfigurationService,
ConfigurationDefaultService
ConfigurationDefaultService,
JobLogService,
JobLogDefaultService
} from './service/index';
import {
ErrorHandler,
@ -74,7 +76,8 @@ export const DefaultServiceConfig: IServiceConfig = {
langMessagePathForHttpLoader: "i18n/langs/",
langMessageFileSuffixForHttpLoader: "-lang.json",
localI18nMessageVariableMap: {},
configurationEndpoint: "/api/configurations"
configurationEndpoint: "/api/configurations",
scanJobEndpoint: "/api/jobs/scan"
};
/**
@ -112,7 +115,10 @@ export interface HarborModuleConfig {
scanningService?: Provider,
//Service implementation for configuration
configService?: Provider
configService?: Provider,
//Service implementation for job log
jobLogService?: Provider
}
/**
@ -197,6 +203,7 @@ export class HarborLibraryModule {
config.tagService || { provide: TagService, useClass: TagDefaultService },
config.scanningService || { provide: ScanningResultService, useClass: ScanningResultDefaultService },
config.configService || { provide: ConfigurationService, useClass: ConfigurationDefaultService },
config.jobLogService || { provide: JobLogService, useClass: JobLogDefaultService },
//Do initializing
TranslateServiceInitializer,
{
@ -224,6 +231,7 @@ export class HarborLibraryModule {
config.tagService || { provide: TagService, useClass: TagDefaultService },
config.scanningService || { provide: ScanningResultService, useClass: ScanningResultDefaultService },
config.configService || { provide: ConfigurationService, useClass: ConfigurationDefaultService },
config.jobLogService || { provide: JobLogService, useClass: JobLogDefaultService },
ChannelService
]
};

12
src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.spec.ts
View File

@ -2,7 +2,7 @@ import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
import { DebugElement } from '@angular/core';
import { Observable } from 'rxjs/Observable';
import { ReplicationService, ReplicationDefaultService } from '../service/index';
import { JobLogService, JobLogDefaultService } from '../service/index';
import { JobLogViewerComponent } from './job-log-viewer.component';
import { SERVICE_CONFIG, IServiceConfig } from '../service.config';
@ -13,7 +13,7 @@ describe('JobLogViewerComponent (inline template)', () => {
let component: JobLogViewerComponent;
let fixture: ComponentFixture<JobLogViewerComponent>;
let serviceConfig: IServiceConfig;
let replicationService: ReplicationService;
let jobLogService: JobLogDefaultService;
let spy: jasmine.Spy;
let testConfig: IServiceConfig = {
replicationJobEndpoint: "/api/jobs/replication/testing"
@ -29,7 +29,7 @@ describe('JobLogViewerComponent (inline template)', () => {
providers: [
ErrorHandler,
{ provide: SERVICE_CONFIG, useValue: testConfig },
{ provide: ReplicationService, useClass: ReplicationDefaultService }
{ provide: JobLogService, useClass: JobLogDefaultService }
]
});
}));
@ -39,9 +39,9 @@ describe('JobLogViewerComponent (inline template)', () => {
component = fixture.componentInstance;
serviceConfig = TestBed.get(SERVICE_CONFIG);
replicationService = fixture.debugElement.injector.get(ReplicationService);
spy = spyOn(replicationService, 'getJobLog')
.and.returnValues(Promise.resolve("job log text"));
jobLogService = fixture.debugElement.injector.get(JobLogService);
spy = spyOn(jobLogService, 'getJobLog')
.and.returnValue(Promise.resolve("job log text"));
fixture.detectChanges();
});

2
src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.template.ts
View File

@ -1,6 +1,6 @@
export const JOB_LOG_VIEWER_TEMPLATE: string = `
<clr-modal [(clrModalOpen)]="opened" [clrModalStaticBackdrop]="true" [clrModalSize]="'xl'">
<h3 class="modal-title" class="log-viewer-title" style="margin-top: 0px;">{{'REPLICATION.JOB_LOG_VIEWER' | translate }}</h3>
<h3 class="modal-title" class="log-viewer-title" style="margin-top: 0px;">{{title | translate }}</h3>
<div class="modal-body">
<div class="loading-back" [hidden]="!onGoing">
<span class="spinner spinner-md"></span>

30
src/ui_ng/lib/src/job-log-viewer/job-log-viewer.component.ts
View File

@ -11,13 +11,15 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { Component } from '@angular/core';
import { Component, Input } from '@angular/core';
import { JOB_LOG_VIEWER_TEMPLATE, JOB_LOG_VIEWER_STYLES } from './job-log-viewer.component.template';
import { ReplicationService } from '../service/index';
import { JobLogService } from '../service/index';
import { ErrorHandler } from '../error-handler/index';
import { toPromise } from '../utils';
const supportSet: string[] = ["replication", "scan"];
@Component({
selector: 'job-log-viewer',
template: JOB_LOG_VIEWER_TEMPLATE,
@ -25,12 +27,32 @@ import { toPromise } from '../utils';
})
export class JobLogViewerComponent {
_jobType: string = "replication";
opened: boolean = false;
log: string = '';
onGoing: boolean = true;
@Input()
get jobType(): string {
return this._jobType;
}
set jobType(v: string) {
if (supportSet.find((t: string) => t === v)) {
this._jobType = v;
}
}
get title(): string {
if(this.jobType === "scan"){
return "VULNERABILITY.JOB_LOG_VIEWER";
}
return "REPLICATION.JOB_LOG_VIEWER";
}
constructor(
private replicationService: ReplicationService,
private jobLogService: JobLogService,
private errorHandler: ErrorHandler
) { }
@ -47,7 +69,7 @@ export class JobLogViewerComponent {
load(jobId: number | string): void {
this.onGoing = true;
toPromise<string>(this.replicationService.getJobLog(jobId))
toPromise<string>(this.jobLogService.getJobLog(this.jobType, jobId))
.then((log: string) => {
this.onGoing = false;
this.log = log;

2
src/ui_ng/lib/src/log/recent-log.template.ts
View File

@ -32,7 +32,7 @@ export const LOG_TEMPLATE: string = `
</clr-dg-row>
<clr-dg-footer>
{{pagination.firstItem + 1}} - {{pagination.lastItem + 1}}
of {{pagination.totalItems}} {{'AUDIT_LOG.ITEMS' | translate}}
{{'AUDIT_LOG.OF' | translate}} {{pagination.totalItems}} {{'AUDIT_LOG.ITEMS' | translate}}
<clr-dg-pagination #pagination [(clrDgPage)]="currentPage" [clrDgPageSize]="pageSize" [clrDgTotalItems]="totalCount"></clr-dg-pagination>
</clr-dg-footer>
</clr-datagrid>

4
src/ui_ng/lib/src/replication/replication.component.spec.ts
View File

@ -19,6 +19,7 @@ import { SERVICE_CONFIG, IServiceConfig } from '../service.config';
import { ReplicationService, ReplicationDefaultService } from '../service/replication.service';
import { EndpointService, EndpointDefaultService } from '../service/endpoint.service';
import { JobLogViewerComponent } from '../job-log-viewer/job-log-viewer.component';
import { JobLogService, JobLogDefaultService } from '../service/index';
describe('Replication Component (inline template)', ()=>{
@ -183,7 +184,8 @@ describe('Replication Component (inline template)', ()=>{
ErrorHandler,
{ provide: SERVICE_CONFIG, useValue: config },
{ provide: ReplicationService, useClass: ReplicationDefaultService },
{ provide: EndpointService, useClass: EndpointDefaultService }
{ provide: EndpointService, useClass: EndpointDefaultService },
{ provide: JobLogService, useClass: JobLogDefaultService }
]
});
}));

4
src/ui_ng/lib/src/repository-stackview/repository-stackview.component.spec.ts
View File

@ -17,6 +17,7 @@ import { SystemInfoService, SystemInfoDefaultService } from '../service/system-i
import { VULNERABILITY_DIRECTIVES } from '../vulnerability-scanning/index';
import { PUSH_IMAGE_BUTTON_DIRECTIVES } from '../push-image/index';
import { INLINE_ALERT_DIRECTIVES } from '../inline-alert/index';
import { JobLogViewerComponent } from '../job-log-viewer/index';
import { click } from '../utils';
@ -101,7 +102,8 @@ describe('RepositoryComponentStackview (inline template)', () => {
FilterComponent,
VULNERABILITY_DIRECTIVES,
PUSH_IMAGE_BUTTON_DIRECTIVES,
INLINE_ALERT_DIRECTIVES
INLINE_ALERT_DIRECTIVES,
JobLogViewerComponent
],
providers: [
ErrorHandler,

8
src/ui_ng/lib/src/service.config.ts
View File

@ -180,4 +180,12 @@ export interface IServiceConfig {
* @memberOf IServiceConfig
*/
configurationEndpoint?: string;
/**
* The base endpoint of scan job service.
*
* @type {string}
* @memberof IServiceConfig
*/
scanJobEndpoint?: string;
}

1
src/ui_ng/lib/src/service/index.ts
View File

@ -8,3 +8,4 @@ export * from './tag.service';
export * from './RequestQueryParams';
export * from './scanning.service';
export * from './configuration.service';
export * from './job-log.service';

1
src/ui_ng/lib/src/service/interface.ts
View File

@ -168,6 +168,7 @@ export interface SystemInfo {
has_ca_root?: boolean;
harbor_version?: string;
clair_vulnerability_status?: ClairDBStatus;
next_scan_all?: number;
}
/**

39
src/ui_ng/lib/src/service/job-log.service.spec.ts Normal file
View File

@ -0,0 +1,39 @@
import { TestBed, inject } from '@angular/core/testing';
import { JobLogService, JobLogDefaultService } from './job-log.service';
import { SharedModule } from '../shared/shared.module';
import { SERVICE_CONFIG, IServiceConfig } from '../service.config';
describe('JobLogService', () => {
const mockConfig: IServiceConfig = {
replicationJobEndpoint: "/api/jobs/replication/testing",
scanJobEndpoint: "/api/jobs/scan/testing"
};
let config: IServiceConfig;
beforeEach(() => {
TestBed.configureTestingModule({
imports: [
SharedModule
],
providers: [
JobLogDefaultService,
{
provide: JobLogService,
useClass: JobLogDefaultService
}, {
provide: SERVICE_CONFIG,
useValue: mockConfig
}]
});
config = TestBed.get(SERVICE_CONFIG);
});
it('should be initialized', inject([JobLogDefaultService], (service: JobLogService) => {
expect(service).toBeTruthy();
expect(config.replicationJobEndpoint).toEqual("/api/jobs/replication/testing");
expect(config.scanJobEndpoint).toEqual("/api/jobs/scan/testing");
}));
});

84
src/ui_ng/lib/src/service/job-log.service.ts Normal file
View File

@ -0,0 +1,84 @@
import { Observable } from 'rxjs/Observable';
import { RequestQueryParams } from './RequestQueryParams';
import { ReplicationJob, ReplicationRule } from './interface';
import { Injectable, Inject } from "@angular/core";
import 'rxjs/add/observable/of';
import { Http, RequestOptions } from '@angular/http';
import { SERVICE_CONFIG, IServiceConfig } from '../service.config';
import { buildHttpRequestOptions, HTTP_JSON_OPTIONS } from '../utils';
/**
* Define the service methods to handle the job log related things.
*
* @export
* @abstract
* @class JobLogService
*/
export abstract class JobLogService {
/**
* Get the log of the specified job
*
* @abstract
* @param {string} jobType
* @param {(number | string)} jobId
* @returns {(Observable<string> | Promise<string> | string)}
* @memberof JobLogService
*/
abstract getJobLog(jobType: string, jobId: number | string): Observable<string> | Promise<string> | string;
}
/**
* Implement default service for job log service.
*
* @export
* @class JobLogDefaultService
* @extends {ReplicationService}
*/
@Injectable()
export class JobLogDefaultService extends JobLogService {
_replicationJobBaseUrl: string;
_scanningJobBaseUrl: string;
_supportedJobTypes: string[];
constructor(
private http: Http,
@Inject(SERVICE_CONFIG) config: IServiceConfig
) {
super();
this._replicationJobBaseUrl = config.replicationJobEndpoint ?
config.replicationJobEndpoint : '/api/jobs/replication';
this._scanningJobBaseUrl = config.scanJobEndpoint ? config.scanJobEndpoint : "/api/jobs/scan";
this._supportedJobTypes = ["replication", "scan"];
}
_getJobLog(logUrl: string): Observable<string> | Promise<string> | string {
return this.http.get(logUrl).toPromise()
.then(response => response.text())
.catch(error => Promise.reject(error));
}
_isSupportedJobType(jobType: string): boolean {
if (this._supportedJobTypes.find((t: string) => t === jobType)) {
return true;
}
return false;
}
public getJobLog(jobType: string, jobId: number | string): Observable<string> | Promise<string> | string {
if (!this._isSupportedJobType(jobType)) {
return Promise.reject("Unsupport job type: " + jobType);
}
if (!jobId || jobId <= 0) {
return Promise.reject('Bad argument');
}
let logUrl: string = `${this._replicationJobBaseUrl}/${jobId}/log`;
if (jobType === "scan") {
logUrl = `${this._scanningJobBaseUrl}/${jobId}/log`;
}
return this._getJobLog(logUrl);
}
}

1
src/ui_ng/lib/src/tag/tag.component.css.ts
View File

@ -14,7 +14,6 @@ export const TAG_STYLE = `
:host >>> .datagrid {
margin: 0;
border-bottom: none;
}
:host >>> .datagrid-placeholder {

5
src/ui_ng/lib/src/tag/tag.component.spec.ts
View File

@ -17,6 +17,8 @@ import { FILTER_DIRECTIVES } from '../filter/index'
import { Observable, Subscription } from 'rxjs/Rx';
import { ChannelService } from '../channel/index';
import { JobLogViewerComponent } from '../job-log-viewer/index';
describe('TagComponent (inline template)', () => {
let comp: TagComponent;
@ -49,7 +51,8 @@ describe('TagComponent (inline template)', () => {
TagComponent,
ConfirmationDialogComponent,
VULNERABILITY_DIRECTIVES,
FILTER_DIRECTIVES
FILTER_DIRECTIVES,
JobLogViewerComponent
],
providers: [
ErrorHandler,

1
src/ui_ng/lib/src/tag/tag.component.ts
View File

@ -241,6 +241,7 @@ export class TagComponent implements OnInit {
//Whether show the 'scan now' menu
canScanNow(t: Tag): boolean {
if (!this.withClair) { return false; }
if (!this.hasProjectAdminRole) { return false; }
let st: string = this.scanStatus(t);
return st !== VULNERABILITY_SCAN_STATUS.pending &&

11
src/ui_ng/lib/src/vulnerability-scanning/result-bar-chart.component.spec.ts
View File

@ -8,13 +8,16 @@ import {
ScanningResultService,
ScanningResultDefaultService,
TagService,
TagDefaultService
TagDefaultService,
JobLogService,
JobLogDefaultService
} from '../service/index';
import { SERVICE_CONFIG, IServiceConfig } from '../service.config';
import { ErrorHandler } from '../error-handler/index';
import { SharedModule } from '../shared/shared.module';
import { VULNERABILITY_SCAN_STATUS } from '../utils';
import { ChannelService } from '../channel/index';
import { JobLogViewerComponent } from '../job-log-viewer/index';
describe('ResultBarChartComponent (inline template)', () => {
let component: ResultBarChartComponent;
@ -52,13 +55,15 @@ describe('ResultBarChartComponent (inline template)', () => {
],
declarations: [
ResultBarChartComponent,
ResultTipComponent],
ResultTipComponent,
JobLogViewerComponent],
providers: [
ErrorHandler,
ChannelService,
{ provide: SERVICE_CONFIG, useValue: testConfig },
{ provide: TagService, useValue: TagDefaultService },
{ provide: ScanningResultService, useValue: ScanningResultDefaultService }
{ provide: ScanningResultService, useValue: ScanningResultDefaultService },
{ provide: JobLogService, useValue: JobLogDefaultService}
]
});

14
src/ui_ng/lib/src/vulnerability-scanning/result-bar-chart.component.ts
View File

@ -4,7 +4,8 @@ import {
OnInit,
OnDestroy,
ChangeDetectionStrategy,
ChangeDetectorRef
ChangeDetectorRef,
ViewChild
} from '@angular/core';
import { SCANNING_STYLES } from './scanning.css';
@ -21,6 +22,7 @@ import { ErrorHandler } from '../error-handler/index';
import { toPromise } from '../utils';
import { Observable, Subscription } from 'rxjs/Rx';
import { ChannelService } from '../channel/index';
import { JobLogViewerComponent } from '../job-log-viewer/index';
const STATE_CHECK_INTERVAL: number = 2000;//2s
const RETRY_TIMES: number = 3;
@ -39,6 +41,9 @@ export class ResultBarChartComponent implements OnInit, OnDestroy {
stateCheckTimer: Subscription;
timerHandler: any;
@ViewChild("scanningLogViewer")
scanningJobLogViewer: JobLogViewerComponent;
constructor(
private tagService: TagService,
private scanningService: ScanningResultService,
@ -188,4 +193,11 @@ export class ResultBarChartComponent implements OnInit, OnDestroy {
}
}, duration);
}
//Check error log
viewLog(): void {
if (this.summary && this.summary.job_id) {
this.scanningJobLogViewer.open(this.summary.job_id);
}
}
}

18
src/ui_ng/lib/src/vulnerability-scanning/result-tip.component.ts
View File

@ -66,9 +66,21 @@ export class ResultTipComponent implements OnInit {
}
});
}
this.translate.get('VULNERABILITY.CHART.TOOLTIPS_TITLE',
{ totalVulnerability: this.packagesWithVul, totalPackages: this.totalPackages })
.subscribe((res: string) => this._tipTitle = res);
this.translate.get(this.packageText(this.totalPackages)).subscribe((p1: string) => {
this.translate.get(this.packageText(this.packagesWithVul)).subscribe((p2: string) => {
this.translate.get(this.unitText(this.packagesWithVul)).subscribe((vul: string) => {
this.translate.get('VULNERABILITY.CHART.TOOLTIPS_TITLE',
{
totalVulnerability: this.packagesWithVul,
totalPackages: this.totalPackages,
package: p1,
packageExt: p2,
vulnerability: vul
})
.subscribe((res: string) => this._tipTitle = res);
});
});
});
}
tipWidth(severity: VulnerabilitySeverity): string {

8
src/ui_ng/lib/src/vulnerability-scanning/scanning.css.ts
View File

@ -20,6 +20,7 @@ export const SCANNING_STYLES: string = `
position: relative;
top: 1px;
margin-left: -5px;
cursor: pointer;
}
.scanning-button {
@ -45,10 +46,10 @@ export const SCANNING_STYLES: string = `
background-color: red;
}
.bar-block-medium {
background-color: orange;
background-color: yellow;
}
.bar-block-low {
background-color: yellow;
background-color: orange;
}
.bar-block-none {
background-color: green;
@ -83,4 +84,7 @@ export const SCANNING_STYLES: string = `
.refresh-btn:hover {
color: #007CBB;
}
.tip-icon-low {
color: orange;
}
`;

19
src/ui_ng/lib/src/vulnerability-scanning/scanning.html.ts
View File

@ -13,23 +13,23 @@ export const TIP_COMPONENT_HTML: string = `
<div class="bar-summary bar-tooltip-fon">
<div *ngIf="hasHigh" class="bar-summary-item">
<clr-icon shape="exclamation-circle" class="is-error" size="24"></clr-icon>
<span>{{highCount}} {{packageText(highCount) | translate }} {{'VULNERABILITY.SEVERITY.HIGH' | translate }}</span>
<span>{{highCount}} {{'VULNERABILITY.SEVERITY.HIGH' | translate }}</span>
</div>
<div *ngIf="hasMedium" class="bar-summary-item">
<clr-icon *ngIf="hasMedium" shape="exclamation-triangle" class="is-warning" size="22"></clr-icon>
<span>{{mediumCount}} {{packageText(mediumCount) | translate }} {{'VULNERABILITY.SEVERITY.MEDIUM' | translate }}</span>
<span>{{mediumCount}} {{'VULNERABILITY.SEVERITY.MEDIUM' | translate }}</span>
</div>
<div *ngIf="hasLow" class="bar-summary-item">
<clr-icon shape="info-circle" class="is-info" size="24"></clr-icon>
<span>{{lowCount}} {{packageText(lowCount) | translate }} {{'VULNERABILITY.SEVERITY.LOW' | translate }}</span>
<clr-icon shape="info-circle" class="tip-icon-low" size="24"></clr-icon>
<span>{{lowCount}} {{'VULNERABILITY.SEVERITY.LOW' | translate }}</span>
</div>
<div *ngIf="hasUnknown" class="bar-summary-item">
<clr-icon shape="help" size="20"></clr-icon>
<span>{{unknownCount}} {{packageText(unknownCount) | translate }} {{'VULNERABILITY.SEVERITY.UNKNOWN' | translate }}</span>
<span>{{unknownCount}} {{'VULNERABILITY.SEVERITY.UNKNOWN' | translate }}</span>
</div>
<div *ngIf="hasNone" class="bar-summary-item">
<clr-icon shape="check-circle" class="is-success" size="24"></clr-icon>
<span>{{noneCount}} {{packageText(noneCount) | translate }} {{'VULNERABILITY.SEVERITY.NONE' | translate }}</span>
<span>{{noneCount}} {{'VULNERABILITY.SEVERITY.NONE' | translate }}</span>
</div>
</div>
<div>
@ -96,8 +96,10 @@ export const BAR_CHART_COMPONENT_HTML: string = `
<span class="label label-orange">{{'VULNERABILITY.STATE.QUEUED' | translate}}</span>
</div>
<div *ngIf="error" class="bar-state bar-state-error">
<clr-icon shape="info-circle" class="is-error" size="24"></clr-icon>
<span class="error-text">{{'VULNERABILITY.STATE.ERROR' | translate}}</span>
<a href="javascript:void(0);" class="error-text" (click)="viewLog()">
<clr-icon shape="error" class="is-error" size="24"></clr-icon>
{{'VULNERABILITY.STATE.ERROR' | translate}}
</a>
</div>
<div *ngIf="scanning" class="bar-state bar-state-chart">
<div>{{'VULNERABILITY.STATE.SCANNING' | translate}}</div>
@ -110,5 +112,6 @@ export const BAR_CHART_COMPONENT_HTML: string = `
<clr-icon shape="warning" class="is-warning" size="24"></clr-icon>
<span style="margin-left:-5px;">{{'VULNERABILITY.STATE.UNKNOWN' | translate}}</span>
</div>
<job-log-viewer #scanningLogViewer [jobType]="'scan'"></job-log-viewer>
</div>
`;

2
src/ui_ng/package.json
View File

@ -31,7 +31,7 @@
"clarity-icons": "^0.9.8",
"clarity-ui": "^0.9.8",
"core-js": "^2.4.1",
"harbor-ui": "0.3.24",
"harbor-ui": "0.3.54",
"intl": "^1.2.5",
"mutationobserver-shim": "^0.3.2",
"ngx-cookie": "^1.0.0",

2
src/ui_ng/src/app/app-config.ts
View File

@ -30,6 +30,7 @@ export class AppConfig {
overall_last_update: 0,
details: []
};
this.next_scan_all = 0;
}
with_notary: boolean;
@ -43,4 +44,5 @@ export class AppConfig {
has_ca_root: boolean;
harbor_version: string;
clair_vulnerability_status?: ClairDBStatus;
next_scan_all: number;
}

2
src/ui_ng/src/app/config/config.component.html
View File

@ -32,7 +32,7 @@
<system-settings [(systemSettings)]="allConfig" [hasAdminRole]="hasAdminRole" [hasCAFile]="hasCAFile"></system-settings>
</section>
<section id="vulnerability" *ngIf="withClair" role="tabpanel" aria-labelledby="config-vulnerability" [hidden]='!isCurrentTabContent("vulnerability")'>
<vulnerability-config [(vulnerabilityConfig)]="allConfig" [clairDBStatus]="clairDB"></vulnerability-config>
<vulnerability-config [(vulnerabilityConfig)]="allConfig"></vulnerability-config>
</section>
<div>
<button type="button" class="btn btn-primary" (click)="save()" [disabled]="!isValid() || !hasChanges()">{{'BUTTON.SAVE' | translate}}</button>

4
src/ui_ng/src/app/config/config.component.ts
View File

@ -85,10 +85,6 @@ export class ConfigurationComponent implements OnInit, OnDestroy {
return this.appConfigService.getConfig().with_clair;
}
public get clairDB(): ClairDBStatus {
return this.appConfigService.getConfig().clair_vulnerability_status;
}
isCurrentTabLink(tabId: string): boolean {
return this.currentTabId === tabId;
}

2
src/ui_ng/src/app/log/audit-log.component.html
View File

@ -58,7 +58,7 @@
<clr-dg-footer>
<span *ngIf="pagination.totalItems">{{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'AUDIT_LOG.OF' | translate}} </span>
{{pagination.totalItems }} {{'AUDIT_LOG.ITEMS' | translate}}
<clr-dg-pagination #pagination [clrDgPageSize]="15"></clr-dg-pagination>
<clr-dg-pagination #pagination [clrDgPageSize]="15" [(clrDgPage)]="currentPage" [clrDgTotalItems]="totalRecordCount"></clr-dg-pagination>
<!--{{totalRecordCount}} {{'AUDIT_LOG.ITEMS' | translate}}
<clr-dg-pagination [clrDgPageSize]="pageOffset" [clrDgTotalItems]="totalPage"></clr-dg-pagination>-->
</clr-dg-footer>

7
src/ui_ng/src/app/log/audit-log.component.ts
View File

@ -69,6 +69,7 @@ export class AuditLogComponent implements OnInit {
pageOffset: number = 1;
pageSize: number = 15;
totalRecordCount: number;
currentPage: number;
totalPage: number;
@ViewChild('fromTime') fromTimeInput: NgModel;
@ -96,14 +97,14 @@ export class AuditLogComponent implements OnInit {
retrieve(state?: State): void {
if(state) {
this.queryParam.page = state.page.to + 1;
this.queryParam.page = Math.ceil((state.page.to + 1) / this.pageSize);
this.currentPage = this.queryParam.page;
}
this.auditLogService
.listAuditLogs(this.queryParam)
.subscribe(
response=>{
this.totalRecordCount = response.headers.get('x-total-count');
this.totalPage = Math.ceil(this.totalRecordCount / this.pageSize);
this.totalRecordCount =parseInt(response.headers.get('x-total-count'));
this.auditLogs = response.json();
},
error=>{

2
src/ui_ng/src/app/replication/total-replication/total-replication-page.component.html
View File

@ -1,3 +1,3 @@
<div style="margin-top: 24px;">
<hbr-replication [withReplicationJob]='false'></hbr-replication>
<hbr-replication [withReplicationJob]='false' (redirect)="customRedirect($event)"></hbr-replication>
</div>

11
src/ui_ng/src/app/replication/total-replication/total-replication-page.component.ts
View File

@ -13,9 +13,20 @@
// limitations under the License.
import { Component } from '@angular/core';
import {Router,ActivatedRoute} from "@angular/router";
import {ReplicationRule} from "harbor-ui";
@Component({
selector: 'total-replication',
templateUrl: 'total-replication-page.component.html'
})
export class TotalReplicationPageComponent {
constructor(private router: Router,
private activeRoute: ActivatedRoute){}
customRedirect(rule: ReplicationRule): void {
if (rule) {
this.router.navigate(['../../projects', rule.project_id, "replications"], { relativeTo: this.activeRoute });
}
}
}

2
src/ui_ng/src/app/repository/tag-repository/tag-repository.component.html
View File

@ -1,5 +1,5 @@
<div>
<a *ngIf="hasSignedIn" [routerLink]="['/harbor', 'projects', projectId, 'repositories']">&lt; {{'REPOSITORY.REPOSITORIES' | translate}}</a>
<a *ngIf="!hasSignedIn" [routerLink]="['/harbor', 'sign-in']">&lt; {{'SEARCH.BACK' | translate}}</a>
<hbr-tag (tagClickEvent)="watchTagClickEvt($event)" [repoName]="repoName" [registryUrl]="registryUrl" [withNotary]="withNotary" [hasSignedIn]="hasSignedIn" [hasProjectAdminRole]="hasProjectAdminRole" [projectId]="projectId" [isEmbedded]="false"></hbr-tag>
<hbr-tag (tagClickEvent)="watchTagClickEvt($event)" [repoName]="repoName" [registryUrl]="registryUrl" [withNotary]="withNotary" [hasSignedIn]="hasSignedIn" [hasProjectAdminRole]="hasProjectAdminRole" [projectId]="projectId" [isEmbedded]="false" [withClair]="withClair"></hbr-tag>
</div>

16
src/ui_ng/src/app/repository/tag-repository/tag-repository.component.ts
View File

@ -29,8 +29,6 @@ export class TagRepositoryComponent implements OnInit {
repoName: string;
hasProjectAdminRole: boolean = false;
registryUrl: string;
withNotary: boolean;
hasSignedIn: boolean;
constructor(
private route: ActivatedRoute,
@ -40,7 +38,6 @@ export class TagRepositoryComponent implements OnInit {
}
ngOnInit() {
this.hasSignedIn = (this.session.getCurrentUser() !== null);
let resolverData = this.route.snapshot.data;
if (resolverData) {
this.hasProjectAdminRole = (<Project>resolverData['projectResolver']).has_project_admin_role;
@ -49,7 +46,18 @@ export class TagRepositoryComponent implements OnInit {
this.repoName = this.route.snapshot.params['repo'];
this.registryUrl = this.appConfigService.getConfig().registry_url;
this.withNotary = this.appConfigService.getConfig().with_notary;
}
get withNotary(): boolean {
return this.appConfigService.getConfig().with_notary;
}
get withClair(): boolean {
return this.appConfigService.getConfig().with_clair;
}
get hasSignedIn(): boolean {
return this.session.getCurrentUser() !== null;
}
watchTagClickEvt(tagEvt: TagClickEvent): void {

5
src/ui_ng/src/app/shared/list-project-ro/list-project-ro.component.html
View File

@ -10,9 +10,8 @@
<clr-dg-cell>{{p.creation_time | date: 'short'}}</clr-dg-cell>
</clr-dg-row>
<clr-dg-footer>
<span *ngIf="pagination.totalItems">{{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'PROJECT.OF' | translate}} </span>
{{pagination.totalItems }} {{'PROJECT.ITEMS' | translate}}
<clr-dg-pagination #pagination [clrDgPageSize]="15"></clr-dg-pagination>
<span *ngIf="pagination.totalItems">{{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'PROJECT.OF' | translate}} </span> {{pagination.totalItems }} {{'PROJECT.ITEMS' | translate}}
<clr-dg-pagination #pagination [clrDgPageSize]="5"></clr-dg-pagination>
<!--{{(projects ? projects.length : 0)}} {{'PROJECT.ITEMS' | translate}}
<clr-dg-pagination [clrDgPageSize]="15"></clr-dg-pagination>-->
</clr-dg-footer>

5
src/ui_ng/src/app/shared/list-repository-ro/list-repository-ro.component.html
View File

@ -8,8 +8,7 @@
<clr-dg-cell>{{r.pull_count}}</clr-dg-cell>
</clr-dg-row>
<clr-dg-footer>
<span *ngIf="pagination.totalItems">{{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'REPOSITORY.OF' | translate}} </span>
{{pagination.totalItems }} {{'REPOSITORY.ITEMS' | translate}}
<clr-dg-pagination #pagination [clrDgPageSize]="15"></clr-dg-pagination>
<span *ngIf="pagination.totalItems">{{pagination.firstItem + 1}} - {{pagination.lastItem +1 }} {{'REPOSITORY.OF' | translate}} </span> {{pagination.totalItems }} {{'REPOSITORY.ITEMS' | translate}}
<clr-dg-pagination #pagination [clrDgPageSize]="5"></clr-dg-pagination>
</clr-dg-footer>
</clr-datagrid>

29
src/ui_ng/src/i18n/lang/en-us-lang.json
View File

@ -117,7 +117,8 @@
"DELETION_TITLE": "Confirm user deletion",
"DELETION_SUMMARY": "Do you want to delete user {{param}}?",
"DELETE_SUCCESS": "User deleted successfully.",
"ITEMS": "items"
"ITEMS": "items",
"OF": "of"
},
"PROJECT": {
"PROJECTS": "Projects",
@ -152,7 +153,8 @@
"DELETED_SUCCESS": "Deleted project successfully.",
"TOGGLED_SUCCESS": "Toggled project successfully.",
"FAILED_TO_DELETE_PROJECT": "Project contains repositories or replication rules cannot be deleted.",
"INLINE_HELP_PUBLIC": "When a project is set to public, anyone has read permission to the repositories under this project, and the user does not need to run \"docker login\" before pulling images under this project."
"INLINE_HELP_PUBLIC": "When a project is set to public, anyone has read permission to the repositories under this project, and the user does not need to run \"docker login\" before pulling images under this project.",
"OF": "of"
},
"PROJECT_DETAIL": {
"REPOSITORIES": "Repositories",
@ -182,7 +184,8 @@
"DELETION_SUMMARY": "Do you want to delete project member {{param}}?",
"ADDED_SUCCESS": "Added member successfully.",
"DELETED_SUCCESS": "Deleted member successfully.",
"SWITCHED_SUCCESS": "Switched member role successfully."
"SWITCHED_SUCCESS": "Switched member role successfully.",
"OF": "of"
},
"AUDIT_LOG": {
"USERNAME": "Username",
@ -201,7 +204,8 @@
"SIMPLE": "Simple",
"ITEMS": "items",
"FILTER_PLACEHOLDER": "Filter Logs",
"INVALID_DATE": "Invalid date."
"INVALID_DATE": "Invalid date.",
"OF": "of"
},
"REPLICATION": {
"REPLICATION_RULE": "Replication Rule",
@ -409,6 +413,7 @@
},
"SCANNING": {
"TRIGGER_SCAN_ALL_SUCCESS": "Trigger scan all successfully!",
"TRIGGER_SCAN_ALL_FAIL": "Failed to trigger scan all with error: {{error}",
"TITLE": "Vulnerability Scanning",
"SCAN_ALL": "Scan All",
"SCAN_NOW": "SCAN NOW",
@ -416,7 +421,8 @@
"DAILY_POLICY": "Daily At",
"REFRESH_POLICY": "Upon Refresh",
"DB_REFRESH_TIME": "Database updated on",
"DB_NOT_READY": "Vulnerability database might not be fully ready!"
"DB_NOT_READY": "Vulnerability database might not be fully ready!",
"NEXT_SCAN": "Available after"
},
"TEST_MAIL_SUCCESS": "Connection to mail server is verified.",
"TEST_LDAP_SUCCESS": "Connection to LDAP server is verified.",
@ -464,7 +470,7 @@
"STATE": {
"STOPPED": "Not Scanned",
"QUEUED": "Queued",
"ERROR": "Error",
"ERROR": "View Log",
"SCANNING": "Scanning",
"UNKNOWN": "Unknown"
},
@ -480,8 +486,8 @@
"FOOT_OF": "of"
},
"CHART": {
"SCANNING_TIME": "Scan completed",
"TOOLTIPS_TITLE": "This tag has {{totalVulnerability}} package(s) with vulnerabilities across all {{totalPackages}} package(s)."
"SCANNING_TIME": "Scan completed datetime",
"TOOLTIPS_TITLE": "This image includes {{totalPackages}} {{package}} with {{vulnerability}} in {{totalVulnerability}} of the {{packageExt}}."
},
"SEVERITY": {
"HIGH": "high",
@ -494,9 +500,10 @@
"SINGULAR": "Vulnerability",
"PLURAL": "Vulnerabilities",
"PLACEHOLDER": "Filter Vulnerabilities",
"PACKAGE": "Package with",
"PACKAGES": "Packages with",
"SCAN_NOW": "Scan"
"PACKAGE": "package",
"PACKAGES": "packages",
"SCAN_NOW": "Scan",
"JOB_LOG_VIEWER": "View Scanning Job Log"
},
"PUSH_IMAGE": {
"TITLE": "Push Image",

29
src/ui_ng/src/i18n/lang/es-es-lang.json
View File

@ -117,7 +117,8 @@
"DELETION_TITLE": "Confirmar eliminación de usuario",
"DELETION_SUMMARY": "¿Quiere eliminar el usuario {{param}}?",
"DELETE_SUCCESS": "Usuario eliminado satisfactoriamente.",
"ITEMS": "elementos"
"ITEMS": "elementos",
"OF": "of"
},
"PROJECT": {
"PROJECTS": "Proyectos",
@ -152,7 +153,8 @@
"DELETED_SUCCESS": "Proyecto eliminado satisfactoriamente.",
"TOGGLED_SUCCESS": "Proyecto alternado satisfactoriamente.",
"FAILED_TO_DELETE_PROJECT": "Los proyectos que contienen repositorios o reglas de replicación no pueden eliminarse.",
"INLINE_HELP_PUBLIC": "Cuando un proyecto se marca como público, todo el mundo tiene permisos de lectura sobre los repositorio de dicho proyecto, y no hace falta hacer \"docker login\" antes de subir imágenes a ellos."
"INLINE_HELP_PUBLIC": "Cuando un proyecto se marca como público, todo el mundo tiene permisos de lectura sobre los repositorio de dicho proyecto, y no hace falta hacer \"docker login\" antes de subir imágenes a ellos.",
"OF": "of"
},
"PROJECT_DETAIL": {
"REPOSITORIES": "Repositorios",
@ -182,7 +184,8 @@
"DELETION_SUMMARY": "¿Quiere eliminar el miembro {{param}} del proyecto?",
"ADDED_SUCCESS": "Miembro añadido satisfactoriamente.",
"DELETED_SUCCESS": "Miembro eliminado satisfactoriamente",
"SWITCHED_SUCCESS": "Rol del miembro cambiado satisfactoriamente."
"SWITCHED_SUCCESS": "Rol del miembro cambiado satisfactoriamente.",
"OF": "of"
},
"AUDIT_LOG": {
"USERNAME": "Nombre de usuario",
@ -201,7 +204,8 @@
"SIMPLE": "Simple",
"ITEMS": "elementos",
"FILTER_PLACEHOLDER": "Filtrar logs",
"INVALID_DATE": "Fecha invalida."
"INVALID_DATE": "Fecha invalida.",
"OF": "of"
},
"REPLICATION": {
"REPLICATION_RULE": "Reglas de Replicación",
@ -410,6 +414,7 @@
},
"SCANNING": {
"TRIGGER_SCAN_ALL_SUCCESS": "Trigger scan all successfully!",
"TRIGGER_SCAN_ALL_FAIL": "Failed to trigger scan all with error: {{error}",
"TITLE": "Vulnerability Scanning",
"SCAN_ALL": "Scan All",
"SCAN_NOW": "SCAN NOW",
@ -417,7 +422,8 @@
"DAILY_POLICY": "Daily At",
"REFRESH_POLICY": "Upon Refresh",
"DB_REFRESH_TIME": "Database updated on",
"DB_NOT_READY": "Vulnerability database might not be fully ready!"
"DB_NOT_READY": "Vulnerability database might not be fully ready!",
"NEXT_SCAN": "Available after"
},
"TEST_MAIL_SUCCESS": "La conexión al servidor de correo ha sido verificada.",
"TEST_LDAP_SUCCESS": "La conexión al servidor LDAP ha sido verificada.",
@ -463,7 +469,7 @@
"STATE": {
"STOPPED": "Not Scanned",
"QUEUED": "Queued",
"ERROR": "Error",
"ERROR": "View Log",
"SCANNING": "Scanning",
"UNKNOWN": "Unknown"
},
@ -479,8 +485,8 @@
"FOOT_OF": "of"
},
"CHART": {
"SCANNING_TIME": "Scan completed",
"TOOLTIPS_TITLE": "This tag has {{totalVulnerability}} package(s) with vulnerabilities across all {{totalPackages}} package(s)."
"SCANNING_TIME": "Scan completed datetime",
"TOOLTIPS_TITLE": "This image includes {{totalPackages}} {{package}} with {{vulnerability}} in {{totalVulnerability}} of the {{packageExt}}."
},
"SEVERITY": {
"HIGH": "high",
@ -493,9 +499,10 @@
"SINGULAR": "Vulnerability",
"PLURAL": "Vulnerabilities",
"PLACEHOLDER": "Filter Vulnerabilities",
"PACKAGE": "Package with",
"PACKAGES": "Packages with",
"SCAN_NOW": "Scan"
"PACKAGE": "package",
"PACKAGES": "packages",
"SCAN_NOW": "Scan",
"JOB_LOG_VIEWER": "View Scanning Job Log"
},
"PUSH_IMAGE": {
"TITLE": "Push Image",

25
src/ui_ng/src/i18n/lang/zh-cn-lang.json
View File

@ -412,7 +412,8 @@
"SCOPE": "LDAP搜索范围"
},
"SCANNING": {
"TRIGGER_SCAN_ALL_SUCCESS": "成功启动扫描所有镜像任务!",
"TRIGGER_SCAN_ALL_SUCCESS": "启动扫描所有镜像任务成功!",
"TRIGGER_SCAN_ALL_FAIL": "启动扫描所有镜像任务失败:{{error}",
"TITLE": "缺陷扫描",
"SCAN_ALL": "扫描所有",
"SCAN_NOW": "开始扫描",
@ -420,7 +421,8 @@
"DAILY_POLICY": "每日定时",
"REFRESH_POLICY": "缺陷库刷新后",
"DB_REFRESH_TIME": "数据库更新于",
"DB_NOT_READY": "缺陷数据库可能没有完全准备好!"
"DB_NOT_READY": "缺陷数据库可能没有完全准备好!",
"NEXT_SCAN": "下次可用时间"
},
"TEST_MAIL_SUCCESS": "邮件服务器的连通正常。",
"TEST_LDAP_SUCCESS": "LDAP服务器的连通正常。",
@ -468,7 +470,7 @@
"STATE": {
"STOPPED": "未扫描",
"QUEUED": "已入队列",
"ERROR": "错误",
"ERROR": "查看日志",
"SCANNING": "扫描中",
"UNKNOWN": "未知"
},
@ -476,7 +478,7 @@
"PLACEHOLDER": "没有扫描结果！",
"COLUMN_ID": "缺陷码",
"COLUMN_SEVERITY": "严重度",
"COLUMN_PACKAGE": "组",
"COLUMN_PACKAGE": "组件",
"COLUMN_VERSION": "当前版本",
"COLUMN_FIXED": "修复版本",
"COLUMN_DESCRIPTION": "简介",
@ -485,7 +487,7 @@
},
"CHART": {
"SCANNING_TIME": "扫描完成",
"TOOLTIPS_TITLE": "在此镜像的{{totalPackages}}包中扫描出{{totalVulnerability}}个有缺陷的包。"
"TOOLTIPS_TITLE": "在此镜像总共包含{{totalPackages}}{{package}},其中{{totalVulnerability}}个{{packageExt}}含有{{vulnerability}}。"
},
"SEVERITY": {
"HIGH": "严重",
@ -495,12 +497,13 @@
"UNKNOWN": "未知",
"NONE": "无"
},
"SINGULAR": "缺陷",
"PLURAL": "缺陷",
"PLACEHOLDER": "过滤缺陷",
"PACKAGE": "个组件有",
"PACKAGES": "个组件有",
"SCAN_NOW": "扫描"
"SINGULAR": "漏洞",
"PLURAL": "漏洞",
"PLACEHOLDER": "过滤漏洞",
"PACKAGE": "组件",
"PACKAGES": "组件",
"SCAN_NOW": "扫描",
"JOB_LOG_VIEWER": "查看扫描日志"
},
"PUSH_IMAGE": {
"TITLE": "推送镜像",

3
tools/migration/changelog.md
View File

@ -46,3 +46,6 @@ Changelog for harbor database schema
- delete foreign key (project_id) references project(project_id)from table `access_log`
- add column `username` varchar (32) to table `access_log`
- alter column `realname` on table `user`: varchar(20)->varchar(255)
- create table `img_scan_job`
- create table `img_scan_overview`
- create table `clair_vuln_timestamp`

10
tools/migration/db_meta.py
View File

@ -169,10 +169,18 @@ class ImageScanJob(Base):
class ImageScanOverview(Base):
__tablename__ = "img_scan_overview"
id = sa.Column(sa.Integer, nullable=False, primary_key=True)
image_digest = sa.Column(sa.String(128), nullable=False)
scan_job_id = sa.Column(sa.Integer, nullable=False)
image_digest = sa.Column(sa.String(128), nullable=False, primary_key=True)
severity = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'"))
components_overview = sa.Column(sa.String(2048))
details_key = sa.Column(sa.String(128))
creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
class ClairVulnTimestamp(Base):
__tablename__ = "clair_vuln_timestamp"
id = sa.Column(sa.Integer, nullable=False, primary_key=True)
namespace = sa.Column(sa.String(128), nullable=False, unique=True)
last_update = sa.Column(mysql.TIMESTAMP)

109
tools/migration/export Executable file
View File

@ -0,0 +1,109 @@
#!/usr/bin/python
import json
import fileinput
from optparse import OptionParser
import os
import MySQLdb
import sys
class Parameters(object):
def __init__(self):
self.dbuser = ''
self.dbpwd = ''
self.exportpath = ''
self.init_from_input()
@staticmethod
def parse_input():
usage = "usage: %prog [options] <dbuser> <dbpwd> <exportpath>"
parser = OptionParser(usage)
parser.add_option("-u", "--dbuser", dest="dbuser", help="db user")
parser.add_option("-p", "--dbpwd", dest="dbpwd", help="db password")
parser.add_option("-o", "--exportpath", dest="exportpath", help="the path of exported json file")
(options, args) = parser.parse_args()
return (options.dbuser, options.dbpwd, options.exportpath)
def init_from_input(self):
(self.dbuser, self.dbpwd, self.exportpath) = Parameters.parse_input()
class Project:
def __init__(self, project_id, name, public):
self.project_id = project_id
self.project_name = name
if public == 0:
self.public = "false"
elif public == 1:
self.public = "true"
else:
self.public = "false"
class HarborUtil:
def __init__(self, dbuser, dbpwd):
self.serverName = 'localhost'
self.user = dbuser
self.password = dbpwd
self.port = '3306'
self.subDB = 'registry'
self.db = None
self.cursor = None
def connect(self):
try:
self.db = MySQLdb.connect(host=self.serverName, user=self.user,
passwd=self.password, db=self.subDB)
self.cursor = self.db.cursor()
except Exception, e:
raise Exception(e)
def close(self):
try:
self.cursor.close()
self.db.close()
except Exception, e:
print str(e)
def get_projects(self):
projects = []
try:
query = "SELECT project_id, name, public from registry.project where deleted=0"
self.cursor.execute(query)
self.cursor.fetchall()
for result in self.cursor:
projects.append(Project(int(result[0]), result[1], result[2]))
return projects
except Exception, e:
raise Exception(e)
def delfile(src):
if not os.path.exists(src):
return
try:
os.remove(src)
except Exception, e:
raise Exception("unable to delete file: %s, error: %s" % (src, str(e)))
def main():
commandline_input = Parameters()
harbor = HarborUtil(commandline_input.dbuser, commandline_input.dbpwd)
try:
harbor.connect()
projects = harbor.get_projects()
if len(projects) == 0:
return
harbor_projects_json = commandline_input.exportpath + '/harbor_projects.json'
delfile(harbor_projects_json)
with open(harbor_projects_json, 'w') as outfile:
json.dump({'projects': [project.__dict__ for project in projects]}, outfile, sort_keys=True, indent=4)
except Exception, e:
print e
sys.exit(1)
finally:
harbor.close()
if __name__ == '__main__':
main()

88
tools/migration/import Executable file
View File

@ -0,0 +1,88 @@
#!/usr/bin/python
import json
from optparse import OptionParser
import os
import urllib2
import sys
import logging
import logging.config
logging.basicConfig(filename="import_project.log", level=logging.INFO)
logger = logging.getLogger()
class Parameters(object):
def __init__(self):
self.admiral_endpoint = ''
self.admiral_token = ''
self.projectsfile = ''
self.init_from_input()
@staticmethod
def parse_input():
usage = "usage: %prog [options] <admiralendpoint> <token> <projectsfile>"
parser = OptionParser(usage)
parser.add_option("-a", "--admiralendpoint", dest="admiral_endpoint", help="admiral endpoint")
parser.add_option("-t", "--token", dest="admiral_token", help="admiral token")
parser.add_option("-f", "--projectsfile", dest="projectsfile", help="the path of exported json file")
(options, args) = parser.parse_args()
return (options.admiral_endpoint, options.admiral_token, options.projectsfile)
def init_from_input(self):
(self.admiral_endpoint, self.admiral_token, self.projectsfile) = Parameters.parse_input()
class Project:
def __init__(self, name, public):
self.project_name = name
self.public = public
class Admiral:
def __init__(self, admiral_url, token):
self.admiral_url = admiral_url + '/projects'
self.token = token
def __import_project(self, project, retry=True):
project_data = json.dumps({ "name": project.project_name, "isPublic": project.public,
"customProperties": {"__enableContentTrust": False, "__preventVulnerableImagesFromRunning":False,
"__preventVulnerableImagesFromRunningSeverity":"high", "__automaticallyScanImagesOnPush":False }})
data_len = len(project_data)
request = urllib2.Request(self.admiral_url, project_data)
request.add_header('x-xenon-auth-token', self.token)
request.add_header('Content-Type', 'application/json')
request.add_header('Content-Length', data_len)
try:
urllib2.urlopen(request)
except Exception, e:
if not retry:
logger.error("failed to import project: %s, admiral_endpoint: %s, error: %s " % (project.project_name, self.admiral_url, str(e)))
return
self.__import_project(project, False)
def import_project(self, projects):
for project in projects:
self.__import_project(project)
def main():
commandline_input = Parameters()
admiral = Admiral(commandline_input.admiral_endpoint, commandline_input.admiral_token)
try:
if not os.path.exists(commandline_input.projectsfile):
raise Exception('Error: %s does not exist' % commandline_input.projectsfile)
with open(commandline_input.projectsfile, 'r') as project_data_file:
project_data = json.load(project_data_file)
projects_import_list = []
for item in project_data['projects']:
projects_import_list.append(Project(item['project_name'], item['public']))
admiral.import_project(projects_import_list)
except Exception, e:
logger.error("failed to import project, admiral_endpoint: %s, error: %s " % (commandline_input.admiral_endpoint, str(e)))
sys.exit(1)
if __name__ == '__main__':
main()

3
tools/migration/migration_harbor/versions/1_2_0.py
View File

@ -59,9 +59,10 @@ def upgrade():
op.drop_column("access_log", "user_id")
op.drop_column("repository", "owner_id")
#create tables: img_scan_job, img_scan_overview
#create tables: img_scan_job, img_scan_overview, clair_vuln_timestamp
ImageScanJob.__table__.create(bind)
ImageScanOverview.__table__.create(bind)
ClairVulnTimestamp.__table__.create(bind)
def downgrade():
"""

8
tools/migration/run.sh
View File

@ -97,6 +97,14 @@ backup)
mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql
echo "Backup performed."
;;
export)
echo "Performing export..."
./export --dbuser ${DB_USR} --dbpwd ${DB_PWD} --exportpath ${EXPORTPATH}
rc="$?"
echo "Export performed."
echo $rc
exit $rc
;;
restore)
echo "Performing restore..."
mysql $DBCNF < ./backup/registry.sql