diff --git a/.travis.yml b/.travis.yml index 7ba2076f8..3888aaf00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,23 @@ sudo: true language: go go: -- 1.12.5 +- 1.12.12 go_import_path: github.com/goharbor/harbor services: - docker dist: trusty matrix: include: - - go: 1.12.5 + - go: 1.12.12 env: - UTTEST=true - - go: 1.12.5 + - go: 1.12.12 env: - APITEST_DB=true - - go: 1.12.5 + - go: 1.12.12 env: - APITEST_LDAP=true - - go: 1.12.5 + - go: 1.12.12 env: - OFFLINE=true - language: node_js diff --git a/ADOPTERS.md b/ADOPTERS.md index f86250459..b0926f2c7 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -80,3 +80,6 @@ feature within Harbor before deploying images into production. **Allegis:** Harbor is used at Allegis as a secure private registry to store and scan customized container images for different business applications, like ELK stack, as part of their CI/CD pipeline. + +# Adding a logo +If you would like to add your logo to the `Users and Partners of Harbor` section of the website, add a PNG version of your logo to the docs/img directory in this repo and submit a pull request with your change. Name the image file something that reflects your company (e.g., if your company is called Acme, name the image acme.png). We will follow up and make the change in the goharbor.io website as well. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 32073195a..71b75b119 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,6 +127,7 @@ Harbor backend is written in [Go](http://golang.org/). If you don't have a Harbo | 1.6 | 1.9.2 | | 1.7 | 1.9.2 | | 1.8 | 1.11.2 | +| 1.9 | 1.12.12 | Ensure your GOPATH and PATH have been configured in accordance with the Go environment instructions. diff --git a/Makefile b/Makefile index 4e8b9bcac..2ee4469b2 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ # compile_golangimage: # compile from golang image # for example: make compile_golangimage -e GOBUILDIMAGE= \ -# golang:1.11.2 +# golang:1.12.12 # compile_core, compile_jobservice: compile specific binary # # build: build Harbor docker images from photon baseimage @@ -111,6 +111,9 @@ CLAIRADAPTERVERSION=c7db8b15 # version of chartmuseum CHARTMUSEUMVERSION=v0.9.0 +# version of registry for pulling the source code +REGISTRY_SRC_TAG=v2.7.1 + define VERSIONS_FOR_PREPARE VERSION_TAG: $(VERSIONTAG) REGISTRY_VERSION: $(REGISTRYVERSION) @@ -138,7 +141,7 @@ GOINSTALL=$(GOCMD) install GOTEST=$(GOCMD) test GODEP=$(GOTEST) -i GOFMT=gofmt -w -GOBUILDIMAGE=golang:1.12.5 +GOBUILDIMAGE=golang:1.12.12 GOBUILDPATH=/harbor GOIMAGEBUILDCMD=/usr/local/go/bin/go GOIMAGEBUILD=$(GOIMAGEBUILDCMD) build -mod vendor diff --git a/OWNERS.md b/OWNERS.md index 646ef821c..d6f61bff1 100644 --- a/OWNERS.md +++ b/OWNERS.md @@ -3,28 +3,6 @@ guidelines. [GOVERNANCE.md](https://github.com/goharbor/community/blob/master/GOVERNANCE.md) describes governance guidelines and maintainer responsibilities. -Maintainer list here is a copy of [MAINTAINERS.md](https://github.com/goharbor/community/blob/master/MAINTAINERS.md). +[MAINTAINERS.md](https://github.com/goharbor/community/blob/master/MAINTAINERS.md) contains the project maintainers. -## Core Maintainers - -| Core Maintainer | GitHub ID | Affiliation | -| --------------- | --------- | ----------- | -| Daniel Jiang | [reasonerjt](https://github.com/reasonerjt) | [VMware](https://www.github.com/vmware/) | -| Steven Ren | [renmaosheng](https://github.com/renmaosheng) | [VMware](https://www.github.com/vmware/) | -| Steven Zou | [steven-zou](https://github.com/steven-zou) | [VMware](https://www.github.com/vmware/) | -| Henry Zhang| [hainingzhang](https://github.com/hainingzhang)| [VMware](https://www.github.com/vmware/) | -| Michael Michael |[michmike](https://github.com/michmike)| [VMware](https://www.github.com/vmware/) | - -## Maintainers - -| Maintainer | GitHub ID | Affiliation | -| ---------- | --------- | ----------- | -| Daojun Zhang | [stonezdj](https://github.com/stonezdj) | [VMware](https://www.github.com/vmware/) | -| Wenkai Yin | [ywk253100](https://github.com/ywk253100) | [VMware](https://www.github.com/vmware/) | -| Yan Wang | [wy65701436](https://github.com/wy65701436) | [VMware](https://www.github.com/vmware/) | -| Qian Deng | [ninjadq](https://github.com/ninjadq) | [VMware](https://www.github.com/vmware/) | -| Mia Zhou | [zhoumeina](https://github.com/zhoumeina) | [VMware](https://www.github.com/vmware/) | -| Nathan Lowe | [nlowe](https://github.com/nlowe) | [Hyland Software](https://github.com/HylandSoftware) | -| De Chen | [cd1989](https://github.com/cd1989) | [Caicloud](https://github.com/caicloud) | -| Mingming Pei | [mmpei](https://github.com/mmpei) | [Netease](https://github.com/netease) | -| Fanjian Kong | [kofj](https://github.com/kofj) | [Qihoo360](https://github.com/Qihoo360) | +[GUIDING_PRINCIPLES.md](https://github.com/goharbor/community/blob/master/GUIDING_PRINCIPLES.md) contains the project vision, values and principles and how we apply them in making decisions. diff --git a/README.md b/README.md index 7e0264fa4..040b0de15 100644 --- a/README.md +++ b/README.md @@ -11,17 +11,16 @@ |![notification](docs/img/bell-outline-badged.svg)Community Meeting| |------------------| -|The Harbor Project holds bi-weekly community calls, to join them and watch previous meeting notes and recordings, please see [meeting schedule](https://github.com/goharbor/community/blob/master/MEETING_SCHEDULE.md).| +|The Harbor Project holds bi-weekly community calls in two different timezones. To join the community calls or to watch previous meeting notes and recordings, please visit the [meeting schedule](https://github.com/goharbor/community/blob/master/MEETING_SCHEDULE.md).| -Welcome to join below Harbor community events and meet with project maintainers and users: +We welcome you to join the below Harbor community events and meet with project maintainers and users: -**May 20-24, 2019**, [KubeCon EU, Barcelona](https://events.linuxfoundation.org/events/kubecon-cloudnativecon-europe-2019/): Harbor Community Reception, Intro and Deep-dive sessions. +**November 18-21, 2019**, [KubeCon US, San Diego](https://events19.linuxfoundation.org/events/kubecon-cloudnativecon-north-america-2019): Harbor Lunch & Learn led by Joe Beda, Intro and Deep-dive sessions. -**June 24-26, 2019**, [KubeCon Shanghai](https://www.lfasiallc.com/events/kubecon-cloudnativecon-china-2019/): Harbor community meetup, Harbor session.

**Note**: The `master` branch may be in an *unstable or even broken state* during development. -Please use [releases](https://github.com/vmware/harbor/releases) instead of the `master` branch in order to get stable binaries. +Please use [releases](https://github.com/vmware/harbor/releases) instead of the `master` branch in order to get a stable set of binaries. Harbor diff --git a/ROADMAP.md b/ROADMAP.md index 18738072a..00cbc8c5b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,41 +1,12 @@ -## Harbor Roadmap - -### About this document - -This document provides description of items that are gathered from the community and planned in Harbor's roadmap. This should serve as a reference point for Harbor users and contributors to understand where the project is heading, and help determine if a contribution could be conflicting with a longer term plan. - -### How to help? - -Discussion on the roadmap can take place in threads under [Issues](https://github.com/vmware/harbor/issues). Please open and comment on an issue if you want to provide suggestions and feedback to an item in the roadmap. Please review the roadmap to avoid potential duplicated effort. - -### How to add an item to the roadmap? -Please open an issue to track any initiative on the roadmap of Harbor. We will work with and rely on our community to focus our efforts to improve Harbor. - - ---- - -### 1. Notary -The notary feature allows publishers to sign their images offline and to push the signed content to a notary server. This ensures the authenticity of images. - -### 2. Vulnerability Scanning -The capability to scan images for vulnerability. - -### 3. Image replication enhancement -To provide more sophisticated rule for image replication. -- Image filtering by tags -- Replication can be scheduled at a certain time using a rule like: one time only, daily, weekly, etc. -- Image deletion can have the option not to be replicated to a remote instance. -- Global replication rule: Instead of setting the rule of individual project, system admin can set a global rule for all projects. -- Project admin can set replication policy of the project. - -### 4. Authentication (OAuth2) -In addition to LDAP/AD and local users, OAuth 2.0 can be used to authenticate a user. - -### 5. High Availability -Support multi-node deployment of Harbor for high availability, scalability and load-balancing purposes. - -### 6. Statistics and description for repositories -User can add a description to a repository. The access count of a repo can be aggregated and displayed. - -### 7. Migration tool to move from an existing registry to Harbor -A tool to migrate images from a vanilla registry server to Harbor, without the need to export/import a large amount of data. +## Harbor Roadmap + +### About this document + +This document provides a link to the [Harbor Project board](https://github.com/orgs/goharbor/projects/1) that serves as the up to date description of items that are in the Harbor release pipeline. The board has separate swim lanes for each release. Most items are gathered from the community or include a feedback loop with the community. This should serve as a reference point for Harbor users and contributors to understand where the project is heading, and help determine if a contribution could be conflicting with a longer term plan. + +### How to help? + +Discussion on the roadmap can take place in threads under [Issues](https://github.com/goharbor/harbor/issues) or in [community meetings](https://github.com/goharbor/community/blob/master/MEETING_SCHEDULE.md). Please open and comment on an issue if you want to provide suggestions and feedback to an item in the roadmap. Please review the roadmap to avoid potential duplicated effort. + +### How to add an item to the roadmap? +Please open an issue to track any initiative on the roadmap of Harbor (Usually driven by new feature requests). We will work with and rely on our community to focus our efforts to improve Harbor. diff --git a/docs/compile_guide.md b/docs/compile_guide.md index 1d28daea5..ab3832fe3 100644 --- a/docs/compile_guide.md +++ b/docs/compile_guide.md @@ -43,25 +43,25 @@ You can compile the code by one of the three approaches: - Get official Golang image from docker hub: ```sh - $ docker pull golang:1.12.5 + $ docker pull golang:1.12.12 ``` - Build, install and bring up Harbor without Notary: ```sh - $ make install GOBUILDIMAGE=golang:1.12.5 COMPILETAG=compile_golangimage + $ make install GOBUILDIMAGE=golang:1.12.12 COMPILETAG=compile_golangimage ``` - Build, install and bring up Harbor with Notary: ```sh - $ make install GOBUILDIMAGE=golang:1.12.5 COMPILETAG=compile_golangimage NOTARYFLAG=true + $ make install GOBUILDIMAGE=golang:1.12.12 COMPILETAG=compile_golangimage NOTARYFLAG=true ``` - Build, install and bring up Harbor with Clair: ```sh - $ make install GOBUILDIMAGE=golang:1.12.5 COMPILETAG=compile_golangimage CLAIRFLAG=true + $ make install GOBUILDIMAGE=golang:1.12.12 COMPILETAG=compile_golangimage CLAIRFLAG=true ``` #### II. Compile code with your own Golang environment, then build Harbor diff --git a/docs/configure_https.md b/docs/configure_https.md index 768636ced..13cd459e8 100644 --- a/docs/configure_https.md +++ b/docs/configure_https.md @@ -13,7 +13,7 @@ You can use certificates that are signed by a trusted third-party CA, or you ca ``` ``` openssl req -x509 -new -nodes -sha512 -days 3650 \ - -subj "/C=TW/ST=Taipei/L=Taipei/O=example/OU=Personal/CN=yourdomain.com" \ + -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=yourdomain.com" \ -key ca.key \ -out ca.crt ``` @@ -36,9 +36,9 @@ If you use FQDN like **yourdomain.com** to connect your registry host, then you ``` openssl req -sha512 -new \ - -subj "/C=TW/ST=Taipei/L=Taipei/O=example/OU=Personal/CN=yourdomain.com" \ + -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=yourdomain.com" \ -key yourdomain.com.key \ - -out yourdomain.com.csr + -out yourdomain.com.csr ``` **3) Generate the certificate of your registry host:** @@ -52,7 +52,7 @@ cat > v3.ext <<-EOF authorityKeyIdentifier=keyid,issuer basicConstraints=CA:FALSE keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment -extendedKeyUsage = serverAuth +extendedKeyUsage = serverAuth subjectAltName = @alt_names [alt_names] @@ -75,17 +75,17 @@ EOF **1) Configure Server Certificate and Key for Harbor** -After obtaining the **yourdomain.com.crt** and **yourdomain.com.key** files, +After obtaining the **yourdomain.com.crt** and **yourdomain.com.key** files, you can put them into directory such as ```/root/cert/```: ``` cp yourdomain.com.crt /data/cert/ - cp yourdomain.com.key /data/cert/ + cp yourdomain.com.key /data/cert/ ``` **2) Configure Server Certificate, Key and CA for Docker** -The Docker daemon interprets ```.crt``` files as CA certificates and ```.cert``` files as client certificates. +The Docker daemon interprets ```.crt``` files as CA certificates and ```.cert``` files as client certificates. Convert server ```yourdomain.com.crt``` to ```yourdomain.com.cert```: @@ -105,7 +105,7 @@ The following illustrates a configuration with custom certificates: ``` /etc/docker/certs.d/ - └── yourdomain.com:port + └── yourdomain.com:port ├── yourdomain.com.cert <-- Server certificate signed by CA ├── yourdomain.com.key <-- Server key signed by CA └── ca.crt <-- Certificate authority that signed the registry certificate @@ -153,11 +153,11 @@ Finally, restart Harbor: ``` After setting up HTTPS for Harbor, you can verify it by the following steps: -* Open a browser and enter the address: https://yourdomain.com. It should display the user interface of Harbor. +* Open a browser and enter the address: https://yourdomain.com. It should display the user interface of Harbor. * Notice that some browser may still shows the warning regarding Certificate Authority (CA) unknown for security reason even though we signed certificates by self-signed CA and deploy the CA to the place mentioned above. It is because self-signed CA essentially is not a trusted third-party CA. You can import the CA to the browser on your own to solve the warning. -* On a machine with Docker daemon, make sure the option "-insecure-registry" for https://yourdomain.com is not present. +* On a machine with Docker daemon, make sure the option "-insecure-registry" for https://yourdomain.com is not present. * If you mapped nginx port 443 to another port, then you should instead create the directory ```/etc/docker/certs.d/yourdomain.com:port``` (or your registry host IP:port). Then run any docker command to verify the setup, e.g. @@ -173,21 +173,21 @@ If you've mapped nginx 443 port to another, you need to add the port to login, l ## Troubleshooting -1. You may get an intermediate certificate from a certificate issuer. In this case, you should merge the intermediate certificate with your own certificate to create a certificate bundle. You can achieve this by the below command: +1. You may get an intermediate certificate from a certificate issuer. In this case, you should merge the intermediate certificate with your own certificate to create a certificate bundle. You can achieve this by the below command: ``` - cat intermediate-certificate.pem >> yourdomain.com.crt + cat intermediate-certificate.pem >> yourdomain.com.crt ``` -2. On some systems where docker daemon runs, you may need to trust the certificate at OS level. - On Ubuntu, this can be done by below commands: - +2. On some systems where docker daemon runs, you may need to trust the certificate at OS level. + On Ubuntu, this can be done by below commands: + ```sh cp yourdomain.com.crt /usr/local/share/ca-certificates/yourdomain.com.crt update-ca-certificates - ``` - - On Red Hat (CentOS etc), the commands are: - + ``` + + On Red Hat (CentOS etc), the commands are: + ```sh cp yourdomain.com.crt /etc/pki/ca-trust/source/anchors/yourdomain.com.crt update-ca-trust diff --git a/docs/registry_landscape.md b/docs/registry_landscape.md index db519f732..b6bdb9f9d 100644 --- a/docs/registry_landscape.md +++ b/docs/registry_landscape.md @@ -1,18 +1,32 @@ # Registry Landscape -The cloud native ecosystem is moving rapidly–registries and their featuresets are no exception. We've made our best effort to survey the container registry landscape and compare to our core featureset. +The cloud native ecosystem is moving rapidly–registries and their feature sets are no exception. We've made our best effort to survey the container registry landscape and compare to our core feature set. If you find something outdated or outright erroneous, please submit a PR and we'll fix it right away. -| Feature | Harbor | Docker Trusted Registry | Quay | Cloud Providers (GCP, AWS, Azure) | Docker Distribution | Artifactory | -| -------------: | :----: | :---------------------: | :--: | :-------------------------------: | :-----------------: | :---------: | -| Local Auth | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | -| LDAP-based Auth | ✓ | ✓ | ✓ | partial | ✗ | ✓ | -| Content Trust and Validation | ✓ | ✓ | ✗ | ✗ | partial | partial | -| Vulnerability Scanning & Monitoring | ✓ | ✓ | ✓ | ✗ | ✗ | ✓ | -| Replication | ✓ | ✓ | ✓ | n/a | ✗ | ✓ | -| Multi-Tenancy (projects, teams, etc.) | ✓ | ✓ | ✓ | partial | ✗ | ✓ | -| Role-Based Access Control | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | -| Custom TLS Certificates | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | -| Ability to Determine Version of Binaries in Containers | ✓ | ✓ | ✓ | ✗ | ✗ | ? | -| Upstream Registry Proxy Cache | ✗ | ✓ | ✗ | ✗ | ✓ | ✓ | -| Audit Logs | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | +Table updated on 10/21/2019 against Harbor 1.9. + +| Feature | Harbor | Docker Trusted Registry | Quay | Cloud Providers (GCP, AWS, Azure) | Docker Distribution | Artifactory | GitLab | +| -------------: | :----: | :---------------------: | :-----: | :-------------------------------: | :-----------------: | :---------: | :------: | +| Ability to Determine Version of Binaries in Containers | ✓ | ✓ | ✓ | ✗ | ✗ | ? | ? | +| Artifact Repository (rpms, git, jar, etc) | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | partial | +| Audit Logs | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | +| Content Trust and Validation | ✓ | ✓ | ✗ | ✗ | partial | partial | ✗ | +| Custom TLS Certificates | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | ✓ | +| Helm Chart Repository Manager | ✓ | ✗ | partial | ✗ | ✗ | ✓ | ✗ | +| LDAP-based Auth | ✓ | ✓ | ✓ | partial | ✗ | ✓ | ✓ | +| Local Auth | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | +| Multi-Tenancy (projects, teams, namespaces, etc) | ✓ | ✓ | ✓ | partial | ✗ | ✓ | ✓ | +| Open Source | ✓ | partial | ✗ | ✗ | ✓ | partial | partial | +| Project Quotas (by image count & storage consumption) | ✓ | ✗ | ✗ | partial | ✗ | ✗ | ✗ | +| Replication between instances | ✓ | ✓ | ✓ | n/a | ✗ | ✓ | ✗ | +| Replication between non-instances | ✓ | ✗ | ✓ | n/a | ✗ | ✗ | ✗ | +| Robot Accounts for Helm Charts | ✓ | ✗ | ✗ | ? | ✗ | ✗ | ✗ | +| Robot Accounts for Images | ✓ | ? | ✓ | ? | ✗ | ? | ? | +| Role-Based Access Control | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | ✗ | +| Single Sign On (OIDC) | ✓ | ✓ | ✓ | ✓ | ✗ | partial | ✗ | +| Tag Retention Policy | ✓ | ✗ | partial | ✗ | ✗ | ✗ | ✗ | +| Upstream Registry Proxy Cache | ✗ | ✓ | ✗ | ✗ | ✓ | ✓ | ✗ | +| Vulnerability Scanning & Monitoring | ✓ | ✓ | ✓ | ✗ | ✗ | ✓ | partial | +| Vulnerability Scanning Plugin Framework | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Vulnerability Whitelisting | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Webhooks | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6b85f001b..3bd2de958 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1089,6 +1089,8 @@ paths: description: Forbidden. '404': description: Repository not found. + '412': + description: Precondition Failed. put: summary: Update description of the repository. description: | diff --git a/docs/use_make.md b/docs/use_make.md index 2d56f6eaa..f6deb28e8 100644 --- a/docs/use_make.md +++ b/docs/use_make.md @@ -36,10 +36,10 @@ version | set harbor version #### EXAMPLE: #### Build and run harbor from source code. -make install GOBUILDIMAGE=golang:1.12.5 COMPILETAG=compile_golangimage NOTARYFLAG=true +make install GOBUILDIMAGE=golang:1.12.12 COMPILETAG=compile_golangimage NOTARYFLAG=true ### Package offline installer -make package_offline GOBUILDIMAGE=golang:1.12.5 COMPILETAG=compile_golangimage NOTARYFLAG=true +make package_offline GOBUILDIMAGE=golang:1.12.12 COMPILETAG=compile_golangimage NOTARYFLAG=true ### Start harbor with notary make -e NOTARYFLAG=true start diff --git a/make/photon/Makefile b/make/photon/Makefile index a1290cb1e..4deb9b647 100644 --- a/make/photon/Makefile +++ b/make/photon/Makefile @@ -97,7 +97,7 @@ DOCKERIMAGENAME_MIGRATOR=goharbor/harbor-migrator # for chart server (chartmuseum) DOCKERFILEPATH_CHART_SERVER=$(DOCKERFILEPATH)/chartserver DOCKERFILENAME_CHART_SERVER=Dockerfile -CHART_SERVER_CODE_BASE=github.com/helm/chartmuseum +CHART_SERVER_CODE_BASE=https://github.com/helm/chartmuseum.git CHART_SERVER_MAIN_PATH=cmd/chartmuseum CHART_SERVER_BIN_NAME=chartm @@ -195,7 +195,7 @@ _build_registry: rm -rf $(DOCKERFILEPATH_REG)/binary && mkdir -p $(DOCKERFILEPATH_REG)/binary && \ $(call _get_binary, https://storage.googleapis.com/harbor-builds/bin/registry/release-$(REGISTRYVERSION)/registry, $(DOCKERFILEPATH_REG)/binary/registry); \ else \ - cd $(DOCKERFILEPATH_REG) && $(DOCKERFILEPATH_REG)/builder $(REGISTRYVERSION) && cd - ; \ + cd $(DOCKERFILEPATH_REG) && $(DOCKERFILEPATH_REG)/builder $(REGISTRY_SRC_TAG) && cd - ; \ fi @echo "building registry container for photon..." @chmod 655 $(DOCKERFILEPATH_REG)/binary/registry && $(DOCKERBUILD) -f $(DOCKERFILEPATH_REG)/$(DOCKERFILENAME_REG) -t $(DOCKERIMAGENAME_REG):$(REGISTRYVERSION)-$(VERSIONTAG) . diff --git a/make/photon/chartserver/builder b/make/photon/chartserver/builder index a1d6c3c3f..9a5c0239a 100755 --- a/make/photon/chartserver/builder +++ b/make/photon/chartserver/builder @@ -30,4 +30,4 @@ cp compile.sh binary/ docker run -it --rm -v $cur/binary:/go/bin --name golang_code_builder $GOLANG_IMAGE /bin/bash /go/bin/compile.sh $GIT_PATH $CODE_VERSION $MAIN_GO_PATH $BIN_NAME #Clear -docker rm -f golang_code_builder +#docker rm -f golang_code_builder diff --git a/make/photon/clair-adapter/Dockerfile.binary b/make/photon/clair-adapter/Dockerfile.binary index 87707a3b5..a1a808711 100644 --- a/make/photon/clair-adapter/Dockerfile.binary +++ b/make/photon/clair-adapter/Dockerfile.binary @@ -1,4 +1,4 @@ -FROM golang:1.12.5 +FROM golang:1.12.12 ADD . /go/src/github.com/goharbor/harbor-scanner-clair/ WORKDIR /go/src/github.com/goharbor/harbor-scanner-clair/ diff --git a/make/photon/clair-adapter/builder b/make/photon/clair-adapter/builder index 857fef0e3..82473d855 100755 --- a/make/photon/clair-adapter/builder +++ b/make/photon/clair-adapter/builder @@ -23,7 +23,7 @@ TEMP=`mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX` git clone https://github.com/danielpacak/harbor-scanner-clair.git $TEMP cd $TEMP; git checkout $VERSION; cd - -echo 'build the clair adapter binary bases on the golang:1.12.5...' +echo 'build the clair adapter binary bases on the golang:1.12.12' cp Dockerfile.binary $TEMP docker build -f $TEMP/Dockerfile.binary -t clair-adapter-golang $TEMP diff --git a/make/photon/clair/Dockerfile.binary b/make/photon/clair/Dockerfile.binary index d478e260f..640962f47 100644 --- a/make/photon/clair/Dockerfile.binary +++ b/make/photon/clair/Dockerfile.binary @@ -1,4 +1,4 @@ -FROM golang:1.11.2 +FROM golang:1.12.12 ADD . /go/src/github.com/coreos/clair/ WORKDIR /go/src/github.com/coreos/clair/ diff --git a/make/photon/clair/builder b/make/photon/clair/builder index 16ae1118c..818f3c0e6 100755 --- a/make/photon/clair/builder +++ b/make/photon/clair/builder @@ -23,7 +23,7 @@ TEMP=`mktemp -d /$TMPDIR/clair.XXXXXX` git clone https://github.com/coreos/clair.git $TEMP cd $TEMP; git checkout $VERSION; cd - -echo 'build the clair binary bases on the golang:1.11.2...' +echo 'build the clair binary bases on the golang:1.12.12' cp Dockerfile.binary $TEMP docker build -f $TEMP/Dockerfile.binary -t clair-golang $TEMP diff --git a/make/photon/notary/binary.Dockerfile b/make/photon/notary/binary.Dockerfile index 911562e53..7a4188fc1 100644 --- a/make/photon/notary/binary.Dockerfile +++ b/make/photon/notary/binary.Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.11.2 +FROM golang:1.12.12 ARG NOTARY_VERSION ARG MIGRATE_VERSION diff --git a/make/photon/notary/builder b/make/photon/notary/builder index 1d11aa027..93404f9c4 100755 --- a/make/photon/notary/builder +++ b/make/photon/notary/builder @@ -24,6 +24,8 @@ docker cp $ID:/go/bin/notary-signer binary/ docker cp $ID:/go/bin/migrate binary/ docker cp $ID:/migrations binary/ +sed -i 's/waiting for $DB_URL/waiting for database/g' binary/migrations/migrate.sh + docker rm -f $ID docker rmi -f notary-binary diff --git a/make/photon/registry/Dockerfile.binary b/make/photon/registry/Dockerfile.binary index de9aa4bdf..5f6311cd8 100644 --- a/make/photon/registry/Dockerfile.binary +++ b/make/photon/registry/Dockerfile.binary @@ -1,4 +1,4 @@ -FROM golang:1.11 +FROM golang:1.12.12 ENV DISTRIBUTION_DIR /go/src/github.com/docker/distribution ENV BUILDTAGS include_oss include_gcs diff --git a/make/photon/registry/builder b/make/photon/registry/builder index 48ec5b3dd..eaacc1ba2 100755 --- a/make/photon/registry/builder +++ b/make/photon/registry/builder @@ -29,7 +29,7 @@ wget https://github.com/docker/distribution/pull/2879.patch git apply 2879.patch cd $cur -echo 'build the registry binary bases on the golang:1.11...' +echo 'build the registry binary ...' cp Dockerfile.binary $TEMP docker build -f $TEMP/Dockerfile.binary -t registry-golang $TEMP diff --git a/src/common/config/metadata/type.go b/src/common/config/metadata/type.go index 745f30868..1feb55b2e 100644 --- a/src/common/config/metadata/type.go +++ b/src/common/config/metadata/type.go @@ -18,6 +18,7 @@ package metadata import ( "encoding/json" "fmt" + "math" "strconv" "strings" @@ -139,12 +140,12 @@ type Int64Type struct { } func (t *Int64Type) validate(str string) error { - _, err := strconv.ParseInt(str, 10, 64) + _, err := parseInt64(str) return err } func (t *Int64Type) get(str string) (interface{}, error) { - return strconv.ParseInt(str, 10, 64) + return parseInt64(str) } // BoolType ... @@ -194,7 +195,7 @@ type QuotaType struct { } func (t *QuotaType) validate(str string) error { - val, err := strconv.ParseInt(str, 10, 64) + val, err := parseInt64(str) if err != nil { return err } @@ -205,3 +206,18 @@ func (t *QuotaType) validate(str string) error { return nil } + +// parseInt64 returns int64 from string which support scientific notation +func parseInt64(str string) (int64, error) { + val, err := strconv.ParseInt(str, 10, 64) + if err == nil { + return val, nil + } + + fval, err := strconv.ParseFloat(str, 64) + if err == nil && fval == math.Trunc(fval) { + return int64(fval), nil + } + + return 0, fmt.Errorf("invalid int64 string: %s", str) +} diff --git a/src/common/config/metadata/type_test.go b/src/common/config/metadata/type_test.go index 31a0730de..ed3ddc526 100644 --- a/src/common/config/metadata/type_test.go +++ b/src/common/config/metadata/type_test.go @@ -15,8 +15,9 @@ package metadata import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestIntType_validate(t *testing.T) { @@ -96,3 +97,33 @@ func TestMapType_get(t *testing.T) { result, _ := test.get(`{"sample":"abc", "another":"welcome"}`) assert.Equal(t, map[string]interface{}{"sample": "abc", "another": "welcome"}, result) } + +func Test_parseInt64(t *testing.T) { + type args struct { + str string + } + tests := []struct { + name string + args args + want int64 + wantErr bool + }{ + {"1", args{"1"}, int64(1), false}, + {"1.0", args{"1.0"}, int64(1), false}, + {"1.1", args{"1.1"}, int64(0), true}, + {"1E2", args{"1E2"}, int64(100), false}, + {"1.073741824e+11", args{"1.073741824e+11"}, int64(107374182400), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseInt64(tt.args.str) + if (err != nil) != tt.wantErr { + t.Errorf("parseInt64() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("parseInt64() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/common/models/repo.go b/src/common/models/repo.go index 6562e9531..aa2bc24ec 100644 --- a/src/common/models/repo.go +++ b/src/common/models/repo.go @@ -73,6 +73,7 @@ type TagDetail struct { Author string `json:"author"` Created time.Time `json:"created"` Config *TagCfg `json:"config"` + Immutable bool `json:"immutable"` } // TagCfg ... diff --git a/src/core/api/chart_repository.go b/src/core/api/chart_repository.go index bc7ad8487..d857629a9 100755 --- a/src/core/api/chart_repository.go +++ b/src/core/api/chart_repository.go @@ -10,7 +10,10 @@ import ( "mime/multipart" "net/http" "net/url" + "path" + "strconv" "strings" + "time" "github.com/goharbor/harbor/src/chartserver" "github.com/goharbor/harbor/src/common" @@ -18,14 +21,10 @@ import ( hlog "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/core/config" "github.com/goharbor/harbor/src/core/label" - "github.com/goharbor/harbor/src/core/middlewares" n_event "github.com/goharbor/harbor/src/core/notifier/event" rep_event "github.com/goharbor/harbor/src/replication/event" "github.com/goharbor/harbor/src/replication/model" - "path" - "strconv" - "time" ) const ( @@ -489,6 +488,12 @@ func (cra *ChartRepositoryAPI) addEventContext(files []formFile, request *http.R extInfo["operator"] = cra.SecurityCtx.GetUsername() extInfo["projectName"] = cra.namespace extInfo["chartName"] = chartDetails.Metadata.Name + + public, err := cra.ProjectMgr.IsPublic(cra.namespace) + if err != nil { + hlog.Errorf("failed to check the public of project %s: %v", cra.namespace, err) + public = false + } e := &rep_event.Event{ Type: rep_event.EventTypeChartUpload, Resource: &model.Resource{ @@ -496,6 +501,9 @@ func (cra *ChartRepositoryAPI) addEventContext(files []formFile, request *http.R Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ Name: fmt.Sprintf("%s/%s", cra.namespace, chartDetails.Metadata.Name), + Metadata: map[string]interface{}{ + "public": strconv.FormatBool(public), + }, }, Vtags: []string{chartDetails.Metadata.Version}, }, diff --git a/src/core/api/repository.go b/src/core/api/repository.go index 6d9a1f7f5..ac3284bea 100755 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -28,7 +28,7 @@ import ( "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/pkg/scan/api/scan" - v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" + "github.com/goharbor/harbor/src/pkg/scan/rest/v1" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" @@ -45,6 +45,8 @@ import ( "github.com/goharbor/harbor/src/core/config" notifierEvt "github.com/goharbor/harbor/src/core/notifier/event" coreutils "github.com/goharbor/harbor/src/core/utils" + "github.com/goharbor/harbor/src/pkg/art" + "github.com/goharbor/harbor/src/pkg/immutabletag/match/rule" "github.com/goharbor/harbor/src/replication" "github.com/goharbor/harbor/src/replication/event" "github.com/goharbor/harbor/src/replication/model" @@ -283,11 +285,6 @@ func (ra *RepositoryAPI) Delete() { } for _, t := range tags { - image := fmt.Sprintf("%s:%s", repoName, t) - if err = dao.DeleteLabelsOfResource(common.ResourceTypeImage, image); err != nil { - ra.SendInternalServerError(fmt.Errorf("failed to delete labels of image %s: %v", image, err)) - return - } if err = rc.DeleteTag(t); err != nil { if regErr, ok := err.(*commonhttp.Error); ok { if regErr.Code == http.StatusNotFound { @@ -298,6 +295,11 @@ func (ra *RepositoryAPI) Delete() { return } log.Infof("delete tag: %s:%s", repoName, t) + image := fmt.Sprintf("%s:%s", repoName, t) + if err = dao.DeleteLabelsOfResource(common.ResourceTypeImage, image); err != nil { + ra.SendInternalServerError(fmt.Errorf("failed to delete labels of image %s: %v", image, err)) + return + } go func(tag string) { e := &event.Event{ @@ -711,6 +713,9 @@ func assembleTag(c chan *models.TagResp, client *registry.Repository, projectID } } + // get immutable status + item.Immutable = isImmutable(projectID, repository, tag) + c <- item } @@ -791,6 +796,21 @@ func populateAuthor(detail *models.TagDetail) { } } +// check whether the tag is immutable +func isImmutable(projectID int64, repo string, tag string) bool { + _, repoName := utils.ParseRepository(repo) + matched, err := rule.NewRuleMatcher(projectID).Match(art.Candidate{ + Repository: repoName, + Tag: tag, + NamespaceID: projectID, + }) + if err != nil { + log.Error(err) + return false + } + return matched +} + // GetManifests returns the manifest of a tag func (ra *RepositoryAPI) GetManifests() { repoName := ra.GetString(":splat") diff --git a/src/core/middlewares/config.go b/src/core/middlewares/config.go index adc2a75c4..fa2b536f5 100644 --- a/src/core/middlewares/config.go +++ b/src/core/middlewares/config.go @@ -35,4 +35,4 @@ var ChartMiddlewares = []string{CHART} var Middlewares = []string{READONLY, URL, MUITIPLEMANIFEST, LISTREPO, CONTENTTRUST, VULNERABLE, SIZEQUOTA, IMMUTABLE, COUNTQUOTA} // MiddlewaresLocal ... -var MiddlewaresLocal = []string{SIZEQUOTA, COUNTQUOTA} +var MiddlewaresLocal = []string{SIZEQUOTA, IMMUTABLE, COUNTQUOTA} diff --git a/src/core/middlewares/immutable/builder.go b/src/core/middlewares/immutable/builder.go new file mode 100644 index 000000000..30707299d --- /dev/null +++ b/src/core/middlewares/immutable/builder.go @@ -0,0 +1,54 @@ +package immutable + +import ( + "fmt" + "github.com/goharbor/harbor/src/core/middlewares/interceptor" + "github.com/goharbor/harbor/src/core/middlewares/interceptor/immutable" + "github.com/goharbor/harbor/src/core/middlewares/util" + "net/http" +) + +var ( + defaultBuilders = []interceptor.Builder{ + &manifestDeletionBuilder{}, + &manifestCreationBuilder{}, + } +) + +type manifestDeletionBuilder struct{} + +func (*manifestDeletionBuilder) Build(req *http.Request) (interceptor.Interceptor, error) { + if match, _, _ := util.MatchDeleteManifest(req); !match { + return nil, nil + } + + info, ok := util.ManifestInfoFromContext(req.Context()) + if !ok { + var err error + info, err = util.ParseManifestInfoFromPath(req) + if err != nil { + return nil, fmt.Errorf("failed to parse manifest, error %v", err) + } + } + + return immutable.NewDeleteMFInteceptor(info), nil +} + +type manifestCreationBuilder struct{} + +func (*manifestCreationBuilder) Build(req *http.Request) (interceptor.Interceptor, error) { + if match, _, _ := util.MatchPushManifest(req); !match { + return nil, nil + } + + info, ok := util.ManifestInfoFromContext(req.Context()) + if !ok { + var err error + info, err = util.ParseManifestInfoFromReq(req) + if err != nil { + return nil, fmt.Errorf("failed to parse manifest, error %v", err) + } + } + + return immutable.NewPushMFInteceptor(info), nil +} diff --git a/src/core/middlewares/immutable/handler.go b/src/core/middlewares/immutable/handler.go index 0566d9df0..e9deb2dbf 100644 --- a/src/core/middlewares/immutable/handler.go +++ b/src/core/middlewares/immutable/handler.go @@ -16,78 +16,74 @@ package immutable import ( "fmt" - "github.com/goharbor/harbor/src/common/dao" - "github.com/goharbor/harbor/src/common/models" - common_util "github.com/goharbor/harbor/src/common/utils" "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/interceptor" "github.com/goharbor/harbor/src/core/middlewares/util" - "github.com/goharbor/harbor/src/pkg/art" - "github.com/goharbor/harbor/src/pkg/immutabletag/match/rule" + middlerware_err "github.com/goharbor/harbor/src/core/middlewares/util/error" "net/http" ) type immutableHandler struct { - next http.Handler + builders []interceptor.Builder + next http.Handler } // New ... -func New(next http.Handler) http.Handler { +func New(next http.Handler, builders ...interceptor.Builder) http.Handler { + if len(builders) == 0 { + builders = defaultBuilders + } + return &immutableHandler{ - next: next, + builders: builders, + next: next, } } // ServeHTTP ... -func (rh immutableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - if match, _, _ := util.MatchPushManifest(req); !match { +func (rh *immutableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + + interceptor, err := rh.getInterceptor(req) + if err != nil { + log.Warningf("Error occurred when to handle request in immutable handler: %v", err) + http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in immutable handler: %v", err)), + http.StatusInternalServerError) + return + } + + if interceptor == nil { rh.next.ServeHTTP(rw, req) return } - info, ok := util.ManifestInfoFromContext(req.Context()) - if !ok { - var err error - info, err = util.ParseManifestInfoFromPath(req) - if err != nil { - log.Error(err) - rh.next.ServeHTTP(rw, req) + + if err := interceptor.HandleRequest(req); err != nil { + log.Warningf("Error occurred when to handle request in immutable handler: %v", err) + if _, ok := err.(middlerware_err.ErrImmutable); ok { + http.Error(rw, util.MarshalError("DENIED", + fmt.Sprintf("The tag is immutable, cannot be overwrite: %v", err)), http.StatusPreconditionFailed) return } + http.Error(rw, util.MarshalError("InternalError", fmt.Sprintf("Error occurred when to handle request in immutable handler: %v", err)), + http.StatusInternalServerError) + return + } + + rh.next.ServeHTTP(rw, req) + + interceptor.HandleResponse(rw, req) +} + +func (rh *immutableHandler) getInterceptor(req *http.Request) (interceptor.Interceptor, error) { + for _, builder := range rh.builders { + interceptor, err := builder.Build(req) + if err != nil { + return nil, err + } + + if interceptor != nil { + return interceptor, nil + } } - _, repoName := common_util.ParseRepository(info.Repository) - matched, err := rule.NewRuleMatcher(info.ProjectID).Match(art.Candidate{ - Repository: repoName, - Tag: info.Tag, - NamespaceID: info.ProjectID, - }) - if err != nil { - log.Error(err) - rh.next.ServeHTTP(rw, req) - return - } - if !matched { - rh.next.ServeHTTP(rw, req) - return - } - - artifactQuery := &models.ArtifactQuery{ - PID: info.ProjectID, - Repo: info.Repository, - Tag: info.Tag, - } - afs, err := dao.ListArtifacts(artifactQuery) - if err != nil { - log.Error(err) - rh.next.ServeHTTP(rw, req) - return - } - if len(afs) == 0 { - rh.next.ServeHTTP(rw, req) - return - } - - // rule matched and non-existent is a immutable tag - http.Error(rw, util.MarshalError("DENIED", - fmt.Sprintf("The tag:%s:%s is immutable, cannot be overwrite.", info.Repository, info.Tag)), http.StatusPreconditionFailed) - return + return nil, nil } diff --git a/src/core/middlewares/interceptor/immutable/deletemf.go b/src/core/middlewares/interceptor/immutable/deletemf.go new file mode 100644 index 000000000..6fb2e659a --- /dev/null +++ b/src/core/middlewares/interceptor/immutable/deletemf.go @@ -0,0 +1,67 @@ +package immutable + +import ( + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + common_util "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/interceptor" + "github.com/goharbor/harbor/src/core/middlewares/util" + middlerware_err "github.com/goharbor/harbor/src/core/middlewares/util/error" + "github.com/goharbor/harbor/src/pkg/art" + "github.com/goharbor/harbor/src/pkg/immutabletag/match/rule" + "net/http" +) + +// NewDeleteMFInteceptor .... +func NewDeleteMFInteceptor(mf *util.ManifestInfo) interceptor.Interceptor { + return &delmfInterceptor{ + mf: mf, + } +} + +type delmfInterceptor struct { + mf *util.ManifestInfo +} + +// HandleRequest ... +func (dmf *delmfInterceptor) HandleRequest(req *http.Request) (err error) { + + artifactQuery := &models.ArtifactQuery{ + Digest: dmf.mf.Digest, + Repo: dmf.mf.Repository, + PID: dmf.mf.ProjectID, + } + var afs []*models.Artifact + afs, err = dao.ListArtifacts(artifactQuery) + if err != nil { + log.Error(err) + return + } + if len(afs) == 0 { + return + } + + for _, af := range afs { + _, repoName := common_util.ParseRepository(dmf.mf.Repository) + var matched bool + matched, err = rule.NewRuleMatcher(dmf.mf.ProjectID).Match(art.Candidate{ + Repository: repoName, + Tag: af.Tag, + NamespaceID: dmf.mf.ProjectID, + }) + if err != nil { + log.Error(err) + return + } + if matched { + return middlerware_err.NewErrImmutable(repoName) + } + } + + return +} + +// HandleRequest ... +func (dmf *delmfInterceptor) HandleResponse(w http.ResponseWriter, r *http.Request) { +} diff --git a/src/core/middlewares/interceptor/immutable/pushmf.go b/src/core/middlewares/interceptor/immutable/pushmf.go new file mode 100644 index 000000000..307579bd5 --- /dev/null +++ b/src/core/middlewares/interceptor/immutable/pushmf.go @@ -0,0 +1,65 @@ +package immutable + +import ( + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/common/models" + common_util "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/common/utils/log" + "github.com/goharbor/harbor/src/core/middlewares/interceptor" + "github.com/goharbor/harbor/src/core/middlewares/util" + middlerware_err "github.com/goharbor/harbor/src/core/middlewares/util/error" + "github.com/goharbor/harbor/src/pkg/art" + "github.com/goharbor/harbor/src/pkg/immutabletag/match/rule" + "net/http" +) + +// NewPushMFInteceptor .... +func NewPushMFInteceptor(mf *util.ManifestInfo) interceptor.Interceptor { + return &pushmfInterceptor{ + mf: mf, + } +} + +type pushmfInterceptor struct { + mf *util.ManifestInfo +} + +// HandleRequest ... +func (pmf *pushmfInterceptor) HandleRequest(req *http.Request) (err error) { + + _, repoName := common_util.ParseRepository(pmf.mf.Repository) + var matched bool + matched, err = rule.NewRuleMatcher(pmf.mf.ProjectID).Match(art.Candidate{ + Repository: repoName, + Tag: pmf.mf.Tag, + NamespaceID: pmf.mf.ProjectID, + }) + if err != nil { + log.Error(err) + return + } + if !matched { + return + } + + artifactQuery := &models.ArtifactQuery{ + PID: pmf.mf.ProjectID, + Repo: pmf.mf.Repository, + Tag: pmf.mf.Tag, + } + var afs []*models.Artifact + afs, err = dao.ListArtifacts(artifactQuery) + if err != nil { + log.Error(err) + return + } + if len(afs) == 0 { + return + } + + return middlerware_err.NewErrImmutable(repoName) +} + +// HandleRequest ... +func (pmf *pushmfInterceptor) HandleResponse(w http.ResponseWriter, r *http.Request) { +} diff --git a/src/core/middlewares/util/error/immutable.go b/src/core/middlewares/util/error/immutable.go new file mode 100644 index 000000000..a7789300d --- /dev/null +++ b/src/core/middlewares/util/error/immutable.go @@ -0,0 +1,20 @@ +package error + +import ( + "fmt" +) + +// ErrImmutable ... +type ErrImmutable struct { + repo string +} + +// Error ... +func (ei ErrImmutable) Error() string { + return fmt.Sprintf("Failed to process request, due to immutable. '%s'", ei.repo) +} + +// NewErrImmutable ... +func NewErrImmutable(msg string) ErrImmutable { + return ErrImmutable{repo: msg} +} diff --git a/src/core/service/notifications/registry/handler.go b/src/core/service/notifications/registry/handler.go index 0dd491c69..7ecb55714 100755 --- a/src/core/service/notifications/registry/handler.go +++ b/src/core/service/notifications/registry/handler.go @@ -17,6 +17,7 @@ package registry import ( "encoding/json" "regexp" + "strconv" "strings" "time" @@ -140,7 +141,6 @@ func (n *NotificationHandler) Post() { log.Errorf("failed to build image push event metadata: %v", err) } - // TODO: handle image delete event and chart event go func() { e := &rep_event.Event{ Type: rep_event.EventTypeImagePush, @@ -149,7 +149,9 @@ func (n *NotificationHandler) Post() { Metadata: &model.ResourceMetadata{ Repository: &model.Repository{ Name: repository, - // TODO filling the metadata + Metadata: map[string]interface{}{ + "public": strconv.FormatBool(pro.IsPublic()), + }, }, Vtags: []string{tag}, }, diff --git a/src/jobservice/job/impl/gc/job.go b/src/jobservice/job/impl/gc/job.go index a946e201a..853e0f36f 100644 --- a/src/jobservice/job/impl/gc/job.go +++ b/src/jobservice/job/impl/gc/job.go @@ -17,6 +17,7 @@ package gc import ( "fmt" "os" + "strconv" "time" "github.com/garyburd/redigo/redis" @@ -29,7 +30,6 @@ import ( "github.com/goharbor/harbor/src/jobservice/logger" "github.com/goharbor/harbor/src/pkg/types" "github.com/goharbor/harbor/src/registryctl/client" - "strconv" ) const ( @@ -38,6 +38,7 @@ const ( dialWriteTimeout = 10 * time.Second blobPrefix = "blobs::*" repoPrefix = "repository::*" + uploadSizePattern = "upload:*:size" ) // GarbageCollector is the struct to run registry's garbage collection @@ -156,15 +157,13 @@ func (gc *GarbageCollector) cleanCache() error { // sample of keys in registry redis: // 1) "blobs::sha256:1a6fd470b9ce10849be79e99529a88371dff60c60aab424c077007f6979b4812" // 2) "repository::library/hello-world::blobs::sha256:4ab4c602aa5eed5528a6620ff18a1dc4faef0e1ab3a5eddeddb410714478c67f" - err = delKeys(con, blobPrefix) - if err != nil { - gc.logger.Errorf("failed to clean registry cache %v, pattern blobs::*", err) - return err - } - err = delKeys(con, repoPrefix) - if err != nil { - gc.logger.Errorf("failed to clean registry cache %v, pattern repository::*", err) - return err + // 3) "upload:fbd2e0a3-262d-40bb-abe4-2f43aa6f9cda:size" + patterns := []string{blobPrefix, repoPrefix, uploadSizePattern} + for _, pattern := range patterns { + if err := delKeys(con, pattern); err != nil { + gc.logger.Errorf("failed to clean registry cache %v, pattern %s", err, pattern) + return err + } } return nil diff --git a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.html b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.html index ed75afa19..55cfdfd5a 100644 --- a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.html +++ b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.html @@ -209,6 +209,10 @@ +
+ + +
diff --git a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts index 55229594d..0a9279a0b 100644 --- a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts +++ b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts @@ -203,6 +203,7 @@ export class CreateEditRuleComponent implements OnInit, OnDestroy { }) }), filters: this.fb.array([]), + enabled: true, deletion: false, override: true }); @@ -228,6 +229,7 @@ export class CreateEditRuleComponent implements OnInit, OnDestroy { } }, deletion: false, + enabled: true, override: true }); this.isPushMode = true; @@ -251,6 +253,7 @@ export class CreateEditRuleComponent implements OnInit, OnDestroy { dest_registry: rule.dest_registry, trigger: rule.trigger, deletion: rule.deletion, + enabled: rule.enabled, override: rule.override }); let filtersArray = this.getFilterArray(rule); diff --git a/src/portal/lib/src/replication/replication.component.html b/src/portal/lib/src/replication/replication.component.html index f02d19304..b140f27d7 100644 --- a/src/portal/lib/src/replication/replication.component.html +++ b/src/portal/lib/src/replication/replication.component.html @@ -24,7 +24,7 @@
{{'REPLICATION.REPLICATION_EXECUTIONS' | translate}}
-
+
-
+
-
-
+
-
@@ -76,8 +81,13 @@ {{scanner.name}} {{scanner.url}} - {{'SCANNER.HEALTHY' | translate}} - {{'SCANNER.UNHEALTHY' | translate}} + + + {{'SCANNER.HEALTHY' | translate}} + + {{'SCANNER.UNHEALTHY' | translate}} + + {{scanner.is_default}} diff --git a/src/portal/src/app/project/scanner/scanner.component.ts b/src/portal/src/app/project/scanner/scanner.component.ts index 19ec8985c..ae43ae53c 100644 --- a/src/portal/src/app/project/scanner/scanner.component.ts +++ b/src/portal/src/app/project/scanner/scanner.component.ts @@ -56,11 +56,24 @@ export class ScannerComponent implements OnInit { .subscribe(response => { if (response && "{}" !== JSON.stringify(response)) { this.scanner = response; + this.getScannerMetadata(); } }, error => { this.errorHandler.error(error); }); } + getScannerMetadata() { + if (this.scanner && this.scanner.uuid) { + this.scanner.loadingMetadata = true; + this.configScannerService.getScannerMetadata(this.scanner.uuid) + .pipe(finalize(() => this.scanner.loadingMetadata = false)) + .subscribe(response => { + this.scanner.metadata = response; + }, error => { + this.scanner.metadata = null; + }); + } + } getScanners() { this.loading = true; this.configScannerService.getScanners() @@ -75,6 +88,22 @@ export class ScannerComponent implements OnInit { this.errorHandler.error(error); }); } + getMetadataForAll() { + if (this.scanners && this.scanners.length > 0) { + this.scanners.forEach((scanner, index) => { + if (scanner.uuid ) { + this.scanners[index].loadingMetadata = true; + this.configScannerService.getScannerMetadata(scanner.uuid) + .pipe(finalize(() => this.scanners[index].loadingMetadata = false)) + .subscribe(response => { + this.scanners[index].metadata = response; + }, error => { + this.scanners[index].metadata = null; + }); + } + }); + } + } close() { this.opened = false; this.selectedScanner = null; @@ -87,6 +116,7 @@ export class ScannerComponent implements OnInit { this.selectedScanner = s; } }); + this.getMetadataForAll(); } get valid(): boolean { return this.selectedScanner diff --git a/src/portal/src/i18n/lang/en-us-lang.json b/src/portal/src/i18n/lang/en-us-lang.json index c819618e7..0b369c003 100644 --- a/src/portal/src/i18n/lang/en-us-lang.json +++ b/src/portal/src/i18n/lang/en-us-lang.json @@ -622,6 +622,7 @@ "PULL_COMMAND": "Pull Command", "PULL_TIME": "Pull Time", "PUSH_TIME": "Push Time", + "IMMUTABLE": "Immutable", "MY_REPOSITORY": "My Repository", "PUBLIC_REPOSITORY": "Public Repository", "DELETION_TITLE_REPO": "Confirm Repository Deletion", @@ -636,7 +637,7 @@ "FILTER_FOR_REPOSITORIES": "Filter Repositories", "TAG": "Tag", "SIZE": "Size", - "VULNERABILITY": "Vulnerability", + "VULNERABILITY": "Vulnerabilities", "BUILD_HISTORY": "Build History", "SIGNED": "Signed", "AUTHOR": "Author", @@ -1291,6 +1292,9 @@ "ENABLED": "Enabled", "ENABLE": "Enable", "DISABLE": "Disable", - "DELETE_SUCCESS": "Successfully deleted" + "DELETE_SUCCESS": "Successfully deleted", + "TOTAL": "Total", + "FIXABLE": "Fixable", + "DURATION": "Duration:" } } diff --git a/src/portal/src/i18n/lang/es-es-lang.json b/src/portal/src/i18n/lang/es-es-lang.json index a333c29e4..500d09b0e 100644 --- a/src/portal/src/i18n/lang/es-es-lang.json +++ b/src/portal/src/i18n/lang/es-es-lang.json @@ -623,6 +623,7 @@ "PULL_COMMAND": "Comando Pull", "PULL_TIME": "Pull Time", "PUSH_TIME": "Push Time", + "IMMUTABLE": "Immutable", "MY_REPOSITORY": "Mi Repositorio", "PUBLIC_REPOSITORY": "Repositorio Público", "DELETION_TITLE_REPO": "Confirmar Eliminación de Repositorio", @@ -637,7 +638,7 @@ "FILTER_FOR_REPOSITORIES": "Filtrar Repositorios", "TAG": "Etiqueta", "SIZE": "Size", - "VULNERABILITY": "Vulnerability", + "VULNERABILITY": "Vulnerabilities", "BUILD_HISTORY": "Construir Historia", "SIGNED": "Firmada", "AUTHOR": "Autor", @@ -1288,6 +1289,9 @@ "ENABLED": "Enabled", "ENABLE": "Enable", "DISABLE": "Disable", - "DELETE_SUCCESS": "Successfully deleted" + "DELETE_SUCCESS": "Successfully deleted", + "TOTAL": "Total", + "FIXABLE": "Fixable", + "DURATION": "Duration:" } } diff --git a/src/portal/src/i18n/lang/fr-fr-lang.json b/src/portal/src/i18n/lang/fr-fr-lang.json index 286d29668..863ce0c0d 100644 --- a/src/portal/src/i18n/lang/fr-fr-lang.json +++ b/src/portal/src/i18n/lang/fr-fr-lang.json @@ -612,6 +612,7 @@ "PULL_COMMAND": "Commande de Pull", "PULL_TIME": "Pull Time", "PUSH_TIME": "Push Time", + "IMMUTABLE": "Immutable", "MY_REPOSITORY": "Mon Dépôt", "PUBLIC_REPOSITORY": "Dépôt Public", "DELETION_TITLE_REPO": "Confirmer la Suppresion du Dépôt", @@ -1260,6 +1261,9 @@ "ENABLED": "Enabled", "ENABLE": "Enable", "DISABLE": "Disable", - "DELETE_SUCCESS": "Successfully deleted" + "DELETE_SUCCESS": "Successfully deleted", + "TOTAL": "Total", + "FIXABLE": "Fixable", + "DURATION": "Duration:" } } diff --git a/src/portal/src/i18n/lang/pt-br-lang.json b/src/portal/src/i18n/lang/pt-br-lang.json index 21c391780..45bc09147 100644 --- a/src/portal/src/i18n/lang/pt-br-lang.json +++ b/src/portal/src/i18n/lang/pt-br-lang.json @@ -622,6 +622,7 @@ "PULL_COMMAND": "Comando de Pull", "PULL_TIME": "Pull Time", "PUSH_TIME": "Push Time", + "IMMUTABLE": "Immutable", "MY_REPOSITORY": "Meu Repositório", "PUBLIC_REPOSITORY": "Repositório Público", "DELETION_TITLE_REPO": "Confirmar remoção de repositório", @@ -1285,7 +1286,10 @@ "ENABLED": "Enabled", "ENABLE": "Enable", "DISABLE": "Disable", - "DELETE_SUCCESS": "Successfully deleted" + "DELETE_SUCCESS": "Successfully deleted", + "TOTAL": "Total", + "FIXABLE": "Fixable", + "DURATION": "Duration:" } } diff --git a/src/portal/src/i18n/lang/tr-tr-lang.json b/src/portal/src/i18n/lang/tr-tr-lang.json index 2956114d9..28ec1051c 100644 --- a/src/portal/src/i18n/lang/tr-tr-lang.json +++ b/src/portal/src/i18n/lang/tr-tr-lang.json @@ -621,6 +621,7 @@ "PULL_COMMAND": "İndirme Komutu", "PULL_TIME": "İndirme Zamanı", "PUSH_TIME": "Yükleme Zamanı", + "IMMUTABLE": "Immutable", "MY_REPOSITORY": "Depom", "PUBLIC_REPOSITORY": "Genel Depo", "DELETION_TITLE_REPO": "Depo Silme İşlemini Onaylayın", @@ -1290,6 +1291,9 @@ "ENABLED": "Enabled", "ENABLE": "Enable", "DISABLE": "Disable", - "DELETE_SUCCESS": "Successfully deleted" + "DELETE_SUCCESS": "Successfully deleted", + "TOTAL": "Total", + "FIXABLE": "Fixable", + "DURATION": "Duration:" } } diff --git a/src/portal/src/i18n/lang/zh-cn-lang.json b/src/portal/src/i18n/lang/zh-cn-lang.json index a964a2719..21abac5bd 100644 --- a/src/portal/src/i18n/lang/zh-cn-lang.json +++ b/src/portal/src/i18n/lang/zh-cn-lang.json @@ -623,6 +623,7 @@ "PULL_COMMAND": "Pull命令", "PULL_TIME": "拉取时间", "PUSH_TIME": "推送时间", + "IMMUTABLE": "保留的", "MY_REPOSITORY": "我的仓库", "PUBLIC_REPOSITORY": "公共仓库", "DELETION_TITLE_REPO": "删除镜像仓库确认", @@ -1287,6 +1288,9 @@ "ENABLED": "启用", "ENABLE": "启用", "DISABLE": "禁用", - "DELETE_SUCCESS": "删除成功" + "DELETE_SUCCESS": "删除成功", + "TOTAL": "总计", + "FIXABLE": "可修复", + "DURATION": "扫描用时:" } } diff --git a/tests/apitests/python/test_project_quota.py b/tests/apitests/python/test_project_quota.py index 8ab8c842b..820568411 100644 --- a/tests/apitests/python/test_project_quota.py +++ b/tests/apitests/python/test_project_quota.py @@ -73,7 +73,7 @@ class TestProjects(unittest.TestCase): #5. Get project quota quota = self.system.get_project_quota("project", TestProjects.project_test_quota_id, **ADMIN_CLIENT) self.assertEqual(quota[0].used["count"], 1) - self.assertEqual(quota[0].used["storage"], 2791709) + self.assertEqual(quota[0].used["storage"], 2789174) #6. Delete repository(RA) by user(UA); self.repo.delete_repoitory(TestProjects.repo_name, **ADMIN_CLIENT) diff --git a/tests/resources/Harbor-Pages/OIDC_Auth.robot b/tests/resources/Harbor-Pages/OIDC_Auth.robot index 021c2bb4d..866cbc19c 100644 --- a/tests/resources/Harbor-Pages/OIDC_Auth.robot +++ b/tests/resources/Harbor-Pages/OIDC_Auth.robot @@ -54,6 +54,6 @@ Generate And Return Secret Retry Element Click ${more_btn} Retry Element Click ${generate_secret_btn} Retry Double Keywords When Error Retry Element Click ${confirm_btn} Retry Wait Until Page Not Contains Element ${confirm_btn} - Retry Wait Until Page Contains generate CLI secret success + Retry Wait Until Page Contains Cli secret setting is successful ${secret}= Get Secrete By API ${url} [Return] ${secret} \ No newline at end of file diff --git a/tests/travis/api_common_install.sh b/tests/travis/api_common_install.sh index f698bd534..ada0a5540 100644 --- a/tests/travis/api_common_install.sh +++ b/tests/travis/api_common_install.sh @@ -24,5 +24,5 @@ sudo curl -o /home/travis/gopath/src/github.com/goharbor/harbor/tests/apitests/p sudo apt-get update && sudo apt-get install -y --no-install-recommends python-dev openjdk-7-jdk libssl-dev && sudo apt-get autoremove -y && sudo rm -rf /var/lib/apt/lists/* sudo wget https://bootstrap.pypa.io/get-pip.py && sudo python ./get-pip.py && sudo pip install --ignore-installed urllib3 chardet requests && sudo pip install robotframework==3.0.4 robotframework-httplibrary requests dbbot robotframework-pabot --upgrade sudo make swagger_client -sudo make install GOBUILDIMAGE=golang:1.12.5 COMPILETAG=compile_golangimage CLARITYIMAGE=goharbor/harbor-clarity-ui-builder:1.6.0 NOTARYFLAG=true CLAIRFLAG=true CHARTFLAG=true +sudo make install GOBUILDIMAGE=golang:1.12.12 COMPILETAG=compile_golangimage CLARITYIMAGE=goharbor/harbor-clarity-ui-builder:1.6.0 NOTARYFLAG=true CLAIRFLAG=true CHARTFLAG=true sleep 10 diff --git a/tests/travis/distro_installer.sh b/tests/travis/distro_installer.sh index 772d76678..c6baaf778 100755 --- a/tests/travis/distro_installer.sh +++ b/tests/travis/distro_installer.sh @@ -2,5 +2,5 @@ set -e -sudo make package_online VERSIONTAG=dev-travis PKGVERSIONTAG=dev-travis UIVERSIONTAG=dev-travis GOBUILDIMAGE=golang:1.12.5 COMPILETAG=compile_golangimage NOTARYFLAG=true CLAIRFLAG=true MIGRATORFLAG=false CHARTFLAG=true HTTPPROXY= -sudo make package_offline VERSIONTAG=dev-travis PKGVERSIONTAG=dev-travis UIVERSIONTAG=dev-travis GOBUILDIMAGE=golang:1.12.5 COMPILETAG=compile_golangimage NOTARYFLAG=true CLAIRFLAG=true MIGRATORFLAG=false CHARTFLAG=true HTTPPROXY= +sudo make package_online VERSIONTAG=dev-travis PKGVERSIONTAG=dev-travis UIVERSIONTAG=dev-travis GOBUILDIMAGE=golang:1.12.12 COMPILETAG=compile_golangimage NOTARYFLAG=true CLAIRFLAG=true MIGRATORFLAG=false CHARTFLAG=true HTTPPROXY= +sudo make package_offline VERSIONTAG=dev-travis PKGVERSIONTAG=dev-travis UIVERSIONTAG=dev-travis GOBUILDIMAGE=golang:1.12.12 COMPILETAG=compile_golangimage NOTARYFLAG=true CLAIRFLAG=true MIGRATORFLAG=false CHARTFLAG=true HTTPPROXY=