Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-22 18:25:56 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into add-2.1-upgrade-pipeline-in-nightly

Browse Source
This commit is contained in:
danfengliu 2020-09-22 13:48:22 +08:00 committed by GitHub
commit 48510fde08
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 126 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/v2.0/legacy_swagger.yaml
View File

@ -3679,6 +3679,10 @@ responses:
description: 'Success'
Created:
description: 'Created'
headers:
Location:
type: string
description: The URL of the created resource
BadRequest:
description: 'Bad Request'
Unauthorized:

4
src/portal/src/i18n/lang/zh-cn-lang.json
View File

@ -674,8 +674,8 @@
"DELETION_SUMMARY_REPO": "确认删除镜像仓库 {{repoName}}?",
"DELETION_TITLE_ARTIFACT": "删除镜像 Artifact 确认",
"DELETION_SUMMARY_ARTIFACT": "确认删除镜像 Artifact {{param}}? 如果您删除此 Artifact，则这个 digest 的所有 Tag 也将被删除。",
"DELETION_TITLE_TAG": "删除镜像 Tag 确认",
"DELETION_SUMMARY_TAG": "确认删除镜像 Tag {{param}}? 如果您删除此 Tag，则这个 Tag 引用的同一个 digest 的所有其他 Tag 也将被删除。",
"DELETION_TITLE_TAG": "删除 Tag 确认",
"DELETION_SUMMARY_TAG": "确认删除 Tag {{param}}?",
"DELETION_TITLE_TAG_DENIED": "已签名的镜像不能被删除",
"DELETION_SUMMARY_TAG_DENIED": "要删除此镜像 Tag 必须首先从 Notary 中删除。\n请执行如下 Notary 命令删除:\n",
"TAGS_NO_DELETE": "在只读模式下删除是被禁止的",

4
src/portal/src/i18n/lang/zh-tw-lang.json
View File

@ -670,8 +670,8 @@
"DELETION_SUMMARY_REPO": "確認刪除鏡像倉庫{{repoName}}?",
"DELETION_TITLE_ARTIFACT": "刪除鏡像Artifact 確認",
"DELETION_SUMMARY_ARTIFACT": "確認刪除鏡像Artifact {{param}}? 如果您刪除此Artifact,則這個digest 的所有Tag 也將被刪除。",
"DELETION_TITLE_TAG": "刪除鏡像Tag 確認",
"DELETION_SUMMARY_TAG": "確認刪除鏡像Tag {{param}}? 如果您刪除此Tag,則這個Tag 引用的同一個digest 的所有其他Tag 也將被刪除。",
"DELETION_TITLE_TAG": "刪除Tag 確認",
"DELETION_SUMMARY_TAG": "確認刪除Tag {{param}}?",
"DELETION_TITLE_TAG_DENIED": "已簽名的鏡像不能被刪除",
"DELETION_SUMMARY_TAG_DENIED": "要刪除此鏡像Tag 必須首先從Notary 中刪除。\n請執行如下Notary 命令刪除:\n",
"TAGS_NO_DELETE": "在只讀模式下刪除是被禁止的",

4
src/portal/src/lib/components/create-edit-endpoint/create-edit-endpoint.component.ts
View File

@ -38,6 +38,7 @@ const FAKE_PASSWORD = "rjGcfuRu";
const FAKE_JSON_KEY = "No Change";
const METADATA_URL = CURRENT_BASE_HREF + "/replication/adapterinfos";
const HELM_HUB = "helm-hub";
const FIXED_PATTERN_TYPE: string = "EndpointPatternTypeFix";
@Component({
selector: "hbr-create-edit-endpoint",
templateUrl: "./create-edit-endpoint.component.html",
@ -271,6 +272,9 @@ export class CreateEditEndpointComponent
if (this.endpointList.length === 1) {
this.target.url = this.endpointList[0].value;
}
if (this.adapterInfo[selectValue].endpoint_pattern.endpoint_type === FIXED_PATTERN_TYPE) {
this.urlDisabled = true;
}
} else {
this.endpointList = [];
}

87
src/replication/adapter/awsecr/adapter.go
View File

@ -16,15 +16,10 @@ package awsecr
import (
"errors"
"net/http"
"regexp"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/lib/log"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
@ -52,11 +47,16 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
if err != nil {
return nil, err
}
authorizer := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure)
svc, err := getAwsSvc(
region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure, nil)
if err != nil {
return nil, err
}
authorizer := NewAuth(registry.Credential.AccessKey, svc)
return &adapter{
registry: registry,
Adapter: native.NewAdapterWithAuthorizer(registry, authorizer),
region: region,
cacheSvc: svc,
}, nil
}
@ -88,9 +88,8 @@ var (
type adapter struct {
*native.Adapter
registry *model.Registry
region string
forceEndpoint *string
registry *model.Registry
cacheSvc *awsecrapi.ECR
}
func (*adapter) Info() (info *model.RegistryInfo, err error) {
@ -205,11 +204,6 @@ func getAdapterInfo() *model.AdapterPattern {
// HealthCheck checks health status of a registry
func (a *adapter) HealthCheck() (model.HealthStatus, error) {
if a.registry.Credential == nil ||
len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
log.Errorf("no credential to ping registry %s", a.registry.URL)
return model.Unhealthy, nil
}
if err := a.Ping(); err != nil {
log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
return model.Unhealthy, nil
@ -242,33 +236,7 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error {
}
func (a *adapter) createRepository(repository string) error {
if a.registry.Credential == nil ||
len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
return errors.New("no credential ")
}
cred := credentials.NewStaticCredentials(
a.registry.Credential.AccessKey,
a.registry.Credential.AccessSecret,
"")
if a.region == "" {
return errors.New("no region parsed")
}
config := &aws.Config{
Credentials: cred,
Region: &a.region,
HTTPClient: &http.Client{
Transport: commonhttp.GetHTTPTransportByInsecure(a.registry.Insecure),
},
}
if a.forceEndpoint != nil {
config.Endpoint = a.forceEndpoint
}
sess := session.Must(session.NewSession(config))
svc := awsecrapi.New(sess)
_, err := svc.CreateRepository(&awsecrapi.CreateRepositoryInput{
_, err := a.cacheSvc.CreateRepository(&awsecrapi.CreateRepositoryInput{
RepositoryName: &repository,
})
if err != nil {
@ -284,40 +252,7 @@ func (a *adapter) createRepository(repository string) error {
// DeleteManifest ...
func (a *adapter) DeleteManifest(repository, reference string) error {
// AWS doesn't implement standard OCI delete manifest API, so use it's sdk.
if a.registry.Credential == nil ||
len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
return errors.New("no credential ")
}
cred := credentials.NewStaticCredentials(
a.registry.Credential.AccessKey,
a.registry.Credential.AccessSecret,
"")
if a.region == "" {
return errors.New("no region parsed")
}
var tr *http.Transport
if a.registry.Insecure {
tr = commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
} else {
tr = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
}
config := &aws.Config{
Credentials: cred,
Region: &a.region,
HTTPClient: &http.Client{
Transport: tr,
},
}
if a.forceEndpoint != nil {
config.Endpoint = a.forceEndpoint
}
sess := session.Must(session.NewSession(config))
svc := awsecrapi.New(sess)
_, err := svc.BatchDeleteImage(&awsecrapi.BatchDeleteImageInput{
_, err := a.cacheSvc.BatchDeleteImage(&awsecrapi.BatchDeleteImageInput{
RepositoryName: &repository,
ImageIds: []*awsecrapi.ImageIdentifier{{ImageTag: &reference}},
})

32
src/replication/adapter/awsecr/adapter_test.go
View File

@ -3,6 +3,8 @@ package awsecr
import (
"errors"
"fmt"
awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
"github.com/stretchr/testify/require"
"io"
"io/ioutil"
"net/http"
@ -149,17 +151,23 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser
Type: model.RegistryTypeAwsEcr,
URL: server.URL,
}
var svc *awsecrapi.ECR
if hasCred {
registry.Credential = &model.Credential{
AccessKey: "xxx",
AccessSecret: "ppp",
}
svc, _ = getAwsSvc(
"test-region", registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure, &server.URL)
} else {
svc, _ = getAwsSvc(
"test-region", "", "", registry.Insecure, &server.URL)
}
return &adapter{
registry: registry,
Adapter: native.NewAdapter(registry),
region: "test-region",
forceEndpoint: &server.URL,
registry: registry,
Adapter: native.NewAdapter(registry),
cacheSvc: svc,
}, server
}
@ -180,7 +188,7 @@ func TestAdapter_HealthCheck(t *testing.T) {
status, err := a.HealthCheck()
assert.Nil(t, err)
assert.NotNil(t, status)
assert.EqualValues(t, model.Unhealthy, status)
assert.EqualValues(t, model.Healthy, status)
a, s = getMockAdapter(t, true, false)
defer s.Close()
@ -260,16 +268,18 @@ func TestAwsAuthCredential_Modify(t *testing.T) {
},
)
defer server.Close()
a, _ := NewAuth("test-region", "xxx", "ppp", true).(*awsAuthCredential)
a.forceEndpoint = &server.URL
svc, err := getAwsSvc(
"test-region", "xxx", "ppp", true, &server.URL)
require.Nil(t, err)
a, _ := NewAuth("xxx", svc).(*awsAuthCredential)
req := httptest.NewRequest(http.MethodGet, "https://1234.dkr.ecr.test-region.amazonaws.com/v2/", nil)
err := a.Modify(req)
assert.Nil(t, err)
err = a.Modify(req)
assert.Nil(t, err)
require.Nil(t, err)
err = a.Modify(req)
require.Nil(t, err)
time.Sleep(time.Second)
err = a.Modify(req)
assert.Nil(t, err)
require.Nil(t, err)
}
var urlForBenchmark = []string{

65
src/replication/adapter/awsecr/auth.go
View File

@ -18,19 +18,19 @@ import (
"encoding/base64"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
"github.com/aws/aws-sdk-go/aws/session"
awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/lib/log"
"net/http"
"net/url"
"strings"
"time"
)
// Credential ...
@ -38,11 +38,8 @@ type Credential modifier.Modifier
// Implements interface Credential
type awsAuthCredential struct {
region string
accessKey string
accessSecret string
insecure bool
forceEndpoint *string
accessKey string
awssvc *awsecrapi.ECR
cacheToken *cacheToken
cacheExpired *time.Time
@ -91,36 +88,44 @@ func (a *awsAuthCredential) Modify(req *http.Request) error {
return nil
}
func (a *awsAuthCredential) getAuthorization() (string, string, string, *time.Time, error) {
log.Infof("Aws Ecr getAuthorization %s", a.accessKey)
cred := credentials.NewStaticCredentials(
a.accessKey,
a.accessSecret,
"")
func getAwsSvc(region, accessKey, accessSecret string, insecure bool, forceEndpoint *string) (*awsecrapi.ECR, error) {
sess, err := session.NewSession()
if err != nil {
return nil, err
}
var cred *credentials.Credentials
log.Debugf("Aws Ecr getAuthorization %s", accessKey)
if accessKey != "" {
cred = credentials.NewStaticCredentials(
accessKey,
accessSecret,
"")
} else {
cred = ec2rolecreds.NewCredentials(sess)
}
var tr *http.Transport
if a.insecure {
if insecure {
tr = commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
} else {
tr = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
}
config := &aws.Config{
Credentials: cred,
Region: &a.region,
Region: &region,
HTTPClient: &http.Client{
Transport: tr,
},
}
if a.forceEndpoint != nil {
config.Endpoint = a.forceEndpoint
}
sess, err := session.NewSession(config)
if err != nil {
return "", "", "", nil, err
if forceEndpoint != nil {
config.Endpoint = forceEndpoint
}
svc := awsecrapi.New(sess)
svc := awsecrapi.New(sess, config)
return svc, nil
}
func (a *awsAuthCredential) getAuthorization() (string, string, string, *time.Time, error) {
svc := a.awssvc
result, err := svc.GetAuthorizationToken(nil)
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
@ -161,11 +166,9 @@ func (a *awsAuthCredential) isTokenValid() bool {
}
// NewAuth new aws auth
func NewAuth(region, accessKey, accessSecret string, insecure bool) Credential {
func NewAuth(accessKey string, awssvc *awsecrapi.ECR) Credential {
return &awsAuthCredential{
region: region,
accessKey: accessKey,
accessSecret: accessSecret,
insecure: insecure,
accessKey: accessKey,
awssvc: awssvc,
}
}

2
tests/resources/Harbor-Pages/Replication.robot
View File

@ -167,7 +167,7 @@ Delete Rule
Select Rule
[Arguments] ${rule}
Retry Element Click //clr-dg-row[contains(.,'${rule}')]//label
Retry Double Keywords When Error Retry Element Click //clr-dg-cell[contains(.,'${rule}')] Retry Wait Element ${replication_exec_id}
Stop Jobs
Retry Element Click ${stop_jobs_button}

5
tests/resources/Helm-Util.robot
View File

@ -18,6 +18,9 @@ Library OperatingSystem
Library Process
*** Keywords ***
Prepare Helm Cert
Wait Unitl Command Success cp harbor_ca.crt /ca/server.crt
Helm Repo Add
[Arguments] ${harbor_url} ${user} ${pwd} ${project_name}=library ${helm_repo_name}=myrepo
${rc} ${output}= Run And Return Rc And Output helm repo remove ${project_name}
@ -35,4 +38,4 @@ Helm Chart Push
[Arguments] ${ip} ${user} ${pwd} ${chart_file} ${archive} ${project} ${repo_name} ${verion}
${rc} ${output}= Run And Return Rc And Output ./tests/robot-cases/Group0-Util/helm_push_chart.sh ${ip} ${user} ${pwd} ${chart_file} ${archive} ${project} ${repo_name} ${verion}
Log ${output}
Should Be Equal As Integers ${rc} 0
Should Be Equal As Integers ${rc} 0

14
tests/resources/Nightly-Util.robot
View File

@ -40,6 +40,20 @@ CA Setup
Log To Console Prepare Docker Cert ...
Prepare Docker Cert ${ip}
Nightly Test Setup For Nightly
[Arguments] ${ip} ${HARBOR_PASSWORD} ${ip1}==${EMPTY}
Run Keyword If '${ip1}' != '${EMPTY}' CA setup For Nightly ${ip1} ${HARBOR_PASSWORD} /ca/ca1.crt
Run Keyword If '${ip1}' != '${EMPTY}' Run rm -rf ./harbor_ca.crt
Run Keyword CA setup For Nightly ${ip} ${HARBOR_PASSWORD}
Run Keyword Start Docker Daemon Locally
CA Setup For Nightly
[Arguments] ${ip} ${HARBOR_PASSWORD} ${cert}=/ca/ca.crt
Run cp ${cert} harbor_ca.crt
Generate Certificate Authority For Chrome ${HARBOR_PASSWORD}
Prepare Docker Cert ${ip}
Prepare Helm Cert
Collect Nightly Logs
[Arguments] ${ip} ${SSH_PWD} ${ip1}==${EMPTY}
Run Keyword Collect Logs ${ip} ${SSH_PWD}

2
tests/robot-cases/Group1-Nightly/Common.robot
View File

@ -619,7 +619,7 @@ Test Case - Push CNAB Bundle and Display
Create An New Project And Go Into Project test${d}
${target}= Set Variable ${ip}/test${d}/cnab${d}:cnab_tag${d}
CNAB Push Bundle ${ip} user010 Test1@34 ${target} ./tests/robot-cases/Group0-Util/bundle.json
Retry Keyword N Times When Error 5 CNAB Push Bundle ${ip} user010 Test1@34 ${target} ./tests/robot-cases/Group0-Util/bundle.json
Go Into Project test${d}
Wait Until Page Contains test${d}/cnab${d}

27
tests/robot-cases/Group1-Nightly/Setup_Nightly.robot Normal file
View File

@ -0,0 +1,27 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
*** Settings ***
Documentation Harbor BATs
Resource ../../resources/Util.robot
Default Tags Nightly
*** Test Cases ***
Test Suites Setup For UI Test
Nightly Test Setup For Nightly ${ip} ${HARBOR_PASSWORD} ${ip1}
Setup API Test
Test Case - Get Harbor Version
#Just get harbor version and log it
Get Harbor Version

5
tests/robot-cases/Group3-Upgrade/prepare.py
View File

@ -544,12 +544,9 @@ class HarborAPI:
image_b = "busybox"
repo_name_a, tag_a = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_a, "latest")
repo_name_b, tag_b = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_b, "latest")
#4. Push an index(IA) to Harbor by docker manifest CLI successfully;
manifests = [args.endpoint+"/"+repo_name_a+":"+tag_a, args.endpoint+"/"+repo_name_b+":"+tag_b]
index = args.endpoint+"/"+project+"/"+name+":"+tag
docker_manifest_push_to_harbor(index, manifests, args.endpoint, 'admin', 'Harbor12345', cfg_file = args.libpath + "/update_docker_cfg.sh")
docker_manifest_push_to_harbor(index, manifests, args.endpoint, 'admin', 'Harbor12345', cfg_file = args.libpath + "/update_docker_cfg.sh"
def request(url, method, user = None, userp = None, **kwargs):
if user is None: