From 5dd75bb0b08b46897aee64cc24ef8b46cdf215b5 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Wed, 28 Mar 2018 19:13:57 +0800 Subject: [PATCH 1/5] Trust Root CA of VIC appliance when accessing Admiral --- src/common/utils/test/adminserver.go | 9 +++++++-- src/ui/config/config.go | 29 ++++++++++++++++++++++------ src/ui/config/config_test.go | 17 +++++++++++++++- src/ui/config/test/ca.crt | 18 +++++++++++++++++ 4 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 src/ui/config/test/ca.crt diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go index c3b6ebe13..72cb01a1d 100644 --- a/src/common/utils/test/adminserver.go +++ b/src/common/utils/test/adminserver.go @@ -61,7 +61,7 @@ var adminServerDefaultConfig = map[string]interface{}{ common.TokenExpiration: 30, common.CfgExpiration: 5, common.AdminInitialPassword: "password", - common.AdmiralEndpoint: "http://www.vmware.com", + common.AdmiralEndpoint: "", common.WithNotary: false, common.WithClair: false, common.ClairDBUsername: "postgres", @@ -84,8 +84,13 @@ func NewAdminserver(config map[string]interface{}) (*httptest.Server, error) { m := []*RequestHandlerMapping{} if config == nil { config = adminServerDefaultConfig + } else { + for k, v := range adminServerDefaultConfig { + if _, ok := config[k]; !ok { + config[k] = v + } + } } - b, err := json.Marshal(config) if err != nil { return nil, err diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 7a0ceb625..62a0693e2 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -16,8 +16,10 @@ package config import ( "crypto/tls" + "crypto/x509" "encoding/json" "fmt" + "io/ioutil" "net/http" "os" "strconv" @@ -58,6 +60,8 @@ var ( TokenReader admiral.TokenReader // GlobalJobserviceClient is a global client for jobservice GlobalJobserviceClient jobservice_client.Client + + defaultCACertPath = "/etc/ui/ca/ca.crt" ) // Init configurations @@ -94,8 +98,12 @@ func InitByURL(adminServerURL string) error { initSecretStore() // init project manager based on deploy mode - initProjectManager() + if err := initProjectManager(); err != nil { + log.Errorf("Failed to initialise project manager, error: %v", err) + return err + } + //TODO: No longer needed after shifting to the new job service. GlobalJobserviceClient = jobservice_client.NewDefaultClient(InternalJobServiceURL(), &jobservice_client.Config{ Secret: UISecret(), @@ -120,20 +128,28 @@ func initSecretStore() { SecretStore = secret.NewStore(m) } -func initProjectManager() { +func initProjectManager() error { var driver pmsdriver.PMSDriver if WithAdmiral() { - // integration with admiral - log.Info("initializing the project manager based on PMS...") - // TODO read ca/cert file and pass it to the TLS config + log.Debugf("Initialising Admiral client with certificate: %s", defaultCACertPath) + content, err := ioutil.ReadFile(defaultCACertPath) + if err != nil { + return err + } + pool := x509.NewCertPool() + if ok := pool.AppendCertsFromPEM(content); !ok { + return fmt.Errorf("failed to append cert content into cert pool") + } AdmiralClient = &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, + RootCAs: pool, }, }, } + // integration with admiral + log.Info("initializing the project manager based on PMS...") path := os.Getenv("SERVICE_TOKEN_FILE_PATH") if len(path) == 0 { path = defaultTokenFilePath @@ -149,6 +165,7 @@ func initProjectManager() { driver = local.NewDriver() } GlobalProjectMgr = promgr.NewDefaultProjectManager(driver, true) + return nil } diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index 016d19769..67e50c6cb 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -15,6 +15,8 @@ package config import ( "os" + "path" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -24,7 +26,12 @@ import ( // test functions under package ui/config func TestConfig(t *testing.T) { - server, err := test.NewAdminserver(nil) + + defaultCACertPath = path.Join(currPath(), "test", "ca.crt") + c := map[string]interface{}{ + common.AdmiralEndpoint: "http://www.vmware.com", + } + server, err := test.NewAdminserver(c) if err != nil { t.Fatalf("failed to create a mock admin server: %v", err) } @@ -190,3 +197,11 @@ func TestConfig(t *testing.T) { assert.Equal("http://myui:8888/service/token", InternalTokenServiceEndpoint()) } + +func currPath() string { + _, f, _, ok := runtime.Caller(0) + if !ok { + panic("Failed to get current directory") + } + return path.Dir(f) +} diff --git a/src/ui/config/test/ca.crt b/src/ui/config/test/ca.crt new file mode 100644 index 000000000..9807f27cd --- /dev/null +++ b/src/ui/config/test/ca.crt @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC7TCCAdWgAwIBAgIJAKmFRnILlp3XMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV +BAMMAmNhMB4XDTE3MDkyNDA3MDA1M1oXDTI3MDkyMjA3MDA1M1owDTELMAkGA1UE +AwwCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCr4+HxXkY81j1p +5OD3htFkbJI+XulBgc7ja5YorU323VB7JfNBnau3rDZS8NdyvkLLEQT4rKw5Dd4p +phlmdKsmIq9ej1OlDjWnCOGr+HG2jG5POgPYRCf5WgCGoQ4eUIA+IXcVroG8f1YM +LDzZEBKlEP80W0zyh0ma/BYN8HG4Ica4q/iIjffJc7ob/tWFGt2HobI9wbTSyBgR +s7JSs6MBIISXGAuOE3cs7vJNzKtWhQSBw4j8FFUZSYCyONFYfOg2OtZG6z1XhpTC +rfVMm6cEsYla/mf9bJB2AqtRiUdUZwAOWQbalWPFKEO73Bj4/5sVNHKFCd/S6J1z +LHaWM0W7AgMBAAGjUDBOMB0GA1UdDgQWBBR0jFgTuL9K2iWE0wzU7r4RZT0k+zAf +BgNVHSMEGDAWgBR0jFgTuL9K2iWE0wzU7r4RZT0k+zAMBgNVHRMEBTADAQH/MA0G +CSqGSIb3DQEBCwUAA4IBAQCemrfEKHPe5ahb2III89+iuIDmbPgVESXqnf88UUdS +Iv+htE8hu9CkSemsErXcC0kUbPSM0vWN9IbHINq78cXucVyi+YTzaKJ8zsK01/zf +x0xYeK5bffYTQzs+BopTCwVqd9zHSs9a2zPnsBVHXCn25j30anQgQH9ODsspXZ3i +WUAkEOmZDnNuX7tGDesA+7h8BPcZ8zrz94kxsrdneMXuHdT1iHxS/hTxTEUUhOMF +FntwT6zx3fGL4cNG06d+pdjjp+CuUR+8GRxeASbYBWhXeiY1ykipiptxkp1zhZ3x +SNandCCdeMRntnNs/+xvRhsEGbhyrvzg2WFL2NrqiKtg +-----END CERTIFICATE----- From 2debdb7229df3cbda3713b42b2d452ffbd7f91b6 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Thu, 29 Mar 2018 16:56:47 +0800 Subject: [PATCH 2/5] Return zero when input version is same as target --- tools/migration/cfg/migrate.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/migration/cfg/migrate.py b/tools/migration/cfg/migrate.py index 25c01a091..38accd860 100644 --- a/tools/migration/cfg/migrate.py +++ b/tools/migration/cfg/migrate.py @@ -5,6 +5,7 @@ from __future__ import print_function import argparse import os +import sys import utils import importlib import glob @@ -24,9 +25,12 @@ def main(): input_version = utils.get_conf_version(args.input_path) curr_dir = os.path.dirname(__file__) chain = [] + if input_version == target_version: + print ("Version of input harbor.cfg is identical to target %s, no need to upgrade" % input_version) + sys.exit(0) if not search(curr_dir, input_version, target_version, chain): print ("No migrator for version: %s" % input_version) - os.exit(1) + sys.exit(1) else: print ("input version: %s, migrator chain: %s" % (input_version, chain)) curr_input_path = args.input_path From 24dfad4afe07611f1d330a567f3a369695263149 Mon Sep 17 00:00:00 2001 From: Daniel Jiang Date: Fri, 30 Mar 2018 14:17:07 +0800 Subject: [PATCH 3/5] Return zero when input version is same as target (#4535) --- tools/migration/cfg/migrate.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/migration/cfg/migrate.py b/tools/migration/cfg/migrate.py index 25c01a091..38accd860 100644 --- a/tools/migration/cfg/migrate.py +++ b/tools/migration/cfg/migrate.py @@ -5,6 +5,7 @@ from __future__ import print_function import argparse import os +import sys import utils import importlib import glob @@ -24,9 +25,12 @@ def main(): input_version = utils.get_conf_version(args.input_path) curr_dir = os.path.dirname(__file__) chain = [] + if input_version == target_version: + print ("Version of input harbor.cfg is identical to target %s, no need to upgrade" % input_version) + sys.exit(0) if not search(curr_dir, input_version, target_version, chain): print ("No migrator for version: %s" % input_version) - os.exit(1) + sys.exit(1) else: print ("input version: %s, migrator chain: %s" % (input_version, chain)) curr_input_path = args.input_path From b1923bf2191d718e7cfc0d8c5822c5e809eca428 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Fri, 30 Mar 2018 14:41:12 +0800 Subject: [PATCH 4/5] Refine the python file to make it run on python3 --- tools/migration/cfg/migrator_1_5_0/__init__.py | 4 ++-- tools/migration/cfg/utils.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/migration/cfg/migrator_1_5_0/__init__.py b/tools/migration/cfg/migrator_1_5_0/__init__.py index a239df0a1..bead0a172 100644 --- a/tools/migration/cfg/migrator_1_5_0/__init__.py +++ b/tools/migration/cfg/migrator_1_5_0/__init__.py @@ -26,12 +26,12 @@ default = { def migrate(input_cfg, output_cfg): d = utils.read_conf(input_cfg) - keys = default.keys() + keys = list(default.keys()) keys.extend(['hostname', 'ui_url_protocol', 'max_job_workers', 'customize_crt', 'ssl_cert', 'ssl_cert_key', 'secretkey_path', 'admiral_url', 'db_password', 'clair_db_password']) val = {} for k in keys: - if d.has_key(k): + if k in d: val[k] = d[k] else: val[k] = default[k] diff --git a/tools/migration/cfg/utils.py b/tools/migration/cfg/utils.py index 5996012c5..34ed654d3 100644 --- a/tools/migration/cfg/utils.py +++ b/tools/migration/cfg/utils.py @@ -29,13 +29,13 @@ def read_conf(path): def get_conf_version(path): d = read_conf(path) # print json.dumps(d,indent=4) - if d.has_key("_version"): # >=1.5.0 + if "_version" in d: # >=1.5.0 return d["_version"] - if not d.has_key("clair_db_password"): + if not "clair_db_password" in d: return "unsupported" - if d.has_key("registry_storage_provider_name"): + if "registry_storage_provider_name" in d: return "1.4.0" - if d.has_key("uaa_endpoint"): + if "uaa_endpoint" in d: return "1.3.0" return "1.2.0" From 892d9bf02303d2868af535b7113d7ac4f6fbc19f Mon Sep 17 00:00:00 2001 From: Yan Date: Thu, 29 Mar 2018 23:49:00 -0700 Subject: [PATCH 5/5] bypass docker client retry on push in readonly mode (#4524) --- src/ui/filter/readonly.go | 1 + src/ui/proxy/interceptor_test.go | 6 ++++-- src/ui/proxy/interceptors.go | 24 ++++++++++++------------ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/ui/filter/readonly.go b/src/ui/filter/readonly.go index 44dbb8982..a734d5f09 100644 --- a/src/ui/filter/readonly.go +++ b/src/ui/filter/readonly.go @@ -42,6 +42,7 @@ func filter(req *http.Request, resp http.ResponseWriter) { } if matchRepoTagDelete(req) { resp.WriteHeader(http.StatusServiceUnavailable) + resp.Write([]byte("The system is in read only mode. Any modification is not prohibited.")) } } diff --git a/src/ui/proxy/interceptor_test.go b/src/ui/proxy/interceptor_test.go index 0a5756c78..d29feb9a0 100644 --- a/src/ui/proxy/interceptor_test.go +++ b/src/ui/proxy/interceptor_test.go @@ -195,8 +195,10 @@ func TestCopyResp(t *testing.T) { func TestMarshalError(t *testing.T) { assert := assert.New(t) - js := marshalError("Not Found") - assert.Equal("{\"errors\":[{\"code\":\"PROJECT_POLICY_VIOLATION\",\"message\":\"Not Found\",\"detail\":\"Not Found\"}]}", js) + js1 := marshalError("PROJECT_POLICY_VIOLATION", "Not Found") + assert.Equal("{\"errors\":[{\"code\":\"PROJECT_POLICY_VIOLATION\",\"message\":\"Not Found\",\"detail\":\"Not Found\"}]}", js1) + js2 := marshalError("DENIED", "The action is denied") + assert.Equal("{\"errors\":[{\"code\":\"DENIED\",\"message\":\"The action is denied\",\"detail\":\"The action is denied\"}]}", js2) } func TestIsDigest(t *testing.T) { diff --git a/src/ui/proxy/interceptors.go b/src/ui/proxy/interceptors.go index 5c9924d0a..fae3ceb3e 100644 --- a/src/ui/proxy/interceptors.go +++ b/src/ui/proxy/interceptors.go @@ -35,7 +35,7 @@ const ( var rec *httptest.ResponseRecorder // NotaryEndpoint , exported for testing. -var NotaryEndpoint ="" +var NotaryEndpoint = "" // MatchPullManifest checks if the request looks like a request to pull manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values func MatchPullManifest(req *http.Request) (bool, string, string) { @@ -123,20 +123,20 @@ func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if flag { components := strings.SplitN(repository, "/", 2) if len(components) < 2 { - http.Error(rw, marshalError(fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest) + http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest) return } client, err := uiutils.NewRepositoryClientForUI(tokenUsername, repository) if err != nil { log.Errorf("Error creating repository Client: %v", err) - http.Error(rw, marshalError(fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError) + http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError) return } digest, _, err := client.ManifestExist(reference) if err != nil { log.Errorf("Failed to get digest for reference: %s, error: %v", reference, err) - http.Error(rw, marshalError(fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError) + http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError) return } @@ -161,7 +161,7 @@ type readonlyHandler struct { func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { if config.ReadOnly() { if req.Method == http.MethodDelete || req.Method == http.MethodPost || req.Method == http.MethodPatch { - http.Error(rw, "Upload/Delete is prohibited in read only mode.", http.StatusServiceUnavailable) + http.Error(rw, marshalError("DENIED", "The system is in read only mode. Any modification is not prohibited."), http.StatusForbidden) return } } @@ -241,12 +241,12 @@ func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Reque } match, err := matchNotaryDigest(img) if err != nil { - http.Error(rw, marshalError("Failed in communication with Notary please check the log"), http.StatusInternalServerError) + http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed in communication with Notary please check the log"), http.StatusInternalServerError) return } if !match { log.Debugf("digest mismatch, failing the response.") - http.Error(rw, marshalError("The image is not signed in Notary."), http.StatusPreconditionFailed) + http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "The image is not signed in Notary."), http.StatusPreconditionFailed) return } cth.next.ServeHTTP(rw, req) @@ -275,19 +275,19 @@ func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) overview, err := dao.GetImgScanOverview(img.digest) if err != nil { log.Errorf("failed to get ImgScanOverview with repo: %s, reference: %s, digest: %s. Error: %v", img.repository, img.reference, img.digest, err) - http.Error(rw, marshalError("Failed to get ImgScanOverview."), http.StatusPreconditionFailed) + http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Failed to get ImgScanOverview."), http.StatusPreconditionFailed) return } // severity is 0 means that the image fails to scan or not scanned successfully. if overview == nil || overview.Sev == 0 { log.Debugf("cannot get the image scan overview info, failing the response.") - http.Error(rw, marshalError("Cannot get the image severity."), http.StatusPreconditionFailed) + http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", "Cannot get the image severity."), http.StatusPreconditionFailed) return } imageSev := overview.Sev if imageSev >= int(projectVulnerableSeverity) { log.Debugf("the image severity: %q is higher then project setting: %q, failing the response.", models.Severity(imageSev), projectVulnerableSeverity) - http.Error(rw, marshalError(fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", models.Severity(imageSev), projectVulnerableSeverity)), http.StatusPreconditionFailed) + http.Error(rw, marshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("The severity of vulnerability of the image: %q is equal or higher than the threshold in project setting: %q.", models.Severity(imageSev), projectVulnerableSeverity)), http.StatusPreconditionFailed) return } vh.next.ServeHTTP(rw, req) @@ -340,12 +340,12 @@ func copyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) { rw.Write(rec.Body.Bytes()) } -func marshalError(msg string) string { +func marshalError(code, msg string) string { var tmpErrs struct { Errors []JSONError `json:"errors,omitempty"` } tmpErrs.Errors = append(tmpErrs.Errors, JSONError{ - Code: "PROJECT_POLICY_VIOLATION", + Code: code, Message: msg, Detail: msg, })