From fd8fd2fbe13bd9ad7b6b1c444eddd19471332ab6 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Mon, 8 May 2017 13:48:20 +0800 Subject: [PATCH] Enable project level content trust, controlled by environment variable --- src/common/utils/notary/helper.go | 13 ++ src/common/utils/notary/helper_test.go | 27 ++++ src/ui/api/repository.go | 17 +-- src/ui/controllers/controllers_test.go | 10 ++ src/ui/controllers/proxy.go | 6 +- src/ui/proxy/interceptor_test.go | 123 ++++++++++++++++ src/ui/proxy/interceptors.go | 194 +++++++++++++++++++++++++ src/ui/proxy/proxy.go | 17 ++- 8 files changed, 383 insertions(+), 24 deletions(-) create mode 100644 src/ui/proxy/interceptor_test.go create mode 100644 src/ui/proxy/interceptors.go diff --git a/src/common/utils/notary/helper.go b/src/common/utils/notary/helper.go index e0bf803bd..60b7dbb89 100644 --- a/src/common/utils/notary/helper.go +++ b/src/common/utils/notary/helper.go @@ -28,6 +28,7 @@ import ( "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry" "github.com/vmware/harbor/src/common/utils/registry/auth" + "github.com/vmware/harbor/src/ui/config" "github.com/opencontainers/go-digest" ) @@ -56,6 +57,18 @@ func init() { trustPin = trustpinning.TrustPinConfig{} } +// GetInternalTargets wraps GetTargets to read config values for getting full-qualified repo from internal notary instance. +func GetInternalTargets(notaryEndpoint string, username string, repo string) ([]Target, error) { + ext, err := config.ExtEndpoint() + if err != nil { + log.Errorf("Error while reading external endpoint: %v", err) + return nil, err + } + endpoint := strings.Split(ext, "//")[1] + fqRepo := path.Join(endpoint, repo) + return GetTargets(notaryEndpoint, username, fqRepo) +} + // GetTargets is a help function called by API to fetch signature information of a given repository. // Per docker's convention the repository should contain the information of endpoint, i.e. it should look // like "10.117.4.117/library/ubuntu", instead of "library/ubuntu" (fqRepo for fully-qualified repo) diff --git a/src/common/utils/notary/helper_test.go b/src/common/utils/notary/helper_test.go index 2150dd863..83988d8a6 100644 --- a/src/common/utils/notary/helper_test.go +++ b/src/common/utils/notary/helper_test.go @@ -17,7 +17,10 @@ import ( "encoding/json" "fmt" "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common" notarytest "github.com/vmware/harbor/src/common/utils/notary/test" + utilstest "github.com/vmware/harbor/src/common/utils/test" + "github.com/vmware/harbor/src/ui/config" "net/http/httptest" "os" @@ -27,10 +30,27 @@ import ( var endpoint = "10.117.4.142" var notaryServer *httptest.Server +var adminServer *httptest.Server func TestMain(m *testing.M) { notaryServer = notarytest.NewNotaryServer(endpoint) defer notaryServer.Close() + var defaultConfig = map[string]interface{}{ + common.ExtEndpoint: "https://" + endpoint, + common.WithNotary: true, + common.CfgExpiration: 5, + } + adminServer, err := utilstest.NewAdminserver(defaultConfig) + if err != nil { + panic(err) + } + defer adminServer.Close() + if err := os.Setenv("ADMIN_SERVER_URL", adminServer.URL); err != nil { + panic(err) + } + if err := config.Init(); err != nil { + panic(err) + } notaryCachePath = "/tmp/notary" result := m.Run() if result != 0 { @@ -38,6 +58,13 @@ func TestMain(m *testing.M) { } } +func TestGetInternalTargets(t *testing.T) { + targets, err := GetInternalTargets(notaryServer.URL, "admin", "notary-demo/busybox") + assert.Nil(t, err, fmt.Sprintf("Unexpected error: %v", err)) + assert.Equal(t, 1, len(targets), "") + assert.Equal(t, "1.0", targets[0].Tag, "") +} + func TestGetTargets(t *testing.T) { targets, err := GetTargets(notaryServer.URL, "admin", path.Join(endpoint, "notary-demo/busybox")) assert.Nil(t, err, fmt.Sprintf("Unexpected error: %v", err)) diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 2b6300dad..07561021e 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -18,9 +18,7 @@ import ( "fmt" "io/ioutil" "net/http" - "path" "sort" - "strings" "time" "github.com/docker/distribution/manifest/schema1" @@ -225,7 +223,7 @@ func (ra *RepositoryAPI) Delete() { if config.WithNotary() { var digest string signedTags := make(map[string]struct{}) - targets, err := getNotaryTargets(user, repoName) + targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(), user, repoName) if err != nil { log.Errorf("Failed to get Notary targets for repository: %s, error: %v", repoName, err) log.Warningf("Failed to check signature status of repository: %s for deletion, there maybe orphaned targets in Notary.", repoName) @@ -589,7 +587,7 @@ func (ra *RepositoryAPI) GetSignatures() { } repoName := ra.GetString(":splat") - targets, err := getNotaryTargets(username, repoName) + targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(), username, repoName) if err != nil { log.Errorf("Error while fetching signature from notary: %v", err) ra.CustomAbort(http.StatusInternalServerError, "internal error") @@ -598,17 +596,6 @@ func (ra *RepositoryAPI) GetSignatures() { ra.ServeJSON() } -func getNotaryTargets(username string, repo string) ([]notary.Target, error) { - ext, err := config.ExtEndpoint() - if err != nil { - log.Errorf("Error while reading external endpoint: %v", err) - return nil, err - } - endpoint := strings.Split(ext, "//")[1] - fqRepo := path.Join(endpoint, repo) - return notary.GetTargets(config.InternalNotaryEndpoint(), username, fqRepo) -} - func newRepositoryClient(endpoint string, insecure bool, username, password, repository, scopeType, scopeName string, scopeActions ...string) (*registry.Repository, error) { diff --git a/src/ui/controllers/controllers_test.go b/src/ui/controllers/controllers_test.go index 7d1953687..9a463ad17 100644 --- a/src/ui/controllers/controllers_test.go +++ b/src/ui/controllers/controllers_test.go @@ -115,4 +115,14 @@ func TestMain(t *testing.T) { w = httptest.NewRecorder() beego.BeeApp.Handlers.ServeHTTP(w, r) assert.Equal(int(200), w.Code, "ping v2 should get a 200 response") + + r, _ = http.NewRequest("GET", "/registryproxy/v2/noproject/manifests/1.0", nil) + w = httptest.NewRecorder() + beego.BeeApp.Handlers.ServeHTTP(w, r) + assert.Equal(int(400), w.Code, "GET v2/noproject/manifests/1.0 should get a 400 response") + + r, _ = http.NewRequest("GET", "/registryproxy/v2/project/notexist/manifests/1.0", nil) + w = httptest.NewRecorder() + beego.BeeApp.Handlers.ServeHTTP(w, r) + assert.Equal(int(404), w.Code, "GET v2/noproject/manifests/1.0 should get a 404 response") } diff --git a/src/ui/controllers/proxy.go b/src/ui/controllers/proxy.go index 862ce080e..607cc72de 100644 --- a/src/ui/controllers/proxy.go +++ b/src/ui/controllers/proxy.go @@ -1,8 +1,6 @@ package controllers import ( - "strings" - "github.com/astaxie/beego" "github.com/vmware/harbor/src/ui/proxy" ) @@ -16,9 +14,7 @@ type RegistryProxy struct { func (p *RegistryProxy) Handle() { req := p.Ctx.Request rw := p.Ctx.ResponseWriter - req.URL.Path = strings.TrimPrefix(req.URL.Path, proxy.RegistryProxyPrefix) - //TODO interceptors - proxy.Proxy.ServeHTTP(rw, req) + proxy.Handle(rw, req) } // Render ... diff --git a/src/ui/proxy/interceptor_test.go b/src/ui/proxy/interceptor_test.go new file mode 100644 index 000000000..d1abcb8df --- /dev/null +++ b/src/ui/proxy/interceptor_test.go @@ -0,0 +1,123 @@ +package proxy + +import ( + "github.com/stretchr/testify/assert" + "github.com/vmware/harbor/src/common" + notarytest "github.com/vmware/harbor/src/common/utils/notary/test" + utilstest "github.com/vmware/harbor/src/common/utils/test" + "github.com/vmware/harbor/src/ui/config" + + "net/http" + "net/http/httptest" + "os" + "testing" +) + +var endpoint = "10.117.4.142" +var notaryServer *httptest.Server +var adminServer *httptest.Server + +func TestMain(m *testing.M) { + notaryServer = notarytest.NewNotaryServer(endpoint) + defer notaryServer.Close() + NotaryEndpoint = notaryServer.URL + var defaultConfig = map[string]interface{}{ + common.ExtEndpoint: "https://" + endpoint, + common.WithNotary: true, + common.CfgExpiration: 5, + } + adminServer, err := utilstest.NewAdminserver(defaultConfig) + if err != nil { + panic(err) + } + defer adminServer.Close() + if err := os.Setenv("ADMIN_SERVER_URL", adminServer.URL); err != nil { + panic(err) + } + if err := config.Init(); err != nil { + panic(err) + } + result := m.Run() + if result != 0 { + os.Exit(result) + } +} + +func TestMatchPullManifest(t *testing.T) { + assert := assert.New(t) + req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil) + res1, _, _ := MatchPullManifest(req1) + assert.False(res1, "%s %v is not a request to pull manifest", req1.Method, req1.URL) + + req2, _ := http.NewRequest("GET", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil) + res2, repo2, tag2 := MatchPullManifest(req2) + assert.True(res2, "%s %v is a request to pull manifest", req2.Method, req2.URL) + assert.Equal("library/ubuntu", repo2) + assert.Equal("14.04", tag2) + + req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/library/ubuntu/manifests/14.04", nil) + res3, _, _ := MatchPullManifest(req3) + assert.False(res3, "%s %v is not a request to pull manifest", req3.Method, req3.URL) + + req4, _ := http.NewRequest("GET", "https://192.168.0.5/v2/library/ubuntu/manifests/14.04", nil) + res4, repo4, tag4 := MatchPullManifest(req4) + assert.True(res4, "%s %v is a request to pull manifest", req4.Method, req4.URL) + assert.Equal("library/ubuntu", repo4) + assert.Equal("14.04", tag4) + + req5, _ := http.NewRequest("GET", "https://myregistry.com/v2/path1/path2/golang/manifests/1.6.2", nil) + res5, repo5, tag5 := MatchPullManifest(req5) + assert.True(res5, "%s %v is a request to pull manifest", req5.Method, req5.URL) + assert.Equal("path1/path2/golang", repo5) + assert.Equal("1.6.2", tag5) + + req6, _ := http.NewRequest("GET", "https://myregistry.com/v2/myproject/registry/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil) + res6, repo6, tag6 := MatchPullManifest(req6) + assert.True(res6, "%s %v is a request to pull manifest", req6.Method, req6.URL) + assert.Equal("myproject/registry", repo6) + assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag6) + + req7, _ := http.NewRequest("GET", "https://myregistry.com/v2/myproject/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil) + res7, repo7, tag7 := MatchPullManifest(req7) + assert.True(res7, "%s %v is a request to pull manifest", req7.Method, req7.URL) + assert.Equal("myproject", repo7) + assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7) +} + +func TestEnvPolicyChecker(t *testing.T) { + assert := assert.New(t) + if err := os.Setenv("PROJECT_CONTENT_TRUST", "1"); err != nil { + t.Fatalf("Failed to set env variable: %v", err) + } + contentTrustFlag := getPolicyChecker().contentTrustEnabled("whatever") + vulFlag := getPolicyChecker().vulnerableEnabled("whatever") + assert.True(contentTrustFlag) + assert.False(vulFlag) +} + +func TestMatchNotaryDigest(t *testing.T) { + assert := assert.New(t) + //The data from common/utils/notary/helper_test.go + img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo"} + img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo"} + res1, err := matchNotaryDigest(img1, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7") + assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1) + assert.True(res1) + res2, err := matchNotaryDigest(img1, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a8") + assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img1) + assert.False(res2) + res3, err := matchNotaryDigest(img2, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7") + assert.Nil(err, "Unexpected error: %v, image: %#v", err, img2) + assert.False(res3) +} + +func TestCopyResp(t *testing.T) { + assert := assert.New(t) + rec1 := httptest.NewRecorder() + rec2 := httptest.NewRecorder() + rec1.Header().Set("X-Test", "mytest") + rec1.WriteHeader(418) + copyResp(rec1, rec2) + assert.Equal(418, rec2.Result().StatusCode) + assert.Equal("mytest", rec2.Header().Get("X-Test")) +} diff --git a/src/ui/proxy/interceptors.go b/src/ui/proxy/interceptors.go new file mode 100644 index 000000000..84d9f1a47 --- /dev/null +++ b/src/ui/proxy/interceptors.go @@ -0,0 +1,194 @@ +package proxy + +import ( + // "github.com/vmware/harbor/src/ui/api" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/common/utils/notary" + "github.com/vmware/harbor/src/ui/config" + + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "regexp" + "strings" +) + +type contextKey string + +const ( + manifestURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})` + imageInfoCtxKey = contextKey("ImageInfo") + //TODO: temp solution, remove after vmware/harbor#2242 is resolved. + tokenUsername = "admin" +) + +// NotaryEndpoint , exported for testing. +var NotaryEndpoint = config.InternalNotaryEndpoint() + +// EnvChecker is the instance of envPolicyChecker +var EnvChecker = envPolicyChecker{} + +// MatchPullManifest checks if the request looks like a request to pull manifest. If it is returns the image and tag/sha256 digest as 2nd and 3rd return values +func MatchPullManifest(req *http.Request) (bool, string, string) { + //TODO: add user agent check. + if req.Method != http.MethodGet { + return false, "", "" + } + re := regexp.MustCompile(manifestURLPattern) + s := re.FindStringSubmatch(req.URL.Path) + if len(s) == 3 { + s[1] = strings.TrimSuffix(s[1], "/") + return true, s[1], s[2] + } + return false, "", "" +} + +// policyChecker checks the policy of a project by project name, to determine if it's needed to check the image's status under this project. +type policyChecker interface { + // contentTrustEnabled returns whether a project has enabled content trust. + contentTrustEnabled(name string) bool + // vulnerableEnabled returns whether a project has enabled content trust. + vulnerableEnabled(name string) bool +} + +//For testing +type envPolicyChecker struct{} + +func (ec envPolicyChecker) contentTrustEnabled(name string) bool { + return os.Getenv("PROJECT_CONTENT_TRUST") == "1" +} +func (ec envPolicyChecker) vulnerableEnabled(name string) bool { + return os.Getenv("PROJECT_VULNERABBLE") == "1" +} + +//TODO: integrate with PMS to get project policies +func getPolicyChecker() policyChecker { + return EnvChecker +} + +type imageInfo struct { + repository string + tag string + projectName string + // digest string +} + +type urlHandler struct { + next http.Handler +} + +//TODO: wrap a ResponseWriter to get the status code? + +func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + log.Debugf("in url handler, path: %s", req.URL.Path) + req.URL.Path = strings.TrimPrefix(req.URL.Path, RegistryProxyPrefix) + flag, repository, tag := MatchPullManifest(req) + if flag { + components := strings.SplitN(repository, "/", 2) + if len(components) < 2 { + http.Error(rw, fmt.Sprintf("Bad repository name: %s", repository), http.StatusBadRequest) + return + } + /* + //Need to get digest of the image. + endpoint, err := config.RegistryURL() + if err != nil { + log.Errorf("Error getting Registry URL: %v", err) + http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError) + return + } + rc, err := api.NewRepositoryClient(endpoint, false, username, repository, "repository", repository, "pull") + if err != nil { + log.Errorf("Error creating repository Client: %v", err) + http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError) + return + } + digest, exist, err := rc.ManifestExist(tag) + if err != nil { + log.Errorf("Failed to get digest for tag: %s, error: %v", tag, err) + http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError) + return + } + */ + + img := imageInfo{ + repository: repository, + tag: tag, + projectName: components[0], + } + log.Debugf("image info of the request: %#v", img) + + ctx := context.WithValue(req.Context(), imageInfoCtxKey, img) + req = req.WithContext(ctx) + + } + uh.next.ServeHTTP(rw, req) +} + +type contentTrustHandler struct { + next http.Handler +} + +func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + imgRaw := req.Context().Value(imageInfoCtxKey) + if imgRaw == nil || !config.WithNotary() { + cth.next.ServeHTTP(rw, req) + return + } + img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo) + if !getPolicyChecker().contentTrustEnabled(img.projectName) { + cth.next.ServeHTTP(rw, req) + return + } + //May need to update status code, let's use recorder + rec := httptest.NewRecorder() + cth.next.ServeHTTP(rec, req) + if rec.Result().StatusCode != http.StatusOK { + copyResp(rec, rw) + return + } + log.Debugf("showing digest") + digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest")) + log.Debugf("digest: %s", digest) + match, err := matchNotaryDigest(img, digest) + if err != nil { + http.Error(rw, "Failed in communication with Notary please check the log", http.StatusInternalServerError) + return + } + if match { + log.Debugf("Passing the response to outter responseWriter") + copyResp(rec, rw) + } else { + log.Debugf("digest miamatch, failing the response.") + http.Error(rw, "Failure in content trust handler", http.StatusPreconditionFailed) + } +} + +func matchNotaryDigest(img imageInfo, digest string) (bool, error) { + targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository) + if err != nil { + return false, err + } + for _, t := range targets { + if t.Tag == img.tag { + log.Debugf("found tag: %s in notary, try to match digest.") + d, err := notary.DigestFromTarget(t) + if err != nil { + return false, err + } + return digest == d, nil + } + } + log.Debugf("image: %#v, not found in notary", img) + return false, nil +} + +func copyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) { + for k, v := range rec.Header() { + rw.Header()[k] = v + } + rw.WriteHeader(rec.Result().StatusCode) + rw.Write(rec.Body.Bytes()) +} diff --git a/src/ui/proxy/proxy.go b/src/ui/proxy/proxy.go index c77f274af..8e63ec67f 100644 --- a/src/ui/proxy/proxy.go +++ b/src/ui/proxy/proxy.go @@ -4,6 +4,7 @@ import ( "github.com/vmware/harbor/src/ui/config" "fmt" + "net/http" "net/http/httputil" "net/url" ) @@ -11,10 +12,16 @@ import ( // Proxy is the instance of the reverse proxy in this package. var Proxy *httputil.ReverseProxy +var handlers handlerChain + // RegistryProxyPrefix is the prefix of url on UI. const RegistryProxyPrefix = "/registryproxy" -// Init initialize the Proxy instance. +type handlerChain struct { + head http.Handler +} + +// Init initialize the Proxy instance and handler chain. func Init(urls ...string) error { var err error var registryURL string @@ -34,9 +41,11 @@ func Init(urls ...string) error { return err } Proxy = httputil.NewSingleHostReverseProxy(targetURL) + handlers = handlerChain{head: urlHandler{next: contentTrustHandler{next: Proxy}}} return nil } -//func StartProxy(registryURL string) { -//http.ListenAndServe(":5000", Proxy) -//} +// Handle handles the request. +func Handle(rw http.ResponseWriter, req *http.Request) { + handlers.head.ServeHTTP(rw, req) +}