Merge pull request #1562 from reasonerjt/notary-integration

provide systeminfo API for UI

docs/swagger.yaml

@@ -1343,6 +1343,22 @@ paths:
           description: User does not have permission of admin role.
         500:
           description: Unexpected internal errors. 
+  /systeminfo:
+    get:
+      summary: Get general system info
+      description: |
+        This API is for retrieving general system info, this can be called by anonymous request.
+      tags:
+        - Products
+      responses:
+        200:
+          description: Get general info successfully.
+          schema:
+            type: object
+            items:
+              $ref: "#/definitions/GeneralInfo"
+        500:
+          description: Unexpected internal error.
   /systeminfo/volumes:
     get:
       summary: Get system volume info (total/free size).
@@ -1967,6 +1983,27 @@ definitions:
         type: integer
         format: int64
         description: Free volume size.
+  GeneralInfo:
+    type: object
+    properties:
+      with_notary:
+        type: boolean
+        description: If the Harbor instance is deployed with nested notary.
+      with_admiral:
+        type: boolean
+        description: If the Harbor instance is deployed with Admiral.
+      admiral_endpoint:
+        type: string
+        description: The url of the endpoint of admiral instance.
+      auth_mode:
+        type: string
+        description: The auth mode of current Harbor instance.
+      project_creation_restriction:
+        type: string
+        description: Indicate who can create projects, it could be 'adminonly' or 'everyone'.
+      self_registration:
+        type: boolean
+        description: Indicate whether the Harbor instance enable user to register himself.
   SystemInfo:
     type: object
     properties:

make/common/templates/adminserver/env

@@ -34,4 +34,6 @@ JOBSERVICE_SECRET=$jobservice_secret
 TOKEN_EXPIRATION=$token_expiration
 CFG_EXPIRATION=5
 USE_COMPRESSED_JS=$use_compressed_js
-GODEBUG=netdns=cgo
+GODEBUG=netdns=cgo
+ADMIRAL_URL=$admiral_url
+WITH_NOTARY=$with_notary

make/harbor.cfg

@@ -40,6 +40,9 @@ ssl_cert_key = /data/cert/server.key
 #The path of secretkey storage
 secretkey_path = /data
 
+#Admiral's url, comment this attribute, or set its value to to NA when Harbor is standalone
+admiral_url = NA
+
 #NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
 #only take effect in the first boot, the subsequent changes of these properties 
 #should be performed on web ui

make/prepare

@@ -147,6 +147,7 @@ token_expiration = rcp.get("configuration", "token_expiration")
 verify_remote_cert = rcp.get("configuration", "verify_remote_cert")
 proj_cre_restriction = rcp.get("configuration", "project_creation_restriction")
 secretkey_path = rcp.get("configuration", "secretkey_path")
+admiral_url = rcp.get("configuration", "admiral_url")
 secret_key = get_secret_key(secretkey_path)
 ########
 
@@ -190,10 +191,10 @@ else:
         nginx_conf)
 		
 render(os.path.join(templates_dir, "adminserver", "env"),
-		adminserver_conf_env,
-		ui_url=ui_url,
-		auth_mode=auth_mode,
-		self_registration=self_registration,
+        adminserver_conf_env,
+        ui_url=ui_url,
+        auth_mode=auth_mode,
+        self_registration=self_registration,
         ldap_url=ldap_url,
         ldap_searchdn =ldap_searchdn, 
         ldap_search_pwd =ldap_search_pwd,
@@ -203,27 +204,29 @@ render(os.path.join(templates_dir, "adminserver", "env"),
         ldap_scope=ldap_scope,
         ldap_timeout=ldap_timeout,
         db_password=db_password,
-		email_host=email_host,
-		email_port=email_port,
-		email_usr=email_usr,
-		email_pwd=email_pwd,
-		email_ssl=email_ssl,
-		email_from=email_from,
-		email_identity=email_identity,
+        email_host=email_host,
+        email_port=email_port,
+        email_usr=email_usr,
+        email_pwd=email_pwd,
+        email_ssl=email_ssl,
+        email_from=email_from,
+        email_identity=email_identity,
         harbor_admin_password=harbor_admin_password,
-		project_creation_restriction=proj_cre_restriction,
-		verify_remote_cert=verify_remote_cert,
-		max_job_workers=max_job_workers,
-		ui_secret=ui_secret,
-		jobservice_secret=jobservice_secret,
-		token_expiration=token_expiration,
-		use_compressed_js=use_compressed_js
+        project_creation_restriction=proj_cre_restriction,
+        verify_remote_cert=verify_remote_cert,
+        max_job_workers=max_job_workers,
+        ui_secret=ui_secret,
+        jobservice_secret=jobservice_secret,
+        token_expiration=token_expiration,
+        admiral_url=admiral_url,
+        with_notary=args.notary_mode,
+        use_compressed_js=use_compressed_js
 	)
 
 render(os.path.join(templates_dir, "ui", "env"), 
-		ui_conf_env, 
-		ui_secret=ui_secret,
-		jobservice_secret=jobservice_secret,)
+        ui_conf_env, 
+        ui_secret=ui_secret,
+        jobservice_secret=jobservice_secret,)
 
 render(os.path.join(templates_dir, "registry", 
 		"config.yml"),
@@ -237,8 +240,8 @@ render(os.path.join(templates_dir, "db", "env"),
 render(os.path.join(templates_dir, "jobservice", "env"),
         job_conf_env,
         ui_secret=ui_secret,
-		jobservice_secret=jobservice_secret)
-		
+        jobservice_secret=jobservice_secret)
+
 print("Generated configuration file: %s" % jobservice_conf)
 shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf)
 
@@ -328,5 +331,6 @@ if args.notary_mode:
     default_alias = ''.join(random.choice(string.ascii_letters) for i in range(8))
     render(os.path.join(notary_temp_dir, "signer_env"), os.path.join(notary_config_dir, "signer_env"), alias = default_alias)
 
+		
 print("The configuration files are ready, please use docker-compose to start the service.")
 

src/adminserver/systemcfg/systemcfg.go

@@ -114,6 +114,11 @@ var (
 		},
 		comcfg.ProjectCreationRestriction: "PROJECT_CREATION_RESTRICTION",
 		comcfg.AdminInitialPassword:       "HARBOR_ADMIN_PASSWORD",
+		comcfg.AdmiralEndpoint:            "ADMIRAL_URL",
+		comcfg.WithNotary: &parser{
+			env:   "WITH_NOTARY",
+			parse: parseStringToBool,
+		},
 	}
 
 	// configurations need read from environment variables
@@ -134,6 +139,11 @@ var (
 			env:   "CFG_EXPIRATION",
 			parse: parseStringToInt,
 		},
+		comcfg.AdmiralEndpoint: "ADMIRAL_URL",
+		comcfg.WithNotary: &parser{
+			env:   "WITH_NOTARY",
+			parse: parseStringToBool,
+		},
 	}
 )
 

src/common/config/config.go

@@ -75,6 +75,8 @@ const (
 	JobLogDir                  = "job_log_dir"
 	UseCompressedJS            = "use_compressed_js"
 	AdminInitialPassword       = "admin_initial_password"
+	AdmiralEndpoint            = "admiral_url"
+	WithNotary                 = "with_notary"
 )
 
 // Manager manages configurations

src/common/utils/notary/helper.go

@@ -28,7 +28,6 @@ import (
 )
 
 var (
-	notaryEndpoint  = "http://notary-server:4443"
 	notaryCachePath = "/root/notary"
 	trustPin        trustpinning.TrustPinConfig
 	mockRetriever   notary.PassRetriever
@@ -55,7 +54,7 @@ func init() {
 // GetTargets is a help function called by API to fetch signature information of a given repository.
 // Per docker's convention the repository should contain the information of endpoint, i.e. it should look
 // like "10.117.4.117/library/ubuntu", instead of "library/ubuntu" (fqRepo for fully-qualified repo)
-func GetTargets(username string, fqRepo string) ([]Target, error) {
+func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]Target, error) {
 	res := []Target{}
 	authorizer := auth.NewNotaryUsernameTokenAuthorizer(username, "repository", fqRepo, "pull")
 	store, err := auth.NewAuthorizerStore(strings.Split(notaryEndpoint, "//")[1], true, authorizer)

src/common/utils/notary/helper_test.go

@@ -5,17 +5,18 @@ import (
 	"github.com/stretchr/testify/assert"
 	notarytest "github.com/vmware/harbor/src/common/utils/notary/test"
 
+	"net/http/httptest"
 	"os"
 	"path"
 	"testing"
 )
 
 var endpoint = "10.117.4.142"
+var notaryServer *httptest.Server
 
 func TestMain(m *testing.M) {
-	notaryServer := notarytest.NewNotaryServer(endpoint)
+	notaryServer = notarytest.NewNotaryServer(endpoint)
 	defer notaryServer.Close()
-	notaryEndpoint = notaryServer.URL
 	notaryCachePath = "/tmp/notary"
 	result := m.Run()
 	if result != 0 {
@@ -24,12 +25,12 @@ func TestMain(m *testing.M) {
 }
 
 func TestGetTargets(t *testing.T) {
-	targets, err := GetTargets("admin", path.Join(endpoint, "notary-demo/busybox"))
+	targets, err := GetTargets(notaryServer.URL, "admin", path.Join(endpoint, "notary-demo/busybox"))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error: %v", err))
 	assert.Equal(t, 1, len(targets), "")
 	assert.Equal(t, "1.0", targets[0].Tag, "")
 
-	targets, err = GetTargets("admin", path.Join(endpoint, "notary-demo/notexist"))
+	targets, err = GetTargets(notaryServer.URL, "admin", path.Join(endpoint, "notary-demo/notexist"))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error: %v", err))
 	assert.Equal(t, 0, len(targets), "Targets list should be empty for non exist repo.")
 }

src/common/utils/test/adminserver.go

@@ -58,6 +58,8 @@ var adminServerDefaultConfig = map[string]interface{}{
 	config.CfgExpiration:              5,
 	config.UseCompressedJS:            true,
 	config.AdminInitialPassword:       "password",
+	config.AdmiralEndpoint:            "http://www.vmware.com",
+	config.WithNotary:                 false,
 }
 
 // NewAdminserver returns a mock admin server

src/ui/api/harborapi_test.go

@@ -97,6 +97,7 @@ func init() {
 	beego.Router("/api/policies/replication", &RepPolicyAPI{}, "get:List")
 	beego.Router("/api/policies/replication", &RepPolicyAPI{}, "post:Post;delete:Delete")
 	beego.Router("/api/policies/replication/:id([0-9]+)/enablement", &RepPolicyAPI{}, "put:UpdateEnablement")
+	beego.Router("/api/systeminfo", &SystemInfoAPI{}, "get:GetGeneralInfo")
 	beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo")
 	beego.Router("/api/systeminfo/getcert", &SystemInfoAPI{}, "get:GetCert")
 	beego.Router("/api/ldap/ping", &LdapAPI{}, "post:Ping")
@@ -953,6 +954,11 @@ func (a testapi) VolumeInfoGet(authInfo usrInfo) (int, apilib.SystemInfo, error)
 	return httpStatusCode, successPayLoad, err
 }
 
+func (a testapi) GetGeneralInfo() (int, []byte, error) {
+	_sling := sling.New().Get(a.basePath).Path("/api/systeminfo")
+	return request(_sling, jsonAcceptHeader)
+}
+
 //Get system cert
 func (a testapi) CertGet(authInfo usrInfo) (int, []byte, error) {
 	_sling := sling.New().Get(a.basePath)

src/ui/api/repository.go

@@ -554,7 +554,7 @@ func (ra *RepositoryAPI) GetSignatures() {
 	if err != nil {
 		log.Warningf("Error when getting username: %v", err)
 	}
-	targets, err := notary.GetTargets(username, fqRepo)
+	targets, err := notary.GetTargets(config.InternalNotaryEndpoint(), username, fqRepo)
 	if err != nil {
 		log.Errorf("Error while fetching signature from notary: %v", err)
 		ra.CustomAbort(http.StatusInternalServerError, "internal error")

src/ui/api/systeminfo.go

@@ -4,11 +4,14 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 
 	"github.com/vmware/harbor/src/common/api"
+	comcfg "github.com/vmware/harbor/src/common/config"
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/ui/config"
 )
 
 //SystemInfoAPI handle requests for getting system info /api/systeminfo
@@ -32,8 +35,19 @@ type Storage struct {
 	Free  uint64 `json:"free"`
 }
 
-// Prepare for validating user if an admin.
-func (sia *SystemInfoAPI) Prepare() {
+//GeneralInfo wraps common systeminfo for anonymous request
+type GeneralInfo struct {
+	WithNotary              bool   `json:"with_notary"`
+	WithAdmiral             bool   `json:"with_admiral"`
+	AdmiralEndpoint         string `json:"admiral_endpoint"`
+	AuthMode                string `json:"auth_mode"`
+	RegistryURL             string `json:"registry_url"`
+	ProjectCreationRestrict string `json:"project_creation_restriction"`
+	SelfRegistration        bool   `json:"self_registration"`
+}
+
+// validate for validating user if an admin.
+func (sia *SystemInfoAPI) validate() {
 	sia.currentUserID = sia.ValidateUser()
 
 	var err error
@@ -46,6 +60,7 @@ func (sia *SystemInfoAPI) Prepare() {
 
 // GetVolumeInfo gets specific volume storage info.
 func (sia *SystemInfoAPI) GetVolumeInfo() {
+	sia.validate()
 	if !sia.isAdmin {
 		sia.RenderError(http.StatusForbidden, "User does not have admin role.")
 		return
@@ -71,6 +86,7 @@ func (sia *SystemInfoAPI) GetVolumeInfo() {
 
 //GetCert gets default self-signed certificate.
 func (sia *SystemInfoAPI) GetCert() {
+	sia.validate()
 	if sia.isAdmin {
 		if _, err := os.Stat(defaultRootCert); !os.IsNotExist(err) {
 			sia.Ctx.Output.Header("Content-Type", "application/octet-stream")
@@ -83,3 +99,29 @@ func (sia *SystemInfoAPI) GetCert() {
 	}
 	sia.CustomAbort(http.StatusForbidden, "")
 }
+
+// GetGeneralInfo returns the general system info, which is to be called by anonymous user
+func (sia *SystemInfoAPI) GetGeneralInfo() {
+	cfg, err := config.GetSystemCfg()
+	if err != nil {
+		log.Errorf("Error occured getting config: %v", err)
+		sia.CustomAbort(http.StatusInternalServerError, "Unexpected error")
+	}
+	var registryURL string
+	if l := strings.Split(cfg[comcfg.ExtEndpoint].(string), "://"); len(l) > 1 {
+		registryURL = l[1]
+	} else {
+		registryURL = l[0]
+	}
+	info := GeneralInfo{
+		AdmiralEndpoint:         cfg[comcfg.AdmiralEndpoint].(string),
+		WithAdmiral:             config.WithAdmiral(),
+		WithNotary:              config.WithNotary(),
+		AuthMode:                cfg[comcfg.AUTHMode].(string),
+		ProjectCreationRestrict: cfg[comcfg.ProjectCreationRestriction].(string),
+		SelfRegistration:        cfg[comcfg.SelfRegistration].(bool),
+		RegistryURL:             registryURL,
+	}
+	sia.Data["json"] = info
+	sia.ServeJSON()
+}

src/ui/api/systeminfo_test.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/stretchr/testify/assert"
 	"testing"
@@ -37,6 +38,18 @@ func TestGetVolumeInfo(t *testing.T) {
 
 }
 
+func TestGetGeneralInfo(t *testing.T) {
+	apiTest := newHarborAPI()
+	code, body, err := apiTest.GetGeneralInfo()
+	assert := assert.New(t)
+	assert.Nil(err, fmt.Sprintf("Unexpected Error: %v", err))
+	assert.Equal(200, code, fmt.Sprintf("Unexpected status code: %d", code))
+	g := &GeneralInfo{}
+	err = json.Unmarshal(body, g)
+	assert.Nil(err, fmt.Sprintf("Unexpected Error: %v", err))
+	assert.Equal(false, g.WithNotary, "with notary should be false")
+}
+
 func TestGetCert(t *testing.T) {
 	fmt.Println("Testing Get Cert")
 	assert := assert.New(t)

src/ui/config/config.go

@@ -18,6 +18,7 @@ package config
 import (
 	"encoding/json"
 	"os"
+	"strings"
 
 	comcfg "github.com/vmware/harbor/src/common/config"
 	"github.com/vmware/harbor/src/common/models"
@@ -132,7 +133,7 @@ func TokenExpiration() (int, error) {
 	return int(cfg[comcfg.TokenExpiration].(float64)), nil
 }
 
-// ExtEndpoint returns the external URL of Harbor: protocal://host:port
+// ExtEndpoint returns the external URL of Harbor: protocol://host:port
 func ExtEndpoint() (string, error) {
 	cfg, err := mg.Get()
 	if err != nil {
@@ -141,6 +142,19 @@ func ExtEndpoint() (string, error) {
 	return cfg[comcfg.ExtEndpoint].(string), nil
 }
 
+// ExtURL returns the external URL: host:port
+func ExtURL() (string, error) {
+	endpoint, err := ExtEndpoint()
+	if err != nil {
+		return "", err
+	}
+	l := strings.Split(endpoint, "://")
+	if len(l) > 0 {
+		return l[1], nil
+	}
+	return endpoint, nil
+}
+
 // SecretKey returns the secret key to encrypt the password of target
 func SecretKey() (string, error) {
 	return keyProvider.Get(nil)
@@ -174,6 +188,12 @@ func InternalTokenServiceEndpoint() string {
 	return "http://ui/service/token"
 }
 
+// InternalNotaryEndpoint returns notary server endpoint for internal communication between Harbor containers
+// This is currently a conventional value and can be unaccessible when Harbor is not deployed with Notary.
+func InternalNotaryEndpoint() string {
+	return "http://notary-server:4443"
+}
+
 // InitialAdminPassword returns the initial password for administrator
 func InitialAdminPassword() (string, error) {
 	cfg, err := mg.Get()
@@ -253,3 +273,32 @@ func UISecret() string {
 func JobserviceSecret() string {
 	return os.Getenv("JOBSERVICE_SECRET")
 }
+
+// WithNotary returns a bool value to indicate if Harbor's deployed with Notary
+func WithNotary() bool {
+	cfg, err := mg.Get()
+	if err != nil {
+		log.Errorf("Failed to get configuration, will return WithNotary == false")
+		return false
+	}
+	return cfg[comcfg.WithNotary].(bool)
+}
+
+// AdmiralEndpoint returns the URL of admiral, if Harbor is not deployed with admiral it should return an empty string.
+func AdmiralEndpoint() string {
+	cfg, err := mg.Get()
+	if err != nil {
+		log.Errorf("Failed to get configuration, will return empty string as admiral's endpoint")
+
+		return ""
+	}
+	if e, ok := cfg[comcfg.AdmiralEndpoint].(string); !ok || e == "NA" {
+		cfg[comcfg.AdmiralEndpoint] = ""
+	}
+	return cfg[comcfg.AdmiralEndpoint].(string)
+}
+
+// WithAdmiral returns a bool to indicate if Harbor's deployed with admiral.
+func WithAdmiral() bool {
+	return len(AdmiralEndpoint()) > 0
+}

src/ui/config/config_test.go

@@ -120,4 +120,24 @@ func TestConfig(t *testing.T) {
 	if _, err := Database(); err != nil {
 		t.Fatalf("failed to get database: %v", err)
 	}
+	if InternalNotaryEndpoint() != "http://notary-server:4443" {
+		t.Errorf("Unexpected notary endpoint: %s", InternalNotaryEndpoint())
+	}
+	if WithNotary() {
+		t.Errorf("Withnotary should be false")
+	}
+	if !WithAdmiral() {
+		t.Errorf("WithAdmiral should be true")
+	}
+	if AdmiralEndpoint() != "http://www.vmware.com" {
+		t.Errorf("Unexpected admiral endpoint: %s", AdmiralEndpoint())
+	}
+
+	extURL, err := ExtURL()
+	if err != nil {
+		t.Errorf("Unexpected error getting external URL: %v", err)
+	}
+	if extURL != "host01.com" {
+		t.Errorf(`extURL should be "host01.com".`)
+	}
 }

src/ui/router.go

@@ -92,6 +92,7 @@ func initRouters() {
 	beego.Router("/api/logs", &api.LogAPI{})
 	beego.Router("/api/configurations", &api.ConfigAPI{})
 
+	beego.Router("/api/systeminfo", &api.SystemInfoAPI{}, "get:GetGeneralInfo")
 	beego.Router("/api/systeminfo/volumes", &api.SystemInfoAPI{}, "get:GetVolumeInfo")
 	beego.Router("/api/systeminfo/getcert", &api.SystemInfoAPI{}, "get:GetCert")
 	beego.Router("/api/ldap/ping", &api.LdapAPI{}, "post:Ping")

src/ui/service/token/creator.go

@@ -43,14 +43,14 @@ func InitCreators() {
 		},
 		"registry": &registryFilter{},
 	}
-	ext, err := config.ExtEndpoint()
+	ext, err := config.ExtURL()
 	if err != nil {
-		log.Warningf("Failed to get ext enpoint, err: %v, the token service will not be functional with notary requests", err)
+		log.Warningf("Failed to get ext url, err: %v, the token service will not be functional with notary requests", err)
 	} else {
 		notaryFilterMap = map[string]accessFilter{
 			"repository": &repositoryFilter{
 				parser: &endpointParser{
-					endpoint: strings.Split(ext, "//")[1],
+					endpoint: ext,
 				},
 			},
 		}