Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2025-02-23 15:21:35 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1310 from ywk253100/161228_config

Browse Source 
configure Harbor
This commit is contained in:
Daniel Jiang 2017-01-22 18:22:20 +08:00 committed by GitHub
commit e271d6ba8a
72 changed files with 2738 additions and 928 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,8 +1,11 @@
harbor
make/common/config/*
make/dev/adminserver/harbor_adminserver
make/dev/ui/harbor_ui
make/dev/jobservice/harbor_jobservice
src/adminserver/adminserver
src/ui/ui
src/jobservice/jobservice
src/common/dao/dao.test
*.pyc
jobservice/test

11
.travis.yml
View File

@ -13,14 +13,13 @@ services:
dist: trusty
env:
DB_HOST: 127.0.0.1
DB_PORT: 3306
DB_USR: root
DB_PWD: root123
MYSQL_HOST: localhost
MYSQL_PORT: 3306
MYSQL_USR: root
MYSQL_PWD: root123
MYSQL_DATABASE: registry
SQLITE_FILE: /tmp/registry.db
ADMIN_SERVER_URL: http://127.0.0.1:8888
DOCKER_COMPOSE_VERSION: 1.7.1
HARBOR_ADMIN: admin
HARBOR_ADMIN_PASSWD: Harbor12345
@ -70,7 +69,7 @@ install:
before_script:
# create tables and load data
# - mysql < ./make/db/registry.sql -uroot --verbose
- sudo sqlite3 /registry.db < make/common/db/registry_sqlite.sql
- sudo sqlite3 /tmp/registry.db < make/common/db/registry_sqlite.sql
script:
- sudo mkdir -p /harbor_storage/ca_download
@ -88,7 +87,7 @@ script:
- goveralls -coverprofile=profile.cov -service=travis-ci
- docker-compose -f make/docker-compose.test.yml down
- sudo make/prepare
- docker-compose -f make/dev/docker-compose.yml up -d
- docker ps

38
make/common/templates/adminserver/env Normal file
View File

@ -0,0 +1,38 @@
LOG_LEVEL=debug
EXT_ENDPOINT=$ui_url
AUTH_MODE=$auth_mode
SELF_REGISTRATION=$self_registration
LDAP_URL=$ldap_url
LDAP_SEARCH_DN=$ldap_searchdn
LDAP_SEARCH_PWD=$ldap_search_pwd
LDAP_BASE_DN=$ldap_basedn
LDAP_FILTER=$ldap_filter
LDAP_UID=$ldap_uid
LDAP_SCOPE=$ldap_scope
LDAP_TIMEOUT=$ldap_timeout
DATABASE_TYPE=mysql
MYSQL_HOST=mysql
MYSQL_PORT=3306
MYSQL_USR=root
MYSQL_PWD=$db_password
MYSQL_DATABASE=registry
REGISTRY_URL=http://registry:5000
TOKEN_SERVICE_URL=http://ui/service/token
EMAIL_HOST=$email_host
EMAIL_PORT=$email_port
EMAIL_USR=$email_usr
EMAIL_PWD=$email_pwd
EMAIL_SSL=$email_ssl
EMAIL_FROM=$email_from
EMAIL_IDENTITY=$email_identity
HARBOR_ADMIN_PASSWORD=$harbor_admin_password
PROJECT_CREATION_RESTRICTION=$project_creation_restriction
VERIFY_REMOTE_CERT=$verify_remote_cert
MAX_JOB_WORKERS=$max_job_workers
LOG_DIR=/var/log/jobs
UI_SECRET=$ui_secret
SECRET_KEY=$secret_key
TOKEN_EXPIRATION=$token_expiration
CFG_EXPIRATION=5
USE_COMPRESSED_JS=$use_compressed_js
GODEBUG=netdns=cgo

15
make/common/templates/jobservice/env
View File

@ -1,15 +1,4 @@
MYSQL_HOST=mysql
MYSQL_PORT=3306
MYSQL_USR=root
MYSQL_PWD=$db_password
UI_SECRET=$ui_secret
SECRET_KEY=$secret_key
CONFIG_PATH=/etc/jobservice/app.conf
REGISTRY_URL=http://registry:5000
VERIFY_REMOTE_CERT=$verify_remote_cert
MAX_JOB_WORKERS=$max_job_workers
LOG_LEVEL=debug
LOG_DIR=/var/log/jobs
CONFIG_PATH=/etc/jobservice/app.conf
UI_SECRET=$ui_secret
GODEBUG=netdns=cgo
EXT_ENDPOINT=$ui_url
TOKEN_ENDPOINT=http://ui

9
make/common/templates/ui/app.conf
View File

@ -7,12 +7,3 @@ names = en-US|zh-CN
[dev]
httpport = 80
[mail]
identity = $email_identity
host = $email_server
port = $email_server_port
username = $email_username
password = $email_password
from = $email_from
ssl = $email_ssl

30
make/common/templates/ui/env
View File

@ -1,30 +1,4 @@
MYSQL_HOST=mysql
MYSQL_PORT=3306
MYSQL_USR=root
MYSQL_PWD=$db_password
REGISTRY_URL=http://registry:5000
JOB_SERVICE_URL=http://jobservice
UI_URL=http://ui
CONFIG_PATH=/etc/ui/app.conf
EXT_REG_URL=$hostname
HARBOR_ADMIN_PASSWORD=$harbor_admin_password
AUTH_MODE=$auth_mode
LDAP_URL=$ldap_url
LDAP_SEARCH_DN=$ldap_searchdn
LDAP_SEARCH_PWD=$ldap_search_pwd
LDAP_BASE_DN=$ldap_basedn
LDAP_FILTER=$ldap_filter
LDAP_UID=$ldap_uid
LDAP_SCOPE=$ldap_scope
LDAP_CONNECT_TIMEOUT=$ldap_connect_timeout
UI_SECRET=$ui_secret
SECRET_KEY=$secret_key
SELF_REGISTRATION=$self_registration
USE_COMPRESSED_JS=$use_compressed_js
LOG_LEVEL=debug
CONFIG_PATH=/etc/ui/app.conf
UI_SECRET=$ui_secret
GODEBUG=netdns=cgo
EXT_ENDPOINT=$ui_url
TOKEN_ENDPOINT=http://ui
VERIFY_REMOTE_CERT=$verify_remote_cert
TOKEN_EXPIRATION=$token_expiration
PROJECT_CREATION_RESTRICTION=$project_creation_restriction

12
make/dev/adminserver/Dockerfile Normal file
View File

@ -0,0 +1,12 @@
FROM golang:1.7.3
MAINTAINER yinw@vmware.com
COPY . /go/src/github.com/vmware/harbor
WORKDIR /go/src/github.com/vmware/harbor/src/adminserver
RUN go build -v -a -o /go/bin/harbor_adminserver \
&& chmod u+x /go/bin/harbor_adminserver
WORKDIR /go/bin/
ENTRYPOINT ["/go/bin/harbor_adminserver"]

19
make/dev/docker-compose.yml
View File

@ -40,6 +40,22 @@ services:
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "mysql"
adminserver:
build:
context: ../../
dockerfile: make/dev/adminserver/Dockerfile
env_file:
- ../common/config/adminserver/env
restart: always
volumes:
- /data/config/:/etc/harbor/
depends_on:
- log
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "adminserver"
ui:
build:
context: ../../
@ -52,6 +68,8 @@ services:
- ../common/config/ui/private_key.pem:/etc/ui/private_key.pem
depends_on:
- log
- adminserver
- registry
logging:
driver: "syslog"
options:
@ -69,6 +87,7 @@ services:
- ../common/config/jobservice/app.conf:/etc/jobservice/app.conf
depends_on:
- ui
- adminserver
logging:
driver: "syslog"
options:

18
make/docker-compose.tpl
View File

@ -41,6 +41,21 @@ services:
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "mysql"
adminserver:
image: vmware/harbor-adminserver
container_name: harbor-adminserver
env_file:
- ./common/config/adminserver/env
restart: always
volumes:
- /data/config/:/etc/harbor/
depends_on:
- log
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "adminserver"
ui:
image: vmware/harbor-ui
container_name: harbor-ui
@ -53,6 +68,8 @@ services:
- /data:/harbor_storage
depends_on:
- log
- adminserver
- registry
logging:
driver: "syslog"
options:
@ -69,6 +86,7 @@ services:
- ./common/config/jobservice/app.conf:/etc/jobservice/app.conf
depends_on:
- ui
- adminserver
logging:
driver: "syslog"
options:

2
make/harbor.cfg
View File

@ -53,7 +53,7 @@ ldap_uid = uid
ldap_scope = 3
#Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds.
ldap_connect_timeout = 5
ldap_timeout = 5
#The password for the root user of mysql db, change this before any production use.
db_password = root123

8
make/photon/adminserver/Dockerfile Normal file
View File

@ -0,0 +1,8 @@
FROM library/photon:1.0
RUN mkdir /harbor/
COPY ./make/dev/adminserver/harbor_adminserver /harbor/
RUN chmod u+x /harbor/harbor_adminserver
WORKDIR /harbor/
ENTRYPOINT ["/harbor/harbor_adminserver"]

72
make/prepare
View File

@ -77,10 +77,10 @@ hostname = rcp.get("configuration", "hostname")
protocol = rcp.get("configuration", "ui_url_protocol")
ui_url = protocol + "://" + hostname
email_identity = rcp.get("configuration", "email_identity")
email_server = rcp.get("configuration", "email_server")
email_server_port = rcp.get("configuration", "email_server_port")
email_username = rcp.get("configuration", "email_username")
email_password = rcp.get("configuration", "email_password")
email_host = rcp.get("configuration", "email_server")
email_port = rcp.get("configuration", "email_server_port")
email_usr = rcp.get("configuration", "email_username")
email_pwd = rcp.get("configuration", "email_password")
email_from = rcp.get("configuration", "email_from")
email_ssl = rcp.get("configuration", "email_ssl")
harbor_admin_password = rcp.get("configuration", "harbor_admin_password")
@ -101,7 +101,7 @@ else:
ldap_filter = ""
ldap_uid = rcp.get("configuration", "ldap_uid")
ldap_scope = rcp.get("configuration", "ldap_scope")
ldap_connect_timeout = rcp.get("configuration", "ldap_connect_timeout")
ldap_timeout = rcp.get("configuration", "ldap_timeout")
db_password = rcp.get("configuration", "db_password")
self_registration = rcp.get("configuration", "self_registration")
use_compressed_js = rcp.get("configuration", "use_compressed_js")
@ -126,6 +126,10 @@ secret_key = get_secret_key(secretkey_path)
ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))
adminserver_config_dir = os.path.join(config_dir,"adminserver")
if not os.path.exists(adminserver_config_dir):
os.makedirs(os.path.join(config_dir, "adminserver"))
ui_config_dir = os.path.join(config_dir,"ui")
if not os.path.exists(ui_config_dir):
os.makedirs(os.path.join(config_dir, "ui"))
@ -152,6 +156,7 @@ def render(src, dest, **kw):
f.write(t.substitute(**kw))
print("Generated configuration file: %s" % dest)
adminserver_conf_env = os.path.join(config_dir, "adminserver", "env")
ui_conf_env = os.path.join(config_dir, "ui", "env")
ui_conf = os.path.join(config_dir, "ui", "app.conf")
jobservice_conf = os.path.join(config_dir, "jobservice", "app.conf")
@ -188,13 +193,11 @@ else:
render(os.path.join(templates_dir, "nginx", "nginx.http.conf"),
nginx_conf)
render(os.path.join(templates_dir, "ui", "env"),
ui_conf_env,
hostname=hostname,
db_password=db_password,
render(os.path.join(templates_dir, "adminserver", "env"),
adminserver_conf_env,
ui_url=ui_url,
auth_mode=auth_mode,
harbor_admin_password=harbor_admin_password,
self_registration=self_registration,
ldap_url=ldap_url,
ldap_searchdn =ldap_searchdn,
ldap_search_pwd =ldap_search_pwd,
@ -202,27 +205,31 @@ render(os.path.join(templates_dir, "ui", "env"),
ldap_filter=ldap_filter,
ldap_uid=ldap_uid,
ldap_scope=ldap_scope,
ldap_connect_timeout=ldap_connect_timeout,
self_registration=self_registration,
use_compressed_js=use_compressed_js,
ldap_timeout=ldap_timeout,
db_password=db_password,
email_host=email_host,
email_port=email_port,
email_usr=email_usr,
email_pwd=email_pwd,
email_ssl=email_ssl,
email_from=email_from,
email_identity=email_identity,
harbor_admin_password=harbor_admin_password,
project_creation_restriction=proj_cre_restriction,
verify_remote_cert=verify_remote_cert,
max_job_workers=max_job_workers,
ui_secret=ui_secret,
secret_key=secret_key,
verify_remote_cert=verify_remote_cert,
project_creation_restriction=proj_cre_restriction,
token_expiration=token_expiration)
token_expiration=token_expiration,
use_compressed_js=use_compressed_js
)
render(os.path.join(templates_dir, "ui", "app.conf"),
ui_conf,
email_identity=email_identity,
email_server=email_server,
email_server_port=email_server_port,
email_username=email_username,
email_password=email_password,
email_from=email_from,
email_ssl=email_ssl,
ui_url=ui_url)
render(os.path.join(templates_dir, "ui", "env"),
ui_conf_env,
ui_secret=ui_secret)
render(os.path.join(templates_dir, "registry", "config.yml"),
render(os.path.join(templates_dir, "registry",
"config.yml"),
registry_conf,
ui_url=ui_url)
@ -232,16 +239,15 @@ render(os.path.join(templates_dir, "db", "env"),
render(os.path.join(templates_dir, "jobservice", "env"),
job_conf_env,
db_password=db_password,
ui_secret=ui_secret,
max_job_workers=max_job_workers,
secret_key=secret_key,
ui_url=ui_url,
verify_remote_cert=verify_remote_cert)
ui_secret=ui_secret)
print("Generated configuration file: %s" % jobservice_conf)
shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf)
print("Generated configuration file: %s" % ui_conf)
shutil.copyfile(os.path.join(templates_dir, "ui", "app.conf"), ui_conf)
def validate_crt_subj(dirty_subj):
subj_list = [item for item in dirty_subj.strip().split("/") \
if len(item.split("=")) == 2 and len(item.split("=")[1]) > 0]

34
src/adminserver/api/base.go Normal file
View File

@ -0,0 +1,34 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package api
import (
"net/http"
)
func handleInternalServerError(w http.ResponseWriter) {
http.Error(w, http.StatusText(http.StatusInternalServerError),
http.StatusInternalServerError)
}
func handleBadRequestError(w http.ResponseWriter, error string) {
http.Error(w, error, http.StatusBadRequest)
}
func handleUnauthorized(w http.ResponseWriter) {
http.Error(w, http.StatusText(http.StatusUnauthorized),
http.StatusUnauthorized)
}

193
src/adminserver/api/cfg.go Normal file
View File

@ -0,0 +1,193 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package api
import (
"encoding/json"
"io/ioutil"
"net/http"
"os"
"strconv"
cfg "github.com/vmware/harbor/src/adminserver/systemcfg"
comcfg "github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
)
func isAuthenticated(r *http.Request) (bool, error) {
secret := os.Getenv("UI_SECRET")
c, err := r.Cookie("secret")
if err != nil {
if err == http.ErrNoCookie {
return false, nil
}
return false, err
}
return c != nil && c.Value == secret, nil
}
// ListCfgs lists configurations
func ListCfgs(w http.ResponseWriter, r *http.Request) {
authenticated, err := isAuthenticated(r)
if err != nil {
log.Errorf("failed to check whether the request is authenticated or not: %v", err)
handleInternalServerError(w)
return
}
if !authenticated {
handleUnauthorized(w)
return
}
cfg, err := cfg.GetSystemCfg()
if err != nil {
log.Errorf("failed to get system configurations: %v", err)
handleInternalServerError(w)
return
}
b, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
log.Errorf("failed to marshal configurations: %v", err)
handleInternalServerError(w)
return
}
if _, err = w.Write(b); err != nil {
log.Errorf("failed to write response: %v", err)
}
}
// UpdateCfgs updates configurations
func UpdateCfgs(w http.ResponseWriter, r *http.Request) {
authenticated, err := isAuthenticated(r)
if err != nil {
log.Errorf("failed to check whether the request is authenticated or not: %v", err)
handleInternalServerError(w)
return
}
if !authenticated {
handleUnauthorized(w)
return
}
b, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Errorf("failed to read request body: %v", err)
handleInternalServerError(w)
return
}
m := &map[string]string{}
if err = json.Unmarshal(b, m); err != nil {
handleBadRequestError(w, err.Error())
return
}
system, err := cfg.GetSystemCfg()
if err != nil {
handleInternalServerError(w)
return
}
if err := populate(system, *m); err != nil {
log.Errorf("failed to populate system configurations: %v", err)
handleInternalServerError(w)
return
}
if err = cfg.UpdateSystemCfg(system); err != nil {
log.Errorf("failed to update system configurations: %v", err)
handleInternalServerError(w)
return
}
}
// populate attrs of cfg according to m
func populate(cfg *models.SystemCfg, m map[string]string) error {
if mode, ok := m[comcfg.AUTHMode]; ok {
cfg.Authentication.Mode = mode
}
if value, ok := m[comcfg.SelfRegistration]; ok {
cfg.Authentication.SelfRegistration = value == "1"
}
if url, ok := m[comcfg.LDAPURL]; ok {
cfg.Authentication.LDAP.URL = url
}
if dn, ok := m[comcfg.LDAPSearchDN]; ok {
cfg.Authentication.LDAP.SearchDN = dn
}
if pwd, ok := m[comcfg.LDAPSearchPwd]; ok {
cfg.Authentication.LDAP.SearchPwd = pwd
}
if dn, ok := m[comcfg.LDAPBaseDN]; ok {
cfg.Authentication.LDAP.BaseDN = dn
}
if uid, ok := m[comcfg.LDAPUID]; ok {
cfg.Authentication.LDAP.UID = uid
}
if filter, ok := m[comcfg.LDAPFilter]; ok {
cfg.Authentication.LDAP.Filter = filter
}
if scope, ok := m[comcfg.LDAPScope]; ok {
i, err := strconv.Atoi(scope)
if err != nil {
return err
}
cfg.Authentication.LDAP.Scope = i
}
if timeout, ok := m[comcfg.LDAPTimeout]; ok {
i, err := strconv.Atoi(timeout)
if err != nil {
return err
}
cfg.Authentication.LDAP.Timeout = i
}
if value, ok := m[comcfg.EmailHost]; ok {
cfg.Email.Host = value
}
if value, ok := m[comcfg.EmailPort]; ok {
cfg.Email.Port = value
}
if value, ok := m[comcfg.EmailUsername]; ok {
cfg.Email.Username = value
}
if value, ok := m[comcfg.EmailPassword]; ok {
cfg.Email.Password = value
}
if value, ok := m[comcfg.EmailSSL]; ok {
cfg.Email.SSL = value == "1"
}
if value, ok := m[comcfg.EmailFrom]; ok {
cfg.Email.From = value
}
if value, ok := m[comcfg.EmailIdentity]; ok {
cfg.Email.Identity = value
}
if value, ok := m[comcfg.ProjectCreationRestriction]; ok {
cfg.ProjectCreationRestriction = value
}
if value, ok := m[comcfg.VerifyRemoteCert]; ok {
cfg.VerifyRemoteCert = value == "1"
}
return nil
}

60
src/adminserver/main.go Normal file
View File

@ -0,0 +1,60 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"net/http"
"os"
cfg "github.com/vmware/harbor/src/adminserver/systemcfg"
"github.com/vmware/harbor/src/common/utils/log"
)
// Server for admin component
type Server struct {
Port string
Handler http.Handler
}
// Serve the API
func (s *Server) Serve() error {
server := &http.Server{
Addr: ":" + s.Port,
Handler: s.Handler,
}
return server.ListenAndServe()
}
func main() {
log.Info("initializing system configurations...")
if err := cfg.Init(); err != nil {
log.Fatalf("failed to initialize the system: %v", err)
}
log.Info("system initialization completed")
port := os.Getenv("PORT")
if len(port) == 0 {
port = "80"
}
server := &Server{
Port: port,
Handler: newHandler(),
}
if err := server.Serve(); err != nil {
log.Fatal(err)
}
}

30
src/adminserver/router.go Normal file
View File

@ -0,0 +1,30 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"net/http"
"github.com/gorilla/mux"
"github.com/vmware/harbor/src/adminserver/api"
)
func newHandler() http.Handler {
r := mux.NewRouter()
r.HandleFunc("/api/configurations", api.ListCfgs).Methods("GET")
r.HandleFunc("/api/configurations", api.UpdateCfgs).Methods("PUT")
return r
}

30
src/adminserver/systemcfg/store/driver.go Normal file
View File

@ -0,0 +1,30 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package store
import (
"github.com/vmware/harbor/src/common/models"
)
// Driver defines methods that a configuration store driver must implement
type Driver interface {
// Name returns a human-readable name of the driver
Name() string
// Read reads the configurations from store
Read() (*models.SystemCfg, error)
// Write writes the configurations to store
Write(*models.SystemCfg) error
}

108
src/adminserver/systemcfg/store/json/driver_json.go Normal file
View File

@ -0,0 +1,108 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package json
import (
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/vmware/harbor/src/adminserver/systemcfg/store"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
)
const (
// the default path of configuration file
defaultPath = "/etc/harbor/config.json"
)
type cfgStore struct {
path string // the path of cfg file
sync.RWMutex
}
// NewCfgStore returns an instance of cfgStore that stores the configurations
// in a json file. The file will be created if it does not exist.
func NewCfgStore(path ...string) (store.Driver, error) {
p := defaultPath
if len(path) > 0 && len(path[0]) > 0 {
p = path[0]
}
log.Debugf("path of configuration file: %s", p)
if _, err := os.Stat(p); os.IsNotExist(err) {
log.Infof("the configuration file %s does not exist, creating it...", p)
if err = os.MkdirAll(filepath.Dir(p), 0600); err != nil {
return nil, err
}
if err = ioutil.WriteFile(p, []byte{}, 0600); err != nil {
return nil, err
}
}
return &cfgStore{
path: p,
}, nil
}
// Name ...
func (c *cfgStore) Name() string {
return "JSON"
}
// Read ...
func (c *cfgStore) Read() (*models.SystemCfg, error) {
c.RLock()
defer c.RUnlock()
b, err := ioutil.ReadFile(c.path)
if err != nil {
return nil, err
}
// empty file
if len(b) == 0 {
return nil, nil
}
config := &models.SystemCfg{}
if err = json.Unmarshal(b, config); err != nil {
return nil, err
}
return config, nil
}
// Write ...
func (c *cfgStore) Write(config *models.SystemCfg) error {
b, err := json.MarshalIndent(config, "", " ")
if err != nil {
return err
}
c.Lock()
defer c.Unlock()
if err = ioutil.WriteFile(c.path, b, 0600); err != nil {
return err
}
return nil
}

59
src/adminserver/systemcfg/store/json/driver_json_test.go Normal file
View File

@ -0,0 +1,59 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package json
import (
"os"
"testing"
"github.com/vmware/harbor/src/common/models"
)
func TestReadWrite(t *testing.T) {
path := "/tmp/config.json"
store, err := NewCfgStore(path)
if err != nil {
t.Fatalf("failed to create json cfg store: %v", err)
}
defer func() {
if err := os.Remove(path); err != nil {
t.Fatalf("failed to remove the json file %s: %v", path, err)
}
}()
if store.Name() != "JSON" {
t.Errorf("unexpected name: %s != %s", store.Name(), "JSON")
return
}
config := &models.SystemCfg{
Authentication: &models.Authentication{
LDAP: &models.LDAP{},
},
Database: &models.Database{
MySQL: &models.MySQL{},
},
}
if err := store.Write(config); err != nil {
t.Errorf("failed to write configurations to json file: %v", err)
return
}
if _, err = store.Read(); err != nil {
t.Errorf("failed to read configurations from json file: %v", err)
return
}
}

188
src/adminserver/systemcfg/systemcfg.go Normal file
View File

@ -0,0 +1,188 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package systemcfg
import (
"fmt"
"os"
"strconv"
"github.com/vmware/harbor/src/adminserver/systemcfg/store"
"github.com/vmware/harbor/src/adminserver/systemcfg/store/json"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
)
var cfgStore store.Driver
// Init system configurations. Read from config store first, if null read from env
func Init() (err error) {
s := getCfgStore()
switch s {
case "json":
path := os.Getenv("JSON_STORE_PATH")
cfgStore, err = json.NewCfgStore(path)
if err != nil {
return
}
default:
return fmt.Errorf("unsupported configuration store driver %s", s)
}
log.Infof("configuration store driver: %s", cfgStore.Name())
cfg, err := cfgStore.Read()
if err != nil {
return err
}
if cfg == nil {
log.Info("configurations read from store driver are null, initializing system from environment variables...")
cfg, err = initFromEnv()
if err != nil {
return err
}
} else {
if err := readFromEnv(cfg); err != nil {
return err
}
}
//sync configurations into cfg store
if err = cfgStore.Write(cfg); err != nil {
return err
}
return nil
}
func getCfgStore() string {
t := os.Getenv("CFG_STORE_TYPE")
if len(t) == 0 {
t = "json"
}
return t
}
//read the following attrs from env every time boots up
func readFromEnv(cfg *models.SystemCfg) error {
cfg.DomainName = os.Getenv("EXT_ENDPOINT")
cfg.Database = &models.Database{
Type: os.Getenv("DATABASE_TYPE"),
MySQL: &models.MySQL{
Host: os.Getenv("MYSQL_HOST"),
Username: os.Getenv("MYSQL_USR"),
Password: os.Getenv("MYSQL_PWD"),
Database: os.Getenv("MYSQL_DATABASE"),
},
SQLite: &models.SQLite{
File: os.Getenv("SQLITE_FILE"),
},
}
port, err := strconv.Atoi(os.Getenv("MYSQL_PORT"))
if err != nil {
return err
}
cfg.Database.MySQL.Port = port
cfg.TokenService = &models.TokenService{
URL: os.Getenv("TOKEN_SERVICE_URL"),
}
cfg.Registry = &models.Registry{
URL: os.Getenv("REGISTRY_URL"),
}
//TODO remove
cfg.JobLogDir = os.Getenv("LOG_DIR")
//TODO remove
cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on"
exp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION"))
if err != nil {
return err
}
cfg.TokenExpiration = exp
cfg.SecretKey = os.Getenv("SECRET_KEY")
cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION"))
if err != nil {
return err
}
cfg.CfgExpiration = cfgExp
workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS"))
if err != nil {
return err
}
cfg.MaxJobWorkers = workers
return nil
}
func initFromEnv() (*models.SystemCfg, error) {
cfg := &models.SystemCfg{}
if err := readFromEnv(cfg); err != nil {
return nil, err
}
cfg.Authentication = &models.Authentication{
Mode: os.Getenv("AUTH_MODE"),
SelfRegistration: os.Getenv("SELF_REGISTRATION") == "on",
LDAP: &models.LDAP{
URL: os.Getenv("LDAP_URL"),
SearchDN: os.Getenv("LDAP_SEARCH_DN"),
SearchPwd: os.Getenv("LDAP_SEARCH_PWD"),
BaseDN: os.Getenv("LDAP_BASE_DN"),
Filter: os.Getenv("LDAP_FILTER"),
UID: os.Getenv("LDAP_UID"),
},
}
scope, err := strconv.Atoi(os.Getenv("LDAP_SCOPE"))
if err != nil {
return nil, err
}
cfg.Authentication.LDAP.Scope = scope
timeout, err := strconv.Atoi(os.Getenv("LDAP_TIMEOUT"))
if err != nil {
return nil, err
}
cfg.Authentication.LDAP.Timeout = timeout
cfg.Email = &models.Email{
Host: os.Getenv("EMAIL_HOST"),
Port: os.Getenv("EMAIL_PORT"),
Username: os.Getenv("EMAIL_USR"),
Password: os.Getenv("EMAIL_PWD"),
SSL: os.Getenv("EMAIL_SSL") == "true",
From: os.Getenv("EMAIL_FROM"),
Identity: os.Getenv("EMAIL_IDENTITY"),
}
cfg.VerifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") == "on"
cfg.ProjectCreationRestriction = os.Getenv("PROJECT_CREATION_RESTRICTION")
cfg.InitialAdminPwd = os.Getenv("HARBOR_ADMIN_PASSWORD")
return cfg, nil
}
// GetSystemCfg returns the system configurations
func GetSystemCfg() (*models.SystemCfg, error) {
return cfgStore.Read()
}
// UpdateSystemCfg updates the system configurations
func UpdateSystemCfg(cfg *models.SystemCfg) error {
return cfgStore.Write(cfg)
}

103
src/adminserver/systemcfg/systemcfg_test.go Normal file
View File

@ -0,0 +1,103 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package systemcfg
import (
"os"
"testing"
comcfg "github.com/vmware/harbor/src/common/config"
)
// test functions under adminserver/systemcfg
func TestSystemcfg(t *testing.T) {
key := "JSON_STORE_PATH"
path := "/tmp/config.json"
if _, err := os.Stat(path); err == nil {
if err := os.Remove(path); err != nil {
t.Fatalf("failed to remove %s: %v", path, err)
}
} else if !os.IsNotExist(err) {
t.Fatalf("failed to check the existence of %s: %v", path, err)
}
if err := os.Setenv(key, path); err != nil {
t.Fatalf("failed to set env %s: %v", key, err)
}
m := map[string]string{
"AUTH_MODE": comcfg.DBAuth,
"LDAP_SCOPE": "1",
"LDAP_TIMEOUT": "30",
"MYSQL_PORT": "3306",
"MAX_JOB_WORKERS": "3",
"TOKEN_EXPIRATION": "30",
"CFG_EXPIRATION": "5",
}
for k, v := range m {
if err := os.Setenv(k, v); err != nil {
t.Fatalf("failed to set env %s: %v", k, err)
}
}
if err := Init(); err != nil {
t.Errorf("failed to initialize system configurations: %v", err)
return
}
defer func() {
if err := os.Remove(path); err != nil {
t.Fatalf("failed to remove %s: %v", path, err)
}
}()
// run Init again to make sure it works well when the configuration file
// already exists
if err := Init(); err != nil {
t.Errorf("failed to initialize system configurations: %v", err)
return
}
cfg, err := GetSystemCfg()
if err != nil {
t.Errorf("failed to get system configurations: %v", err)
return
}
if cfg.Authentication.Mode != comcfg.DBAuth {
t.Errorf("unexpected auth mode: %s != %s",
cfg.Authentication.Mode, comcfg.DBAuth)
return
}
cfg.Authentication.Mode = comcfg.LDAPAuth
if err = UpdateSystemCfg(cfg); err != nil {
t.Errorf("failed to update system configurations: %v", err)
return
}
cfg, err = GetSystemCfg()
if err != nil {
t.Errorf("failed to get system configurations: %v", err)
return
}
if cfg.Authentication.Mode != comcfg.LDAPAuth {
t.Errorf("unexpected auth mode: %s != %s",
cfg.Authentication.Mode, comcfg.DBAuth)
return
}
}

6
src/common/api/base.go
View File

@ -22,7 +22,6 @@ import (
"strconv"
"github.com/astaxie/beego/validation"
"github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
@ -210,8 +209,3 @@ func (b *BaseAPI) GetPaginationParams() (page, pageSize int64) {
return page, pageSize
}
// GetIsInsecure ...
func GetIsInsecure() bool {
return !config.VerifyRemoteCert()
}

18
src/common/api/base_test.go
View File

@ -13,21 +13,3 @@
limitations under the License.
*/
package api
import (
"github.com/vmware/harbor/src/common/config"
"os"
"testing"
)
func TestGetIsInsecure(t *testing.T) {
os.Setenv("VERIFY_REMOTE_CERT", "off")
err := config.Reload()
if err != nil {
t.Errorf("Failed to load config, error: %v", err)
}
if !GetIsInsecure() {
t.Errorf("GetIsInsecure() should be true when VERIFY_REMOTE_CERT is off, in fact: false")
}
os.Unsetenv("VERIFY_REMOTE_CERT")
}

326
src/common/config/config.go
View File

@ -17,162 +17,218 @@
package config
import (
"bytes"
"encoding/json"
"fmt"
"os"
"io/ioutil"
"net/http"
"strings"
"time"
"github.com/astaxie/beego/cache"
"github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
)
// ConfLoader is the interface to load configurations
type ConfLoader interface {
// Load will load configuration from different source into a string map, the values in the map will be parsed in to configurations.
Load() (map[string]string, error)
// const variables
const (
DBAuth = "db_auth"
LDAPAuth = "ldap_auth"
ProCrtRestrEveryone = "everyone"
ProCrtRestrAdmOnly = "adminonly"
LDAPScopeBase = "1"
LDAPScopeOnelevel = "2"
LDAPScopeSubtree = "3"
AUTHMode = "auth_mode"
SelfRegistration = "self_registration"
LDAPURL = "ldap_url"
LDAPSearchDN = "ldap_search_dn"
LDAPSearchPwd = "ldap_search_password"
LDAPBaseDN = "ldap_base_dn"
LDAPUID = "ldap_uid"
LDAPFilter = "ldap_filter"
LDAPScope = "ldap_scope"
LDAPTimeout = "ldap_timeout"
EmailHost = "email_host"
EmailPort = "email_port"
EmailUsername = "email_username"
EmailPassword = "email_password"
EmailFrom = "email_from"
EmailSSL = "email_ssl"
EmailIdentity = "email_identity"
ProjectCreationRestriction = "project_creation_restriction"
VerifyRemoteCert = "verify_remote_cert"
MaxJobWorkers = "max_job_workers"
CfgExpiration = "cfg_expiration"
)
// Manager manages configurations
type Manager struct {
Loader *Loader
Parser Parser
Cache bool
cache cache.Cache
key string
}
// EnvConfigLoader loads the config from env vars.
type EnvConfigLoader struct {
Keys []string
// Parser parses []byte to a specific configuration
type Parser interface {
// Parse ...
Parse([]byte) (interface{}, error)
}
// Load ...
func (ec *EnvConfigLoader) Load() (map[string]string, error) {
m := make(map[string]string)
for _, k := range ec.Keys {
m[k] = os.Getenv(k)
}
return m, nil
// NewManager returns an instance of Manager
// url: the url from which loader loads configurations
func NewManager(url, secret string, parser Parser, enableCache bool) *Manager {
m := &Manager{
Loader: NewLoader(url, secret),
Parser: parser,
}
// ConfParser ...
type ConfParser interface {
//Parse parse the input raw map into a config map
Parse(raw map[string]string, config map[string]interface{}) error
if enableCache {
m.Cache = true
m.cache = cache.NewMemoryCache()
m.key = "cfg"
}
// Config wraps a map for the processed configuration values,
// and loader parser to read configuration from external source and process the values.
type Config struct {
Config map[string]interface{}
Loader ConfLoader
Parser ConfParser
return m
}
// Load reload the configurations
func (conf *Config) Load() error {
rawMap, err := conf.Loader.Load()
// Init loader
func (m *Manager) Init() error {
return m.Loader.Init()
}
// Load configurations, if cache is enabled, cache the configurations
func (m *Manager) Load() (interface{}, error) {
b, err := m.Loader.Load()
if err != nil {
return nil, err
}
c, err := m.Parser.Parse(b)
if err != nil {
return nil, err
}
if m.Cache {
expi, err := parseExpiration(b)
if err != nil {
return nil, err
}
if err = m.cache.Put(m.key, c,
time.Duration(expi)*time.Second); err != nil {
return nil, err
}
}
return c, nil
}
func parseExpiration(b []byte) (int, error) {
expi := &struct {
Expi int `json:"cfg_expiration"`
}{}
if err := json.Unmarshal(b, expi); err != nil {
return 0, err
}
return expi.Expi, nil
}
// Get : if cache is enabled, read configurations from cache,
// if cache is null or cache is disabled it loads configurations directly
func (m *Manager) Get() (interface{}, error) {
if m.Cache {
c := m.cache.Get(m.key)
if c != nil {
return c, nil
}
}
return m.Load()
}
// Upload configurations
func (m *Manager) Upload(b []byte) error {
return m.Loader.Upload(b)
}
// Loader loads and uploads configurations
type Loader struct {
url string
secret string
client *http.Client
}
// NewLoader ...
func NewLoader(url, secret string) *Loader {
return &Loader{
url: url,
secret: secret,
client: &http.Client{},
}
}
// Init waits remote server to be ready by testing connections with it
func (l *Loader) Init() error {
addr := l.url
if strings.Contains(addr, "://") {
addr = strings.Split(addr, "://")[1]
}
if !strings.Contains(addr, ":") {
addr = addr + ":80"
}
return utils.TestTCPConn(addr, 60, 2)
}
// Load configurations from remote server
func (l *Loader) Load() ([]byte, error) {
log.Debug("loading configurations...")
req, err := http.NewRequest("GET", l.url+"/api/configurations", nil)
if err != nil {
return nil, err
}
req.AddCookie(&http.Cookie{
Name: "secret",
Value: l.secret,
})
resp, err := l.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
log.Debug("configurations load completed")
return b, nil
}
// Upload configuratons to remote server
func (l *Loader) Upload(b []byte) error {
req, err := http.NewRequest("PUT", l.url+"/api/configurations", bytes.NewReader(b))
if err != nil {
return err
}
err = conf.Parser.Parse(rawMap, conf.Config)
req.AddCookie(&http.Cookie{
Name: "secret",
Value: l.secret,
})
resp, err := l.client.Do(req)
if err != nil {
return err
}
// MySQLSetting wraps the settings of a MySQL DB
type MySQLSetting struct {
Database string
User string
Password string
Host string
Port string
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected http status code: %d", resp.StatusCode)
}
// SQLiteSetting wraps the settings of a SQLite DB
type SQLiteSetting struct {
FilePath string
}
type commonParser struct{}
// Parse parses the db settings, veryfy_remote_cert, ext_endpoint, token_endpoint
func (cp *commonParser) Parse(raw map[string]string, config map[string]interface{}) error {
db := strings.ToLower(raw["DATABASE"])
if db == "mysql" || db == "" {
db = "mysql"
mySQLDB := raw["MYSQL_DATABASE"]
if len(mySQLDB) == 0 {
mySQLDB = "registry"
}
setting := MySQLSetting{
mySQLDB,
raw["MYSQL_USR"],
raw["MYSQL_PWD"],
raw["MYSQL_HOST"],
raw["MYSQL_PORT"],
}
config["mysql"] = setting
} else if db == "sqlite" {
f := raw["SQLITE_FILE"]
if len(f) == 0 {
f = "registry.db"
}
setting := SQLiteSetting{
f,
}
config["sqlite"] = setting
} else {
return fmt.Errorf("Invalid DB: %s", db)
}
config["database"] = db
//By default it's true
config["verify_remote_cert"] = raw["VERIFY_REMOTE_CERT"] != "off"
config["ext_endpoint"] = raw["EXT_ENDPOINT"]
config["token_endpoint"] = raw["TOKEN_ENDPOINT"]
config["log_level"] = raw["LOG_LEVEL"]
return nil
}
var commonConfig *Config
func init() {
commonKeys := []string{"DATABASE", "MYSQL_DATABASE", "MYSQL_USR", "MYSQL_PWD", "MYSQL_HOST", "MYSQL_PORT", "SQLITE_FILE", "VERIFY_REMOTE_CERT", "EXT_ENDPOINT", "TOKEN_ENDPOINT", "LOG_LEVEL"}
commonConfig = &Config{
Config: make(map[string]interface{}),
Loader: &EnvConfigLoader{Keys: commonKeys},
Parser: &commonParser{},
}
if err := commonConfig.Load(); err != nil {
panic(err)
}
}
// Reload will reload the configuration.
func Reload() error {
return commonConfig.Load()
}
// Database returns the DB type in configuration.
func Database() string {
return commonConfig.Config["database"].(string)
}
// MySQL returns the mysql setting in configuration.
func MySQL() MySQLSetting {
return commonConfig.Config["mysql"].(MySQLSetting)
}
// SQLite returns the SQLite setting
func SQLite() SQLiteSetting {
return commonConfig.Config["sqlite"].(SQLiteSetting)
}
// VerifyRemoteCert returns bool value.
func VerifyRemoteCert() bool {
return commonConfig.Config["verify_remote_cert"].(bool)
}
// ExtEndpoint ...
func ExtEndpoint() string {
return commonConfig.Config["ext_endpoint"].(string)
}
// TokenEndpoint returns the endpoint string of token service, which can be accessed by internal service of Harbor.
func TokenEndpoint() string {
return commonConfig.Config["token_endpoint"].(string)
}
// LogLevel returns the log level in string format.
func LogLevel() string {
return commonConfig.Config["log_level"].(string)
}

96
src/common/config/config_test.go
View File

@ -13,99 +13,3 @@
limitations under the License.
*/
package config
import (
"os"
"testing"
)
func TestEnvConfLoader(t *testing.T) {
os.Unsetenv("KEY2")
os.Setenv("KEY1", "V1")
os.Setenv("KEY3", "V3")
keys := []string{"KEY1", "KEY2"}
ecl := EnvConfigLoader{
keys,
}
m, err := ecl.Load()
if err != nil {
t.Errorf("Error loading the configuration via env: %v", err)
}
if m["KEY1"] != "V1" {
t.Errorf("The value for key KEY1 should be V1, but infact: %s", m["KEY1"])
}
if len(m["KEY2"]) > 0 {
t.Errorf("The value for key KEY2 should be emptye, but infact: %s", m["KEY2"])
}
if _, ok := m["KEY3"]; ok {
t.Errorf("The KEY3 should not be in result as it's not in the initial key list")
}
os.Unsetenv("KEY1")
os.Unsetenv("KEY3")
}
func TestCommonConfig(t *testing.T) {
mysql := MySQLSetting{"registry", "root", "password", "127.0.0.1", "3306"}
sqlite := SQLiteSetting{"file.db"}
verify := "off"
ext := "http://harbor"
token := "http://token"
loglevel := "info"
os.Setenv("DATABASE", "")
os.Setenv("MYSQL_DATABASE", mysql.Database)
os.Setenv("MYSQL_USR", mysql.User)
os.Setenv("MYSQL_PWD", mysql.Password)
os.Setenv("MYSQL_HOST", mysql.Host)
os.Setenv("MYSQL_PORT", mysql.Port)
os.Setenv("SQLITE_FILE", sqlite.FilePath)
os.Setenv("VERIFY_REMOTE_CERT", verify)
os.Setenv("EXT_ENDPOINT", ext)
os.Setenv("TOKEN_ENDPOINT", token)
os.Setenv("LOG_LEVEL", loglevel)
err := Reload()
if err != nil {
t.Errorf("Unexpected error when loading the configurations, error: %v", err)
}
if Database() != "mysql" {
t.Errorf("Expected Database value: mysql, fact: %s", mysql)
}
if MySQL() != mysql {
t.Errorf("Expected MySQL setting: %+v, fact: %+v", mysql, MySQL())
}
if VerifyRemoteCert() {
t.Errorf("Expected VerifyRemoteCert: false, env var: %s, fact: %v", verify, VerifyRemoteCert())
}
if ExtEndpoint() != ext {
t.Errorf("Expected ExtEndpoint: %s, fact: %s", ext, ExtEndpoint())
}
if TokenEndpoint() != token {
t.Errorf("Expected TokenEndpoint: %s, fact: %s", token, TokenEndpoint())
}
if LogLevel() != loglevel {
t.Errorf("Expected LogLevel: %s, fact: %s", loglevel, LogLevel())
}
os.Setenv("DATABASE", "sqlite")
err = Reload()
if err != nil {
t.Errorf("Unexpected error when loading the configurations, error: %v", err)
}
if SQLite() != sqlite {
t.Errorf("Expected SQLite setting: %+v, fact %+v", sqlite, SQLite())
}
os.Unsetenv("DATABASE")
os.Unsetenv("MYSQL_DATABASE")
os.Unsetenv("MYSQL_USR")
os.Unsetenv("MYSQL_PWD")
os.Unsetenv("MYSQL_HOST")
os.Unsetenv("MYSQL_PORT")
os.Unsetenv("SQLITE_FILE")
os.Unsetenv("VERIFY_REMOTE_CERT")
os.Unsetenv("EXT_ENDPOINT")
os.Unsetenv("TOKEN_ENDPOINT")
os.Unsetenv("LOG_LEVEL")
}

32
src/common/dao/base.go
View File

@ -17,11 +17,12 @@ package dao
import (
"fmt"
"strconv"
"strings"
"sync"
"github.com/astaxie/beego/orm"
"github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
)
@ -39,27 +40,32 @@ type Database interface {
}
// InitDatabase initializes the database
func InitDatabase() {
database, err := getDatabase()
func InitDatabase(database *models.Database) error {
db, err := getDatabase(database)
if err != nil {
panic(err)
return err
}
log.Infof("initializing database: %s", database.String())
if err := database.Register(); err != nil {
panic(err)
log.Infof("initializing database: %s", db.String())
if err := db.Register(); err != nil {
return err
}
log.Info("initialize database completed")
return nil
}
func getDatabase() (db Database, err error) {
switch config.Database() {
func getDatabase(database *models.Database) (db Database, err error) {
switch database.Type {
case "", "mysql":
db = NewMySQL(config.MySQL().Host, config.MySQL().Port, config.MySQL().User,
config.MySQL().Password, config.MySQL().Database)
db = NewMySQL(database.MySQL.Host,
strconv.Itoa(database.MySQL.Port),
database.MySQL.Username,
database.MySQL.Password,
database.MySQL.Database)
case "sqlite":
db = NewSQLite(config.SQLite().FilePath)
db = NewSQLite(database.SQLite.File)
default:
err = fmt.Errorf("invalid database: %s", config.Database())
err = fmt.Errorf("invalid database: %s", database.Type)
}
return
}

32
src/common/dao/config.go Normal file
View File

@ -0,0 +1,32 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package dao
import (
"github.com/vmware/harbor/src/common/models"
)
// AuthModeCanBeModified determines whether auth mode can be
// modified or not. Auth mode can modified when there is only admin
// user in database.
func AuthModeCanBeModified() (bool, error) {
c, err := GetOrmer().QueryTable(&models.User{}).Count()
if err != nil {
return false, err
}
return c == 1, nil
}

72
src/common/dao/config_test.go Normal file
View File

@ -0,0 +1,72 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package dao
import (
"testing"
"github.com/vmware/harbor/src/common/models"
)
func TestAuthModeCanBeModified(t *testing.T) {
c, err := GetOrmer().QueryTable(&models.User{}).Count()
if err != nil {
t.Fatalf("failed to count users: %v", err)
}
if c == 1 {
flag, err := AuthModeCanBeModified()
if err != nil {
t.Fatalf("failed to determine whether auth mode can be modified: %v", err)
}
if !flag {
t.Errorf("unexpected result: %t != %t", flag, true)
}
user := models.User{
Username: "user_for_config_test",
Email: "user_for_config_test@vmware.com",
Password: "P@ssword",
Realname: "user_for_config_test",
}
id, err := Register(user)
if err != nil {
t.Fatalf("failed to register user: %v", err)
}
defer func(id int64) {
if err := deleteUser(id); err != nil {
t.Fatalf("failed to delete user %d: %v", id, err)
}
}(id)
flag, err = AuthModeCanBeModified()
if err != nil {
t.Fatalf("failed to determine whether auth mode can be modified: %v", err)
}
if flag {
t.Errorf("unexpected result: %t != %t", flag, false)
}
} else {
flag, err := AuthModeCanBeModified()
if err != nil {
t.Fatalf("failed to determine whether auth mode can be modified: %v", err)
}
if flag {
t.Errorf("unexpected result: %t != %t", flag, false)
}
}
}

93
src/common/dao/dao_test.go
View File

@ -17,10 +17,12 @@ package dao
import (
"os"
"strconv"
"testing"
"time"
"github.com/astaxie/beego/orm"
//"github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
@ -42,7 +44,7 @@ func execUpdate(o orm.Ormer, sql string, params ...interface{}) error {
func clearUp(username string) {
var err error
o := orm.NewOrm()
o := GetOrmer()
o.Begin()
err = execUpdate(o, `delete
@ -135,7 +137,7 @@ const publicityOn = 1
const publicityOff = 0
func TestMain(m *testing.M) {
databases := []string{"mysql", "sqlite"}
databases := []string{"mysql"}
for _, database := range databases {
log.Infof("run test cases for database: %s", database)
@ -156,53 +158,63 @@ func TestMain(m *testing.M) {
}
func testForMySQL(m *testing.M) int {
db := os.Getenv("DATABASE")
defer os.Setenv("DATABASE", db)
os.Setenv("DATABASE", "mysql")
dbHost := os.Getenv("DB_HOST")
dbHost := os.Getenv("MYSQL_HOST")
if len(dbHost) == 0 {
log.Fatalf("environment variable DB_HOST is not set")
log.Fatalf("environment variable MYSQL_HOST is not set")
}
dbUser := os.Getenv("DB_USR")
dbUser := os.Getenv("MYSQL_USR")
if len(dbUser) == 0 {
log.Fatalf("environment variable DB_USR is not set")
log.Fatalf("environment variable MYSQL_USR is not set")
}
dbPort := os.Getenv("DB_PORT")
if len(dbPort) == 0 {
log.Fatalf("environment variable DB_PORT is not set")
dbPortStr := os.Getenv("MYSQL_PORT")
if len(dbPortStr) == 0 {
log.Fatalf("environment variable MYSQL_PORT is not set")
}
dbPort, err := strconv.Atoi(dbPortStr)
if err != nil {
log.Fatalf("invalid MYSQL_PORT: %v", err)
}
dbPassword := os.Getenv("DB_PWD")
log.Infof("DB_HOST: %s, DB_USR: %s, DB_PORT: %s, DB_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword)
dbPassword := os.Getenv("MYSQL_PWD")
dbDatabase := os.Getenv("MYSQL_DATABASE")
if len(dbDatabase) == 0 {
log.Fatalf("environment variable MYSQL_DATABASE is not set")
}
os.Setenv("MYSQL_HOST", dbHost)
os.Setenv("MYSQL_PORT", dbPort)
os.Setenv("MYSQL_USR", dbUser)
os.Setenv("MYSQL_PWD", dbPassword)
database := &models.Database{
Type: "mysql",
MySQL: &models.MySQL{
Host: dbHost,
Port: dbPort,
Username: dbUser,
Password: dbPassword,
Database: dbDatabase,
},
}
return testForAll(m)
log.Infof("MYSQL_HOST: %s, MYSQL_USR: %s, MYSQL_PORT: %s, MYSQL_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword)
return testForAll(m, database)
}
func testForSQLite(m *testing.M) int {
db := os.Getenv("DATABASE")
defer os.Setenv("DATABASE", db)
os.Setenv("DATABASE", "sqlite")
file := os.Getenv("SQLITE_FILE")
if len(file) == 0 {
os.Setenv("SQLITE_FILE", "/registry.db")
defer os.Setenv("SQLITE_FILE", "")
log.Fatalf("environment variable SQLITE_FILE is not set")
}
return testForAll(m)
database := &models.Database{
Type: "sqlite",
SQLite: &models.SQLite{
File: file,
},
}
func testForAll(m *testing.M) int {
os.Setenv("AUTH_MODE", "db_auth")
initDatabaseForTest()
return testForAll(m, database)
}
func testForAll(m *testing.M, database *models.Database) int {
initDatabaseForTest(database)
clearUp(username)
return m.Run()
@ -210,8 +222,8 @@ func testForAll(m *testing.M) int {
var defaultRegistered = false
func initDatabaseForTest() {
database, err := getDatabase()
func initDatabaseForTest(db *models.Database) {
database, err := getDatabase(db)
if err != nil {
panic(err)
}
@ -226,6 +238,12 @@ func initDatabaseForTest() {
if err := database.Register(alias); err != nil {
panic(err)
}
if alias != "default" {
if err = globalOrm.Using(alias); err != nil {
log.Fatalf("failed to create new orm: %v", err)
}
}
}
func TestRegister(t *testing.T) {
@ -980,13 +998,6 @@ func TestGetRecentLogs(t *testing.T) {
}
}
func TestGetTopRepos(t *testing.T) {
_, err := GetTopRepos(10)
if err != nil {
t.Fatalf("error occured in getting top repos, error: %v", err)
}
}
var targetID, policyID, policyID2, policyID3, jobID, jobID2, jobID3 int64
func TestAddRepTarget(t *testing.T) {

33
src/common/dao/mysql.go
View File

@ -16,15 +16,11 @@
package dao
import (
"errors"
"fmt"
"net"
"time"
"github.com/astaxie/beego/orm"
_ "github.com/go-sql-driver/mysql" //register mysql driver
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils"
)
type mysql struct {
@ -48,7 +44,8 @@ func NewMySQL(host, port, usr, pwd, database string) Database {
// Register registers MySQL as the underlying database used
func (m *mysql) Register(alias ...string) error {
if err := m.testConn(m.host, m.port); err != nil {
if err := utils.TestTCPConn(m.host+":"+m.port, 60, 2); err != nil {
return err
}
@ -65,30 +62,6 @@ func (m *mysql) Register(alias ...string) error {
return orm.RegisterDataBase(an, "mysql", conn)
}
func (m *mysql) testConn(host, port string) error {
ch := make(chan int, 1)
go func() {
var err error
var c net.Conn
for {
c, err = net.DialTimeout("tcp", host+":"+port, 20*time.Second)
if err == nil {
c.Close()
ch <- 1
} else {
log.Errorf("failed to connect to db, retry after 2 seconds :%v", err)
time.Sleep(2 * time.Second)
}
}
}()
select {
case <-ch:
return nil
case <-time.After(60 * time.Second):
return errors.New("failed to connect to database after 60 seconds")
}
}
// Name returns the name of MySQL
func (m *mysql) Name() string {
return "MySQL"

16
src/common/dao/repository.go
View File

@ -102,12 +102,22 @@ func GetRepositoryByProjectName(name string) ([]*models.RepoRecord, error) {
}
//GetTopRepos returns the most popular repositories
func GetTopRepos(count int) ([]models.TopRepo, error) {
func GetTopRepos(userID int, count int) ([]models.TopRepo, error) {
topRepos := []models.TopRepo{}
sql := `select r.name, r.pull_count from repository r
inner join project p on r.project_id = p.project_id
where (
p.deleted = 0 and (
p.public = 1 or (
? <> ? and exists (
select 1 from project_member pm
where pm.project_id = p.project_id and pm.user_id = ?
))))
order by r.pull_count desc, r.name limit ?`
repositories := []*models.RepoRecord{}
if _, err := GetOrmer().QueryTable(&models.RepoRecord{}).
OrderBy("-PullCount", "Name").Limit(count).All(&repositories); err != nil {
_, err := GetOrmer().Raw(sql, userID, NonExistUserID, userID, count).QueryRows(&repositories)
if err != nil {
return topRepos, err
}

158
src/common/dao/repository_test.go
View File

@ -16,8 +16,11 @@
package dao
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/vmware/harbor/src/common/models"
)
@ -160,6 +163,161 @@ func TestGetTotalOfUserRelevantRepositories(t *testing.T) {
}
}
func TestGetTopRepos(t *testing.T) {
var err error
require := require.New(t)
require.NoError(GetOrmer().Begin())
defer func() {
require.NoError(GetOrmer().Rollback())
}()
admin, err := GetUser(models.User{Username: "admin"})
require.NoError(err)
user := models.User{
Username: "user",
Password: "user",
Email: "user@test.com",
}
userID, err := Register(user)
require.NoError(err)
user.UserID = int(userID)
//
// public project with 1 repository
// non-public project with 2 repositories visible by admin
// non-public project with 1 repository visible by admin and user
// deleted public project with 1 repository
//
project1 := models.Project{
OwnerID: admin.UserID,
Name: "project1",
CreationTime: time.Now(),
OwnerName: admin.Username,
Public: 0,
}
project1.ProjectID, err = AddProject(project1)
require.NoError(err)
project2 := models.Project{
OwnerID: admin.UserID,
Name: "project2",
CreationTime: time.Now(),
OwnerName: admin.Username,
Public: 0,
}
project2.ProjectID, err = AddProject(project2)
require.NoError(err)
require.NoError(AddProjectMember(project2.ProjectID, user.UserID, models.PROJECTADMIN))
err = AddRepository(*repository)
require.NoError(err)
repository1 := &models.RepoRecord{
Name: fmt.Sprintf("%v/repository1", project1.Name),
OwnerName: admin.Username,
ProjectName: project1.Name,
}
err = AddRepository(*repository1)
require.NoError(err)
require.NoError(IncreasePullCount(repository1.Name))
repository1, err = GetRepositoryByName(repository1.Name)
require.NoError(err)
repository2 := &models.RepoRecord{
Name: fmt.Sprintf("%v/repository2", project1.Name),
OwnerName: admin.Username,
ProjectName: project1.Name,
}
err = AddRepository(*repository2)
require.NoError(err)
require.NoError(IncreasePullCount(repository2.Name))
require.NoError(IncreasePullCount(repository2.Name))
repository2, err = GetRepositoryByName(repository2.Name)
require.NoError(err)
repository3 := &models.RepoRecord{
Name: fmt.Sprintf("%v/repository3", project2.Name),
OwnerName: admin.Username,
ProjectName: project2.Name,
}
err = AddRepository(*repository3)
require.NoError(err)
require.NoError(IncreasePullCount(repository3.Name))
require.NoError(IncreasePullCount(repository3.Name))
require.NoError(IncreasePullCount(repository3.Name))
repository3, err = GetRepositoryByName(repository3.Name)
require.NoError(err)
deletedPublicProject := models.Project{
OwnerID: admin.UserID,
Name: "public-deleted",
CreationTime: time.Now(),
OwnerName: admin.Username,
Public: 1,
}
deletedPublicProject.ProjectID, err = AddProject(deletedPublicProject)
require.NoError(err)
deletedPublicRepository1 := &models.RepoRecord{
Name: fmt.Sprintf("%v/repository1", deletedPublicProject.Name),
OwnerName: admin.Username,
ProjectName: deletedPublicProject.Name,
}
err = AddRepository(*deletedPublicRepository1)
require.NoError(err)
DeleteProject(deletedPublicProject.ProjectID)
var topRepos []models.TopRepo
// anonymous should retrieve public non-deleted repositories
topRepos, err = GetTopRepos(NonExistUserID, 3)
require.NoError(err)
require.Len(topRepos, 1)
require.Equal(topRepos, []models.TopRepo{
models.TopRepo{
RepoName: repository.Name,
AccessCount: repository.PullCount,
},
})
// admin should retrieve all visible repositories limited by count
topRepos, err = GetTopRepos(admin.UserID, 3)
require.NoError(err)
require.Len(topRepos, 3)
require.Equal(topRepos, []models.TopRepo{
models.TopRepo{
RepoName: repository3.Name,
AccessCount: repository3.PullCount,
},
models.TopRepo{
RepoName: repository2.Name,
AccessCount: repository2.PullCount,
},
models.TopRepo{
RepoName: repository1.Name,
AccessCount: repository1.PullCount,
},
})
// user should retrieve all visible repositories
topRepos, err = GetTopRepos(user.UserID, 3)
require.NoError(err)
require.Len(topRepos, 2)
require.Equal(topRepos, []models.TopRepo{
models.TopRepo{
RepoName: repository3.Name,
AccessCount: repository3.PullCount,
},
models.TopRepo{
RepoName: repository.Name,
AccessCount: repository.PullCount,
},
})
}
func addRepository(repository *models.RepoRecord) error {
return AddRepository(*repository)
}

13
src/common/dao/user_test.go
View File

@ -38,6 +38,11 @@ func TestDeleteUser(t *testing.T) {
if err != nil {
t.Fatalf("failed to register user: %v", err)
}
defer func(id int64) {
if err := deleteUser(id); err != nil {
t.Fatalf("failed to delete user %d: %v", id, err)
}
}(id)
err = DeleteUser(int(id))
if err != nil {
@ -67,3 +72,11 @@ func TestDeleteUser(t *testing.T) {
expected)
}
}
func deleteUser(id int64) error {
if _, err := GetOrmer().QueryTable(&models.User{}).
Filter("UserID", id).Delete(); err != nil {
return err
}
return nil
}

96
src/common/models/config.go Normal file
View File

@ -0,0 +1,96 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package models
// Authentication ...
type Authentication struct {
Mode string `json:"mode"`
SelfRegistration bool `json:"self_registration"`
LDAP *LDAP `json:"ldap,omitempty"`
}
// LDAP ...
type LDAP struct {
URL string `json:"url"`
SearchDN string `json:"search_dn"`
SearchPwd string `json:"search_pwd"`
BaseDN string `json:"base_dn"`
Filter string `json:"filter"`
UID string `json:"uid"`
Scope int `json:"scope"`
Timeout int `json:"timeout"` // in second
}
// Database ...
type Database struct {
Type string `json:"type"`
MySQL *MySQL `json:"mysql,omitempty"`
SQLite *SQLite `json:"sqlite,omitempty"`
}
// MySQL ...
type MySQL struct {
Host string `json:"host"`
Port int `json:"port"`
Username string `json:"username"`
Password string `json:"password,omitempty"`
Database string `json:"database"`
}
// SQLite ...
type SQLite struct {
File string `json:"file"`
}
// Email ...
type Email struct {
Host string `json:"host"`
Port string `json:"port"`
Username string `json:"username"`
Password string `json:"password"`
SSL bool `json:"ssl"`
Identity string `json:"identity"`
From string `json:"from"`
}
// Registry ...
type Registry struct {
URL string `json:"url"`
}
// TokenService ...
type TokenService struct {
URL string `json:"url"`
}
// SystemCfg holds all configurations of system
type SystemCfg struct {
DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port
Authentication *Authentication `json:"authentication"`
Database *Database `json:"database"`
TokenService *TokenService `json:"token_service"`
Registry *Registry `json:"registry"`
Email *Email `json:"email"`
VerifyRemoteCert bool `json:"verify_remote_cert"`
ProjectCreationRestriction string `json:"project_creation_restriction"`
MaxJobWorkers int `json:"max_job_workers"`
JobLogDir string `json:"job_log_dir"`
InitialAdminPwd string `json:"initial_admin_pwd,omitempty"`
CompressJS bool `json:"compress_js"` //TODO remove
TokenExpiration int `json:"token_expiration"` // in minute
SecretKey string `json:"secret_key,omitempty"`
CfgExpiration int `json:"cfg_expiration"`
}

31
src/common/utils/mail.go → src/common/utils/email/mail.go
View File

@ -13,17 +13,19 @@
limitations under the License.
*/
package utils
package email
import (
"bytes"
"crypto/tls"
"strings"
//"strings"
"net/smtp"
"text/template"
"github.com/astaxie/beego"
//"github.com/astaxie/beego"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/ui/config"
)
// Mail holds information about content of Email
@ -34,24 +36,15 @@ type Mail struct {
Message string
}
// MailConfig holds information about Email configurations
type MailConfig struct {
Identity string
Host string
Port string
Username string
Password string
TLS bool
}
var mc MailConfig
var mc models.Email
// SendMail sends Email according to the configurations
func (m Mail) SendMail() error {
if mc.Host == "" {
loadConfig()
mc, err := config.Email()
if err != nil {
return err
}
mailTemplate, err := template.ParseFiles("views/mail.tpl")
if err != nil {
return err
@ -64,7 +57,7 @@ func (m Mail) SendMail() error {
content := mailContent.Bytes()
auth := smtp.PlainAuth(mc.Identity, mc.Username, mc.Password, mc.Host)
if mc.TLS {
if mc.SSL {
err = sendMailWithTLS(m, auth, content)
} else {
err = sendMail(m, auth, content)
@ -123,6 +116,7 @@ func sendMailWithTLS(m Mail, auth smtp.Auth, content []byte) error {
return client.Quit()
}
/*
func loadConfig() {
config, err := beego.AppConfig.GetSection("mail")
if err != nil {
@ -142,3 +136,4 @@ func loadConfig() {
TLS: useTLS,
}
}
*/

4
src/common/utils/log/logger.go
View File

@ -22,8 +22,6 @@ import (
"runtime"
"sync"
"time"
"github.com/vmware/harbor/src/common/config"
)
var logger = New(os.Stdout, NewTextFormatter(), WarningLevel)
@ -31,7 +29,7 @@ var logger = New(os.Stdout, NewTextFormatter(), WarningLevel)
func init() {
logger.callDepth = 4
lvl := config.LogLevel()
lvl := os.Getenv("LOG_LEVEL")
if len(lvl) == 0 {
logger.SetLevel(InfoLevel)
return

5
src/common/utils/registry/auth/tokenauthorizer.go
View File

@ -25,7 +25,7 @@ import (
"sync"
"time"
"github.com/vmware/harbor/src/common/config"
//"github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils/registry"
registry_error "github.com/vmware/harbor/src/common/utils/registry/error"
@ -234,12 +234,15 @@ func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []
// 2. the realm field returned by registry is an IP which can not reachable
// inside Harbor
func tokenURL(realm string) string {
//TODO
/*
extEndpoint := config.ExtEndpoint()
tokenEndpoint := config.TokenEndpoint()
if len(extEndpoint) != 0 && len(tokenEndpoint) != 0 &&
strings.Contains(realm, extEndpoint) {
realm = strings.TrimRight(tokenEndpoint, "/") + "/service/token"
}
*/
return realm
}

59
src/common/utils/test/adminserver.go Normal file
View File

@ -0,0 +1,59 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package test
import (
"encoding/json"
"net/http"
"net/http/httptest"
"github.com/vmware/harbor/src/common/models"
)
// NewAdminserver returns a mock admin server
func NewAdminserver() (*httptest.Server, error) {
m := []*RequestHandlerMapping{}
b, err := json.Marshal(&models.SystemCfg{
Authentication: &models.Authentication{
Mode: "db_auth",
},
Registry: &models.Registry{},
})
if err != nil {
return nil, err
}
resp := &Response{
StatusCode: http.StatusOK,
Body: b,
}
m = append(m, &RequestHandlerMapping{
Method: "GET",
Pattern: "/api/configurations",
Handler: Handler(resp),
})
m = append(m, &RequestHandlerMapping{
Method: "PUT",
Pattern: "/api/configurations",
Handler: Handler(&Response{
StatusCode: http.StatusOK,
}),
})
return NewServer(m...), nil
}

8
src/common/utils/test/test.go
View File

@ -21,6 +21,8 @@ import (
"net/http"
"net/http/httptest"
"strings"
"github.com/gorilla/mux"
)
// RequestHandlerMapping is a mapping between request and its handler
@ -78,11 +80,11 @@ func Handler(resp *Response) func(http.ResponseWriter, *http.Request) {
// NewServer creates a HTTP server for unit test
func NewServer(mappings ...*RequestHandlerMapping) *httptest.Server {
mux := http.NewServeMux()
r := mux.NewRouter()
for _, mapping := range mappings {
mux.Handle(mapping.Pattern, mapping)
r.PathPrefix(mapping.Pattern).Handler(mapping).Methods(mapping.Method)
}
return httptest.NewServer(mux)
return httptest.NewServer(r)
}

41
src/common/utils/utils.go
View File

@ -16,10 +16,14 @@
package utils
import (
"fmt"
"math/rand"
"net"
"net/url"
"strings"
"time"
"github.com/vmware/harbor/src/common/utils/log"
)
// FormatEndpoint formats endpoint
@ -70,3 +74,40 @@ func GenerateRandomString() string {
}
return string(result)
}
// TestTCPConn tests TCP connection
// timeout: the total time before returning if something is wrong
// with the connection, in second
// interval: the interval time for retring after failure, in second
func TestTCPConn(addr string, timeout, interval int) error {
success := make(chan int)
cancel := make(chan int)
go func() {
for {
select {
case <-cancel:
break
default:
conn, err := net.DialTimeout("tcp", addr, time.Duration(timeout)*time.Second)
if err != nil {
log.Errorf("failed to connect to tcp://%s, retry after %d seconds :%v",
addr, interval, err)
time.Sleep(time.Duration(interval) * time.Second)
continue
}
conn.Close()
success <- 1
break
}
}
}()
select {
case <-success:
return nil
case <-time.After(time.Duration(timeout) * time.Second):
cancel <- 1
return fmt.Errorf("failed to connect to tcp:%s after %d seconds", addr, timeout)
}
}

10
src/common/utils/utils_test.go
View File

@ -17,6 +17,7 @@ package utils
import (
"encoding/base64"
"net/http/httptest"
"strings"
"testing"
)
@ -178,3 +179,12 @@ func TestParseLink(t *testing.T) {
t.Errorf("unexpected prev: %s != %s", links.Next(), next)
}
}
func TestTestTCPConn(t *testing.T) {
server := httptest.NewServer(nil)
defer server.Close()
addr := strings.TrimLeft(server.URL, "http://")
if err := TestTCPConn(addr, 60, 2); err != nil {
t.Fatalf("failed to test tcp connection of %s: %v", addr, err)
}
}

14
src/jobservice/api/replication.go
View File

@ -25,12 +25,12 @@ import (
"github.com/vmware/harbor/src/common/api"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/jobservice/job"
"github.com/vmware/harbor/src/jobservice/config"
"github.com/vmware/harbor/src/jobservice/utils"
"github.com/vmware/harbor/src/common/models"
u "github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/jobservice/config"
"github.com/vmware/harbor/src/jobservice/job"
"github.com/vmware/harbor/src/jobservice/utils"
)
// ReplicationJob handles /api/replicationJobs /api/replicationJobs/:id/log
@ -171,7 +171,13 @@ func (rj *ReplicationJob) GetLog() {
rj.RenderError(http.StatusBadRequest, "Invalid job id")
return
}
logFile := utils.GetJobLogPath(jid)
logFile, err := utils.GetJobLogPath(jid)
if err != nil {
log.Errorf("failed to get log path of job %s: %v", idStr, err)
rj.RenderError(http.StatusInternalServerError,
http.StatusText(http.StatusInternalServerError))
return
}
rj.Ctx.Output.Download(logFile)
}

174
src/jobservice/config/config.go
View File

@ -16,121 +16,125 @@
package config
import (
"fmt"
"encoding/json"
"os"
"strconv"
"github.com/astaxie/beego"
"github.com/vmware/harbor/src/common/utils/log"
comcfg "github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/models"
)
const defaultMaxWorkers int = 10
var mg *comcfg.Manager
var maxJobWorkers int
var localUIURL string
var localRegURL string
var logDir string
var uiSecret string
var secretKey string
var verifyRemoteCert string
// Configuration of Jobservice
type Configuration struct {
Database *models.Database `json:"database"`
Registry *models.Registry `json:"registry"`
VerifyRemoteCert bool `json:"verify_remote_cert"`
MaxJobWorkers int `json:"max_job_workers"`
JobLogDir string `json:"job_log_dir"`
SecretKey string `json:"secret_key"`
CfgExpiration int `json:"cfg_expiration"`
}
func init() {
maxWorkersEnv := os.Getenv("MAX_JOB_WORKERS")
maxWorkers64, err := strconv.ParseInt(maxWorkersEnv, 10, 32)
maxJobWorkers = int(maxWorkers64)
type parser struct {
}
func (p *parser) Parse(b []byte) (interface{}, error) {
c := &Configuration{}
if err := json.Unmarshal(b, c); err != nil {
return nil, err
}
return c, nil
}
// Init configurations
func Init() error {
adminServerURL := os.Getenv("ADMIN_SERVER_URL")
if len(adminServerURL) == 0 {
adminServerURL = "http://adminserver"
}
mg = comcfg.NewManager(adminServerURL, UISecret(), &parser{}, true)
if err := mg.Init(); err != nil {
return err
}
if _, err := mg.Load(); err != nil {
return err
}
return nil
}
func get() (*Configuration, error) {
c, err := mg.Get()
if err != nil {
log.Warningf("Failed to parse max works setting, error: %v, the default value: %d will be used", err, defaultMaxWorkers)
maxJobWorkers = defaultMaxWorkers
return nil, err
}
return c.(*Configuration), nil
}
localRegURL = os.Getenv("REGISTRY_URL")
if len(localRegURL) == 0 {
localRegURL = "http://registry:5000"
}
localUIURL = os.Getenv("UI_URL")
if len(localUIURL) == 0 {
localUIURL = "http://ui"
}
logDir = os.Getenv("LOG_DIR")
if len(logDir) == 0 {
logDir = "/var/log"
}
f, err := os.Open(logDir)
defer f.Close()
// VerifyRemoteCert returns bool value.
func VerifyRemoteCert() (bool, error) {
cfg, err := get()
if err != nil {
panic(err)
return true, err
}
finfo, err := f.Stat()
return cfg.VerifyRemoteCert, nil
}
// Database ...
func Database() (*models.Database, error) {
cfg, err := get()
if err != nil {
panic(err)
return nil, err
}
if !finfo.IsDir() {
panic(fmt.Sprintf("%s is not a direcotry", logDir))
}
uiSecret = os.Getenv("UI_SECRET")
if len(uiSecret) == 0 {
panic("UI Secret is not set")
}
verifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT")
if len(verifyRemoteCert) == 0 {
verifyRemoteCert = "on"
}
configPath := os.Getenv("CONFIG_PATH")
if len(configPath) != 0 {
log.Infof("Config path: %s", configPath)
beego.LoadAppConfig("ini", configPath)
}
secretKey = os.Getenv("SECRET_KEY")
if len(secretKey) != 16 {
panic("The length of secretkey has to be 16 characters!")
}
log.Debugf("config: maxJobWorkers: %d", maxJobWorkers)
log.Debugf("config: localUIURL: %s", localUIURL)
log.Debugf("config: localRegURL: %s", localRegURL)
log.Debugf("config: verifyRemoteCert: %s", verifyRemoteCert)
log.Debugf("config: logDir: %s", logDir)
log.Debugf("config: uiSecret: ******")
return cfg.Database, nil
}
// MaxJobWorkers ...
func MaxJobWorkers() int {
return maxJobWorkers
func MaxJobWorkers() (int, error) {
cfg, err := get()
if err != nil {
return 0, err
}
return cfg.MaxJobWorkers, nil
}
// LocalUIURL returns the local ui url, job service will use this URL to call API hosted on ui process
func LocalUIURL() string {
return localUIURL
return "http://ui"
}
// LocalRegURL returns the local registry url, job service will use this URL to pull image from the registry
func LocalRegURL() string {
return localRegURL
func LocalRegURL() (string, error) {
cfg, err := get()
if err != nil {
return "", err
}
return cfg.Registry.URL, nil
}
// LogDir returns the absolute path to which the log file will be written
func LogDir() string {
return logDir
func LogDir() (string, error) {
cfg, err := get()
if err != nil {
return "", err
}
// UISecret will return the value of secret cookie for jobsevice to call UI API.
func UISecret() string {
return uiSecret
return cfg.JobLogDir, nil
}
// SecretKey will return the secret key for encryption/decryption password in target.
func SecretKey() string {
return secretKey
func SecretKey() (string, error) {
cfg, err := get()
if err != nil {
return "", err
}
return cfg.SecretKey, nil
}
// VerifyRemoteCert return the flag to tell jobservice whether or not verify the cert of remote registry
func VerifyRemoteCert() bool {
return verifyRemoteCert != "off"
// UISecret returns the value of UI secret cookie, used for communication between UI and JobService
// TODO
func UISecret() string {
return os.Getenv("UI_SECRET")
}

64
src/jobservice/config/config_test.go
View File

@ -1,9 +1,71 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"os"
"testing"
"github.com/vmware/harbor/src/common/utils/test"
)
func TestMain(t *testing.T) {
// test functions under package jobservice/config
func TestConfig(t *testing.T) {
server, err := test.NewAdminserver()
if err != nil {
t.Fatalf("failed to create a mock admin server: %v", err)
}
defer server.Close()
url := os.Getenv("ADMIN_SERVER_URL")
defer os.Setenv("ADMIN_SERVER_URL", url)
if err := os.Setenv("ADMIN_SERVER_URL", server.URL); err != nil {
t.Fatalf("failed to set env %s: %v", "ADMIN_SERVER_URL", err)
}
if err := Init(); err != nil {
t.Fatalf("failed to initialize configurations: %v", err)
}
if _, err := VerifyRemoteCert(); err != nil {
t.Fatalf("failed to get verify remote cert: %v", err)
}
if _, err := Database(); err != nil {
t.Fatalf("failed to get database settings: %v", err)
}
if _, err := MaxJobWorkers(); err != nil {
t.Fatalf("failed to get max job workers: %v", err)
}
LocalUIURL()
if _, err := LocalRegURL(); err != nil {
t.Fatalf("failed to get registry URL: %v", err)
}
if _, err := LogDir(); err != nil {
t.Fatalf("failed to get log directory: %v", err)
}
if _, err := SecretKey(); err != nil {
t.Fatalf("failed to get secret key: %v", err)
}
UISecret()
}

32
src/jobservice/job/statemachine.go
View File

@ -20,12 +20,12 @@ import (
"sync"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/jobservice/config"
"github.com/vmware/harbor/src/jobservice/replication"
"github.com/vmware/harbor/src/jobservice/utils"
"github.com/vmware/harbor/src/common/models"
uti "github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/jobservice/config"
"github.com/vmware/harbor/src/jobservice/replication"
"github.com/vmware/harbor/src/jobservice/utils"
)
// RepJobParm wraps the parm of a job
@ -184,14 +184,17 @@ func (sm *SM) Init() {
}
// Reset resets the state machine so it will start handling another job.
func (sm *SM) Reset(jid int64) error {
func (sm *SM) Reset(jid int64) (err error) {
//To ensure the new jobID is visible to the thread to stop the SM
sm.lock.Lock()
sm.JobID = jid
sm.desiredState = ""
sm.lock.Unlock()
sm.Logger = utils.NewLogger(sm.JobID)
sm.Logger, err = utils.NewLogger(sm.JobID)
if err != nil {
return
}
//init parms
job, err := dao.GetRepJob(sm.JobID)
if err != nil {
@ -207,13 +210,22 @@ func (sm *SM) Reset(jid int64) error {
if policy == nil {
return fmt.Errorf("The policy doesn't exist in DB, policy id:%d", job.PolicyID)
}
regURL, err := config.LocalRegURL()
if err != nil {
return err
}
verify, err := config.VerifyRemoteCert()
if err != nil {
return err
}
sm.Parms = &RepJobParm{
LocalRegURL: config.LocalRegURL(),
LocalRegURL: regURL,
Repository: job.Repository,
Tags: job.TagList,
Enabled: policy.Enabled,
Operation: job.Operation,
Insecure: !config.VerifyRemoteCert(),
Insecure: !verify,
}
if policy.Enabled == 0 {
//worker will cancel this job
@ -231,7 +243,11 @@ func (sm *SM) Reset(jid int64) error {
pwd := target.Password
if len(pwd) != 0 {
pwd, err = uti.ReversibleDecrypt(pwd, config.SecretKey())
key, err := config.SecretKey()
if err != nil {
return err
}
pwd, err = uti.ReversibleDecrypt(pwd, key)
if err != nil {
return fmt.Errorf("failed to decrypt password: %v", err)
}

17
src/jobservice/job/workerpool.go
View File

@ -17,9 +17,9 @@ package job
import (
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/jobservice/config"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/jobservice/config"
)
type workerPool struct {
@ -111,17 +111,22 @@ func NewWorker(id int) *Worker {
}
// InitWorkerPool create workers according to configuration.
func InitWorkerPool() {
WorkerPool = &workerPool{
workerChan: make(chan *Worker, config.MaxJobWorkers()),
workerList: make([]*Worker, 0, config.MaxJobWorkers()),
func InitWorkerPool() error {
n, err := config.MaxJobWorkers()
if err != nil {
return err
}
for i := 0; i < config.MaxJobWorkers(); i++ {
WorkerPool = &workerPool{
workerChan: make(chan *Worker, n),
workerList: make([]*Worker, 0, n),
}
for i := 0; i < n; i++ {
worker := NewWorker(i)
WorkerPool.workerList = append(WorkerPool.workerList, worker)
worker.Start()
log.Debugf("worker %d started", worker.ID)
}
return nil
}
// Dispatch will listen to the jobQueue of job service and try to pick a free worker from the worker pool and assign the job to it.

29
src/jobservice/main.go
View File

@ -16,15 +16,32 @@
package main
import (
"os"
"github.com/astaxie/beego"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/jobservice/job"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/jobservice/config"
"github.com/vmware/harbor/src/jobservice/job"
)
func main() {
dao.InitDatabase()
log.Info("initializing configurations...")
if err := config.Init(); err != nil {
log.Fatalf("failed to initialize configurations: %v", err)
}
log.Info("configurations initialization completed")
database, err := config.Database()
if err != nil {
log.Fatalf("failed to get database configurations: %v", err)
}
if err := dao.InitDatabase(database); err != nil {
log.Fatalf("failed to initialize database: %v", err)
}
initRouters()
job.InitWorkerPool()
go job.Dispatch()
@ -48,3 +65,11 @@ func resumeJobs() {
log.Warningf("Failed to jobs to resume, error: %v", err)
}
}
func init() {
configPath := os.Getenv("CONFIG_PATH")
if len(configPath) != 0 {
log.Infof("Config path: %s", configPath)
beego.LoadAppConfig("ini", configPath)
}
}

24
src/jobservice/utils/logger.go
View File

@ -18,16 +18,20 @@ package utils
import (
"fmt"
"github.com/vmware/harbor/src/jobservice/config"
"github.com/vmware/harbor/src/common/utils/log"
"os"
"path/filepath"
"strconv"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/jobservice/config"
)
// NewLogger create a logger for a speicified job
func NewLogger(jobID int64) *log.Logger {
logFile := GetJobLogPath(jobID)
func NewLogger(jobID int64) (*log.Logger, error) {
logFile, err := GetJobLogPath(jobID)
if err != nil {
return nil, err
}
d := filepath.Dir(logFile)
if _, err := os.Stat(d); os.IsNotExist(err) {
err := os.MkdirAll(d, 0660)
@ -40,11 +44,11 @@ func NewLogger(jobID int64) *log.Logger {
log.Errorf("Failed to open log file %s, the log of job %d will be printed to standard output, the error: %v", logFile, jobID, err)
f = os.Stdout
}
return log.New(f, log.NewTextFormatter(), log.InfoLevel)
return log.New(f, log.NewTextFormatter(), log.InfoLevel), nil
}
// GetJobLogPath returns the absolute path in which the job log file is located.
func GetJobLogPath(jobID int64) string {
func GetJobLogPath(jobID int64) (string, error) {
f := fmt.Sprintf("job_%d.log", jobID)
k := jobID / 1000
p := ""
@ -61,6 +65,10 @@ func GetJobLogPath(jobID int64) string {
p = filepath.Join(d, p)
}
p = filepath.Join(config.LogDir(), p, f)
return p
base, err := config.LogDir()
if err != nil {
return "", err
}
p = filepath.Join(base, p, f)
return p, nil
}

259
src/ui/api/config.go Normal file
View File

@ -0,0 +1,259 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package api
import (
"fmt"
"net/http"
"strconv"
//"strings"
"github.com/vmware/harbor/src/common/api"
comcfg "github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/dao"
//"github.com/vmware/harbor/src/common/models"
//"github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/config"
)
// ConfigAPI ...
type ConfigAPI struct {
api.BaseAPI
}
// Prepare validates the user
func (c *ConfigAPI) Prepare() {
userID := c.ValidateUser()
isSysAdmin, err := dao.IsAdminRole(userID)
if err != nil {
log.Errorf("failed to check the role of user: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if !isSysAdmin {
c.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden))
}
}
// Get returns configurations
func (c *ConfigAPI) Get() {
cfg, err := config.GetSystemCfg()
if err != nil {
log.Errorf("failed to get configurations: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if cfg.Database.MySQL != nil {
cfg.Database.MySQL.Password = ""
}
cfg.InitialAdminPwd = ""
cfg.SecretKey = ""
m := map[string]interface{}{}
m["config"] = cfg
editable, err := dao.AuthModeCanBeModified()
if err != nil {
log.Errorf("failed to determinie whether auth mode can be modified: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
m["auth_mode_editable"] = editable
c.Data["json"] = m
c.ServeJSON()
}
// Put updates configurations
func (c *ConfigAPI) Put() {
m := map[string]string{}
c.DecodeJSONReq(&m)
if err := validateCfg(m); err != nil {
c.CustomAbort(http.StatusBadRequest, err.Error())
}
if value, ok := m[comcfg.AUTHMode]; ok {
mode, err := config.AuthMode()
if err != nil {
log.Errorf("failed to get auth mode: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if mode != value {
flag, err := authModeCanBeModified()
if err != nil {
log.Errorf("failed to determine whether auth mode can be modified: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if !flag {
c.CustomAbort(http.StatusBadRequest,
fmt.Sprintf("%s can not be modified as new users have been inserted into database",
comcfg.AUTHMode))
}
}
}
if err := config.Upload(m); err != nil {
log.Errorf("failed to upload configurations: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if err := config.Load(); err != nil {
log.Errorf("failed to load configurations: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
}
func validateCfg(c map[string]string) error {
if value, ok := c[comcfg.AUTHMode]; ok {
if value != comcfg.DBAuth && value != comcfg.LDAPAuth {
return fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTHMode, comcfg.DBAuth, comcfg.LDAPAuth)
}
if value == comcfg.LDAPAuth {
if _, ok := c[comcfg.LDAPURL]; !ok {
return fmt.Errorf("%s is missing", comcfg.LDAPURL)
}
if _, ok := c[comcfg.LDAPBaseDN]; !ok {
return fmt.Errorf("%s is missing", comcfg.LDAPBaseDN)
}
if _, ok := c[comcfg.LDAPUID]; !ok {
return fmt.Errorf("%s is missing", comcfg.LDAPUID)
}
if _, ok := c[comcfg.LDAPScope]; !ok {
return fmt.Errorf("%s is missing", comcfg.LDAPScope)
}
}
}
if ldapURL, ok := c[comcfg.LDAPURL]; ok && len(ldapURL) == 0 {
return fmt.Errorf("%s is empty", comcfg.LDAPURL)
}
if baseDN, ok := c[comcfg.LDAPBaseDN]; ok && len(baseDN) == 0 {
return fmt.Errorf("%s is empty", comcfg.LDAPBaseDN)
}
if uID, ok := c[comcfg.LDAPUID]; ok && len(uID) == 0 {
return fmt.Errorf("%s is empty", comcfg.LDAPUID)
}
if scope, ok := c[comcfg.LDAPScope]; ok &&
scope != comcfg.LDAPScopeBase &&
scope != comcfg.LDAPScopeOnelevel &&
scope != comcfg.LDAPScopeSubtree {
return fmt.Errorf("invalid %s, should be %s, %s or %s",
comcfg.LDAPScope,
comcfg.LDAPScopeBase,
comcfg.LDAPScopeOnelevel,
comcfg.LDAPScopeSubtree)
}
if timeout, ok := c[comcfg.LDAPTimeout]; ok {
if t, err := strconv.Atoi(timeout); err != nil || t < 0 {
return fmt.Errorf("invalid %s", comcfg.LDAPTimeout)
}
}
if self, ok := c[comcfg.SelfRegistration]; ok &&
self != "0" && self != "1" {
return fmt.Errorf("%s should be %s or %s",
comcfg.SelfRegistration, "0", "1")
}
if port, ok := c[comcfg.EmailPort]; ok {
if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 {
return fmt.Errorf("invalid %s", comcfg.EmailPort)
}
}
if ssl, ok := c[comcfg.EmailSSL]; ok && ssl != "0" && ssl != "1" {
return fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1")
}
if crt, ok := c[comcfg.ProjectCreationRestriction]; ok &&
crt != comcfg.ProCrtRestrEveryone &&
crt != comcfg.ProCrtRestrAdmOnly {
return fmt.Errorf("invalid %s, should be %s or %s",
comcfg.ProjectCreationRestriction,
comcfg.ProCrtRestrAdmOnly,
comcfg.ProCrtRestrEveryone)
}
if verify, ok := c[comcfg.VerifyRemoteCert]; ok && verify != "0" && verify != "1" {
return fmt.Errorf("invalid %s, should be %s or %s",
comcfg.VerifyRemoteCert, "0", "1")
}
return nil
}
/*
func convert() ([]*models.Config, error) {
cfgs := []*models.Config{}
var err error
pwdKeys := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD}
for _, pwdKey := range pwdKeys {
if pwd, ok := c[pwdKey]; ok && len(pwd) != 0 {
c[pwdKey], err = utils.ReversibleEncrypt(pwd, ui_cfg.SecretKey())
if err != nil {
return nil, err
}
}
}
for _, key := range configKeys {
if value, ok := c[key]; ok {
cfgs = append(cfgs, &models.Config{
Key: key,
Value: value,
})
}
}
return cfgs, nil
}
*/
/*
//[]*models.Config >> cfgForGet
func convert(cfg *config.Configuration) (map[string]interface{}, error) {
result := map[string]interface{}{}
for _, config := range configs {
cfg[config.Key] = &value{
Value: config.Value,
Editable: true,
}
}
dels := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD}
for _, del := range dels {
if _, ok := cfg[del]; ok {
delete(cfg, del)
}
}
flag, err := authModeCanBeModified()
if err != nil {
return nil, err
}
cfg[config.AUTH_MODE].Editable = flag
return cfgForGet(cfg), nil
}
*/
func authModeCanBeModified() (bool, error) {
return dao.AuthModeCanBeModified()
}

16
src/ui/api/harborapi_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/ui/config"
"github.com/vmware/harbor/tests/apitests/apilib"
// "strconv"
// "strings"
@ -57,7 +58,14 @@ type usrInfo struct {
}
func init() {
dao.InitDatabase()
if err := config.Init(); err != nil {
log.Fatalf("failed to initialize configurations: %v", err)
}
database, err := config.Database()
if err != nil {
log.Fatalf("failed to get database configurations: %v", err)
}
dao.InitDatabase(database)
_, file, _, _ := runtime.Caller(1)
apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator))))
beego.BConfig.WebConfig.Session.SessionOn = true
@ -513,7 +521,7 @@ func (a testapi) GetReposTop(authInfo usrInfo, count string) (int, error) {
//-------------------------Targets Test---------------------------------------//
//Create a new replication target
func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (int, error) {
func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (int, string, error) {
_sling := sling.New().Post(a.basePath)
path := "/api/targets"
@ -521,8 +529,8 @@ func (a testapi) AddTargets(authInfo usrInfo, repTarget apilib.RepTargetPost) (i
_sling = _sling.Path(path)
_sling = _sling.BodyJSON(repTarget)
httpStatusCode, _, err := request(_sling, jsonAcceptHeader, authInfo)
return httpStatusCode, err
httpStatusCode, body, err := request(_sling, jsonAcceptHeader, authInfo)
return httpStatusCode, string(body), err
}
//List filters targets by name

15
src/ui/api/project.go
View File

@ -44,7 +44,8 @@ type projectReq struct {
}
const projectNameMaxLen int = 30
const projectNameMinLen int = 4
const projectNameMinLen int = 2
const restrictedNameChars = `[a-z0-9]+(?:[._-][a-z0-9]+)*`
const dupProjectPattern = `Duplicate entry '\w+' for key 'name'`
// Prepare validates the URL and the user
@ -77,7 +78,13 @@ func (p *ProjectAPI) Post() {
if err != nil {
log.Errorf("Failed to check admin role: %v", err)
}
if !isSysAdmin && config.OnlyAdminCreateProject() {
onlyAdmin, err := config.OnlyAdminCreateProject()
if err != nil {
log.Errorf("failed to determine whether only admin can create projects: %v", err)
p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if !isSysAdmin && onlyAdmin {
log.Errorf("Only sys admin can create project")
p.RenderError(http.StatusForbidden, "Only system admin can create project")
return
@ -417,9 +424,9 @@ func isProjectAdmin(userID int, pid int64) bool {
func validateProjectReq(req projectReq) error {
pn := req.ProjectName
if isIllegalLength(req.ProjectName, projectNameMinLen, projectNameMaxLen) {
return fmt.Errorf("Project name is illegal in length. (greater than 4 or less than 30)")
return fmt.Errorf("Project name is illegal in length. (greater than 2 or less than 30)")
}
validProjectName := regexp.MustCompile(`^[a-z0-9](?:-*[a-z0-9])*(?:[._][a-z0-9](?:-*[a-z0-9])*)*$`)
validProjectName := regexp.MustCompile(`^` + restrictedNameChars + `$`)
legal := validProjectName.MatchString(pn)
if !legal {
return fmt.Errorf("project name is not in lower case or contains illegal characters")

21
src/ui/api/repository.go
View File

@ -361,11 +361,19 @@ func (ra *RepositoryAPI) GetManifests() {
}
func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) {
endpoint := config.InternalRegistryURL()
endpoint, err := config.RegistryURL()
if err != nil {
return nil, err
}
verify, err := config.VerifyRemoteCert()
if err != nil {
return nil, err
}
username, password, ok := ra.Ctx.Request.BasicAuth()
if ok {
return newRepositoryClient(endpoint, api.GetIsInsecure(), username, password,
return newRepositoryClient(endpoint, !verify, username, password,
repoName, "repository", repoName, "pull", "push", "*")
}
@ -374,7 +382,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo
return nil, err
}
return cache.NewRepositoryClient(endpoint, api.GetIsInsecure(), username, repoName,
return cache.NewRepositoryClient(endpoint, !verify, username, repoName,
"repository", repoName, "pull", "push", "*")
}
@ -416,7 +424,12 @@ func (ra *RepositoryAPI) GetTopRepos() {
ra.CustomAbort(http.StatusBadRequest, "invalid count")
}
repos, err := dao.GetTopRepos(count)
userID, _, ok := ra.GetUserIDForRequest()
if !ok {
userID = dao.NonExistUserID
}
repos, err := dao.GetTopRepos(userID, count)
if err != nil {
log.Errorf("failed to get top repos: %v", err)
ra.CustomAbort(http.StatusInternalServerError, "internal server error")

14
src/ui/api/target.go
View File

@ -41,7 +41,12 @@ type TargetAPI struct {
// Prepare validates the user
func (t *TargetAPI) Prepare() {
t.secretKey = config.SecretKey()
var err error
t.secretKey, err = config.SecretKey()
if err != nil {
log.Errorf("failed to get secret key: %v", err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
userID := t.ValidateUser()
isSysAdmin, err := dao.IsAdminRole(userID)
@ -97,7 +102,12 @@ func (t *TargetAPI) Ping() {
password = t.GetString("password")
}
registry, err := newRegistryClient(endpoint, api.GetIsInsecure(), username, password,
verify, err := config.VerifyRemoteCert()
if err != nil {
log.Errorf("failed to check whether insecure or not: %v", err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
registry, err := newRegistryClient(endpoint, !verify, username, password,
"", "", "")
if err != nil {
// timeout, dns resolve error, connection refused, etc.

9
src/ui/api/target_test.go
View File

@ -30,17 +30,18 @@ func TestTargetsPost(t *testing.T) {
//-------------------case 1 : response code = 201------------------------//
fmt.Println("case 1 : response code = 201")
httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets)
httpStatusCode, body, err := apiTest.AddTargets(*admin, *repTargets)
if err != nil {
t.Error("Error whihle add targets", err.Error())
t.Log(err)
} else {
assert.Equal(int(201), httpStatusCode, "httpStatusCode should be 201")
t.Log(body)
}
//-----------case 2 : response code = 409,name is already used-----------//
fmt.Println("case 2 : response code = 409,name is already used")
httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets)
httpStatusCode, _, err = apiTest.AddTargets(*admin, *repTargets)
if err != nil {
t.Error("Error whihle add targets", err.Error())
t.Log(err)
@ -51,7 +52,7 @@ func TestTargetsPost(t *testing.T) {
//-----------case 3 : response code = 409,name is already used-----------//
fmt.Println("case 3 : response code = 409,endPoint is already used")
repTargets.Username = "errName"
httpStatusCode, err = apiTest.AddTargets(*admin, *repTargets)
httpStatusCode, _, err = apiTest.AddTargets(*admin, *repTargets)
if err != nil {
t.Error("Error whihle add targets", err.Error())
t.Log(err)
@ -61,7 +62,7 @@ func TestTargetsPost(t *testing.T) {
//--------case 4 : response code = 401,User need to log in first.--------//
fmt.Println("case 4 : response code = 401,User need to log in first.")
httpStatusCode, err = apiTest.AddTargets(*unknownUsr, *repTargets)
httpStatusCode, _, err = apiTest.AddTargets(*unknownUsr, *repTargets)
if err != nil {
t.Error("Error whihle add targets", err.Error())
t.Log(err)

18
src/ui/api/user.go
View File

@ -46,10 +46,21 @@ type passwordReq struct {
// Prepare validates the URL and parms
func (ua *UserAPI) Prepare() {
mode, err := config.AuthMode()
if err != nil {
log.Errorf("failed to get auth mode: %v", err)
ua.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
ua.AuthMode = config.AuthMode()
ua.AuthMode = mode
ua.SelfRegistration = config.SelfRegistration()
self, err := config.SelfRegistration()
if err != nil {
log.Errorf("failed to get self registration: %v", err)
ua.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
ua.SelfRegistration = self
if ua.Ctx.Input.IsPost() {
sessionUserID := ua.GetSession("userId")
@ -82,7 +93,6 @@ func (ua *UserAPI) Prepare() {
}
}
var err error
ua.IsAdmin, err = dao.IsAdminRole(ua.currentUserID)
if err != nil {
log.Errorf("Error occurred in IsAdminRole:%v", err)
@ -234,7 +244,7 @@ func (ua *UserAPI) Delete() {
return
}
if config.AuthMode() == "ldap_auth" {
if ua.AuthMode == "ldap_auth" {
ua.CustomAbort(http.StatusForbidden, "user can not be deleted in LDAP authentication mode")
}

46
src/ui/api/utils.go
View File

@ -20,11 +20,9 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
"sort"
"strings"
"time"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
@ -242,7 +240,7 @@ func addAuthentication(req *http.Request) {
// SyncRegistry syncs the repositories of registry with database.
func SyncRegistry() error {
log.Debugf("Start syncing repositories from registry to DB... ")
log.Infof("Start syncing repositories from registry to DB... ")
reposInRegistry, err := catalog()
if err != nil {
@ -304,7 +302,7 @@ func SyncRegistry() error {
}
}
log.Debugf("Sync repositories from registry to DB is done.")
log.Infof("Sync repositories from registry to DB is done.")
return nil
}
@ -350,7 +348,10 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string
}
// TODO remove the workaround when the bug of registry is fixed
endpoint := config.InternalRegistryURL()
endpoint, err := config.RegistryURL()
if err != nil {
return needsAdd, needsDel, err
}
client, err := cache.NewRepositoryClient(endpoint, true,
"admin", repoInR, "repository", repoInR)
if err != nil {
@ -372,7 +373,10 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string
j++
} else {
// TODO remove the workaround when the bug of registry is fixed
endpoint := config.InternalRegistryURL()
endpoint, err := config.RegistryURL()
if err != nil {
return needsAdd, needsDel, err
}
client, err := cache.NewRepositoryClient(endpoint, true,
"admin", repoInR, "repository", repoInR)
if err != nil {
@ -422,32 +426,18 @@ func projectExists(repository string) (bool, error) {
}
func initRegistryClient() (r *registry.Registry, err error) {
endpoint := config.InternalRegistryURL()
endpoint, err := config.RegistryURL()
if err != nil {
return nil, err
}
addr := endpoint
if strings.Contains(endpoint, "/") {
addr = endpoint[strings.LastIndex(endpoint, "/")+1:]
if strings.Contains(endpoint, "://") {
addr = strings.Split(endpoint, "://")[1]
}
ch := make(chan int, 1)
go func() {
var err error
var c net.Conn
for {
c, err = net.DialTimeout("tcp", addr, 20*time.Second)
if err == nil {
c.Close()
ch <- 1
} else {
log.Errorf("failed to connect to registry client, retry after 2 seconds :%v", err)
time.Sleep(2 * time.Second)
}
}
}()
select {
case <-ch:
case <-time.After(60 * time.Second):
panic("Failed to connect to registry client after 60 seconds")
if err := utils.TestTCPConn(addr, 60, 2); err != nil {
return nil, err
}
registryClient, err := cache.NewRegistryClient(endpoint, true, "admin",

5
src/ui/auth/authenticator.go
View File

@ -50,7 +50,10 @@ func Register(name string, authenticator Authenticator) {
// Login authenticates user credentials based on setting.
func Login(m models.AuthModel) (*models.User, error) {
var authMode = config.AuthMode()
authMode, err := config.AuthMode()
if err != nil {
return nil, err
}
if authMode == "" || m.Principal == "admin" {
authMode = "db_auth"
}

40
src/ui/auth/ldap/ldap.go
View File

@ -16,18 +16,15 @@
package ldap
import (
"crypto/tls"
"errors"
"fmt"
"strconv"
"strings"
"time"
"crypto/tls"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/auth"
"github.com/vmware/harbor/src/ui/config"
@ -41,9 +38,8 @@ const metaChars = "&|!=~*<>()"
// Connect checks the LDAP configuration directives, and connects to the LDAP URL
// Returns an LDAP connection
func Connect() (*goldap.Conn, error) {
ldapURL := config.LDAP().URL
func Connect(settings *models.LDAP) (*goldap.Conn, error) {
ldapURL := settings.URL
if ldapURL == "" {
return nil, errors.New("can not get any available LDAP_URL")
}
@ -70,13 +66,10 @@ func Connect() (*goldap.Conn, error) {
}
// Sets a Dial Timeout for LDAP
cTimeout := config.LDAP().ConnectTimeout
connectTimeout, _ := strconv.Atoi(cTimeout)
goldap.DefaultTimeout = time.Duration(connectTimeout) * time.Second
goldap.DefaultTimeout = time.Duration(settings.Timeout) * time.Second
var ldap *goldap.Conn
var err error
switch protocol {
case "ldap":
ldap, err = goldap.Dial("tcp", fmt.Sprintf("%s:%s", host, port))
@ -104,21 +97,26 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) {
}
}
ldap, err := Connect()
settings, err := config.LDAP()
if err != nil {
return nil, err
}
ldapBaseDn := config.LDAP().BaseDn
ldap, err := Connect(settings)
if err != nil {
return nil, err
}
ldapBaseDn := settings.BaseDN
if ldapBaseDn == "" {
return nil, errors.New("can not get any available LDAP_BASE_DN")
}
log.Debug("baseDn:", ldapBaseDn)
ldapSearchDn := config.LDAP().SearchDn
ldapSearchDn := settings.SearchDN
if ldapSearchDn != "" {
log.Debug("Search DN: ", ldapSearchDn)
ldapSearchPwd := config.LDAP().SearchPwd
ldapSearchPwd := settings.SearchPwd
err = ldap.Bind(ldapSearchDn, ldapSearchPwd)
if err != nil {
log.Debug("Bind search dn error", err)
@ -126,8 +124,8 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) {
}
}
attrName := config.LDAP().UID
filter := config.LDAP().Filter
attrName := settings.UID
filter := settings.Filter
if filter != "" {
filter = "(&" + filter + "(" + attrName + "=" + m.Principal + "))"
} else {
@ -135,11 +133,11 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) {
}
log.Debug("one or more filter", filter)
ldapScope := config.LDAP().Scope
ldapScope := settings.Scope
var scope int
if ldapScope == "1" {
if ldapScope == 1 {
scope = goldap.ScopeBaseObject
} else if ldapScope == "2" {
} else if ldapScope == 2 {
scope = goldap.ScopeSingleLevel
} else {
scope = goldap.ScopeWholeSubtree

257
src/ui/config/config.go
View File

@ -13,145 +13,220 @@
limitations under the License.
*/
// Package config provides methods to get configurations required by code in src/ui
package config
import (
"strconv"
"strings"
"encoding/json"
"os"
commonConfig "github.com/vmware/harbor/src/common/config"
comcfg "github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
)
// LDAPSetting wraps the setting of an LDAP server
type LDAPSetting struct {
URL string
BaseDn string
SearchDn string
SearchPwd string
UID string
Filter string
Scope string
ConnectTimeout string
var mg *comcfg.Manager
// Configuration of UI
type Configuration struct {
DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port
Authentication *models.Authentication `json:"authentication"`
Database *models.Database `json:"database"`
TokenService *models.TokenService `json:"token_service"`
Registry *models.Registry `json:"registry"`
Email *models.Email `json:"email"`
VerifyRemoteCert bool `json:"verify_remote_cert"`
ProjectCreationRestriction string `json:"project_creation_restriction"`
InitialAdminPwd string `json:"initial_admin_pwd"`
//TODO remove
CompressJS bool `json:"compress_js"`
TokenExpiration int `json:"token_expiration"`
SecretKey string `json:"secret_key"`
CfgExpiration int `json:"cfg_expiration"`
}
type uiParser struct{}
type parser struct {
}
// Parse parses the auth settings url settings and other configuration consumed by code under src/ui
func (up *uiParser) Parse(raw map[string]string, config map[string]interface{}) error {
mode := raw["AUTH_MODE"]
if mode == "ldap_auth" {
setting := LDAPSetting{
URL: raw["LDAP_URL"],
BaseDn: raw["LDAP_BASE_DN"],
SearchDn: raw["LDAP_SEARCH_DN"],
SearchPwd: raw["LDAP_SEARCH_PWD"],
UID: raw["LDAP_UID"],
Filter: raw["LDAP_FILTER"],
Scope: raw["LDAP_SCOPE"],
ConnectTimeout: raw["LDAP_CONNECT_TIMEOUT"],
func (p *parser) Parse(b []byte) (interface{}, error) {
c := &Configuration{}
if err := json.Unmarshal(b, c); err != nil {
return nil, err
}
config["ldap"] = setting
return c, nil
}
config["auth_mode"] = mode
var tokenExpiration = 30 //minutes
if len(raw["TOKEN_EXPIRATION"]) > 0 {
i, err := strconv.Atoi(raw["TOKEN_EXPIRATION"])
if err != nil {
log.Warningf("failed to parse token expiration: %v, using default value %d", err, tokenExpiration)
} else if i <= 0 {
log.Warningf("invalid token expiration, using default value: %d minutes", tokenExpiration)
} else {
tokenExpiration = i
// Init configurations
func Init() error {
adminServerURL := os.Getenv("ADMIN_SERVER_URL")
if len(adminServerURL) == 0 {
adminServerURL = "http://adminserver"
}
log.Debugf("admin server URL: %s", adminServerURL)
mg = comcfg.NewManager(adminServerURL, UISecret(), &parser{}, true)
if err := mg.Init(); err != nil {
return err
}
config["token_exp"] = tokenExpiration
config["admin_password"] = raw["HARBOR_ADMIN_PASSWORD"]
config["ext_reg_url"] = raw["EXT_REG_URL"]
config["ui_secret"] = raw["UI_SECRET"]
config["secret_key"] = raw["SECRET_KEY"]
config["self_registration"] = raw["SELF_REGISTRATION"] != "off"
config["admin_create_project"] = strings.ToLower(raw["PROJECT_CREATION_RESTRICTION"]) == "adminonly"
registryURL := raw["REGISTRY_URL"]
registryURL = strings.TrimRight(registryURL, "/")
config["internal_registry_url"] = registryURL
jobserviceURL := raw["JOB_SERVICE_URL"]
jobserviceURL = strings.TrimRight(jobserviceURL, "/")
config["internal_jobservice_url"] = jobserviceURL
if _, err := mg.Load(); err != nil {
return err
}
return nil
}
var uiConfig *commonConfig.Config
func init() {
uiKeys := []string{"AUTH_MODE", "LDAP_URL", "LDAP_BASE_DN", "LDAP_SEARCH_DN", "LDAP_SEARCH_PWD", "LDAP_UID", "LDAP_FILTER", "LDAP_SCOPE", "LDAP_CONNECT_TIMEOUT", "TOKEN_EXPIRATION", "HARBOR_ADMIN_PASSWORD", "EXT_REG_URL", "UI_SECRET", "SECRET_KEY", "SELF_REGISTRATION", "PROJECT_CREATION_RESTRICTION", "REGISTRY_URL", "JOB_SERVICE_URL"}
uiConfig = &commonConfig.Config{
Config: make(map[string]interface{}),
Loader: &commonConfig.EnvConfigLoader{Keys: uiKeys},
Parser: &uiParser{},
}
if err := uiConfig.Load(); err != nil {
panic(err)
func get() (*Configuration, error) {
c, err := mg.Get()
if err != nil {
return nil, err
}
return c.(*Configuration), nil
}
// Reload ...
func Reload() error {
return uiConfig.Load()
// Load configurations
func Load() error {
_, err := mg.Load()
return err
}
// Upload uploads all system configutations to admin server
func Upload(cfg map[string]string) error {
b, err := json.Marshal(cfg)
if err != nil {
return err
}
return mg.Upload(b)
}
// GetSystemCfg returns the system configurations
func GetSystemCfg() (*models.SystemCfg, error) {
raw, err := mg.Loader.Load()
if err != nil {
return nil, err
}
cfg := &models.SystemCfg{}
if err = json.Unmarshal(raw, cfg); err != nil {
return nil, err
}
return cfg, nil
}
// AuthMode ...
func AuthMode() string {
return uiConfig.Config["auth_mode"].(string)
func AuthMode() (string, error) {
cfg, err := get()
if err != nil {
return "", err
}
return cfg.Authentication.Mode, nil
}
// LDAP returns the setting of ldap server
func LDAP() LDAPSetting {
return uiConfig.Config["ldap"].(LDAPSetting)
func LDAP() (*models.LDAP, error) {
cfg, err := get()
if err != nil {
return nil, err
}
return cfg.Authentication.LDAP, nil
}
// TokenExpiration returns the token expiration time (in minute)
func TokenExpiration() int {
return uiConfig.Config["token_exp"].(int)
func TokenExpiration() (int, error) {
cfg, err := get()
if err != nil {
return 0, err
}
return cfg.TokenExpiration, nil
}
// ExtRegistryURL returns the registry URL to exposed to external client
func ExtRegistryURL() string {
return uiConfig.Config["ext_reg_url"].(string)
// DomainName returns the external URL of Harbor: protocal://host:port
func DomainName() (string, error) {
cfg, err := get()
if err != nil {
return "", err
}
// UISecret returns the value of UI secret cookie, used for communication between UI and JobService
func UISecret() string {
return uiConfig.Config["ui_secret"].(string)
return cfg.DomainName, nil
}
// SecretKey returns the secret key to encrypt the password of target
func SecretKey() string {
return uiConfig.Config["secret_key"].(string)
func SecretKey() (string, error) {
cfg, err := get()
if err != nil {
return "", err
}
return cfg.SecretKey, nil
}
// SelfRegistration returns the enablement of self registration
func SelfRegistration() bool {
return uiConfig.Config["self_registration"].(bool)
func SelfRegistration() (bool, error) {
cfg, err := get()
if err != nil {
return false, err
}
return cfg.Authentication.SelfRegistration, nil
}
// InternalRegistryURL returns registry URL for internal communication between Harbor containers
func InternalRegistryURL() string {
return uiConfig.Config["internal_registry_url"].(string)
// RegistryURL ...
func RegistryURL() (string, error) {
cfg, err := get()
if err != nil {
return "", err
}
return cfg.Registry.URL, nil
}
// InternalJobServiceURL returns jobservice URL for internal communication between Harbor containers
func InternalJobServiceURL() string {
return uiConfig.Config["internal_jobservice_url"].(string)
return "http://jobservice"
}
// InitialAdminPassword returns the initial password for administrator
func InitialAdminPassword() string {
return uiConfig.Config["admin_password"].(string)
func InitialAdminPassword() (string, error) {
cfg, err := get()
if err != nil {
return "", err
}
return cfg.InitialAdminPwd, nil
}
// OnlyAdminCreateProject returns the flag to restrict that only sys admin can create project
func OnlyAdminCreateProject() bool {
return uiConfig.Config["admin_create_project"].(bool)
func OnlyAdminCreateProject() (bool, error) {
cfg, err := get()
if err != nil {
return true, err
}
return cfg.ProjectCreationRestriction == comcfg.ProCrtRestrAdmOnly, nil
}
// VerifyRemoteCert returns bool value.
func VerifyRemoteCert() (bool, error) {
cfg, err := get()
if err != nil {
return true, err
}
return cfg.VerifyRemoteCert, nil
}
// Email returns email server settings
func Email() (*models.Email, error) {
cfg, err := get()
if err != nil {
return nil, err
}
return cfg.Email, nil
}
// Database returns database settings
func Database() (*models.Database, error) {
cfg, err := get()
if err != nil {
return nil, err
}
return cfg.Database, nil
}
// UISecret returns the value of UI secret cookie, used for communication between UI and JobService
// TODO
func UISecret() string {
return os.Getenv("UI_SECRET")
}

171
src/ui/config/config_test.go
View File

@ -17,131 +17,92 @@ package config
import (
"os"
"testing"
"github.com/vmware/harbor/src/common/utils/test"
)
var (
auth = "ldap_auth"
ldap = LDAPSetting{
"ldap://test.ldap.com",
"ou=people",
"dc=whatever,dc=org",
"1234567",
"cn",
"uid",
"2",
"5",
}
tokenExp = "3"
tokenExpRes = 3
adminPassword = "password"
externalRegURL = "127.0.0.1"
uiSecret = "ffadsdfsdf"
secretKey = "keykey"
selfRegistration = "off"
projectCreationRestriction = "adminonly"
internalRegistryURL = "http://registry:5000"
jobServiceURL = "http://jobservice"
)
func TestMain(m *testing.M) {
os.Setenv("AUTH_MODE", auth)
os.Setenv("LDAP_URL", ldap.URL)
os.Setenv("LDAP_BASE_DN", ldap.BaseDn)
os.Setenv("LDAP_SEARCH_DN", ldap.SearchDn)
os.Setenv("LDAP_SEARCH_PWD", ldap.SearchPwd)
os.Setenv("LDAP_UID", ldap.UID)
os.Setenv("LDAP_SCOPE", ldap.Scope)
os.Setenv("LDAP_FILTER", ldap.Filter)
os.Setenv("LDAP_CONNECT_TIMEOUT", ldap.ConnectTimeout)
os.Setenv("TOKEN_EXPIRATION", tokenExp)
os.Setenv("HARBOR_ADMIN_PASSWORD", adminPassword)
os.Setenv("EXT_REG_URL", externalRegURL)
os.Setenv("UI_SECRET", uiSecret)
os.Setenv("SECRET_KEY", secretKey)
os.Setenv("SELF_REGISTRATION", selfRegistration)
os.Setenv("PROJECT_CREATION_RESTRICTION", projectCreationRestriction)
os.Setenv("REGISTRY_URL", internalRegistryURL)
os.Setenv("JOB_SERVICE_URL", jobServiceURL)
err := Reload()
// test functions under package ui/config
func TestConfig(t *testing.T) {
server, err := test.NewAdminserver()
if err != nil {
panic(err)
t.Fatalf("failed to create a mock admin server: %v", err)
}
rc := m.Run()
defer server.Close()
os.Unsetenv("AUTH_MODE")
os.Unsetenv("LDAP_URL")
os.Unsetenv("LDAP_BASE_DN")
os.Unsetenv("LDAP_SEARCH_DN")
os.Unsetenv("LDAP_SEARCH_PWD")
os.Unsetenv("LDAP_UID")
os.Unsetenv("LDAP_SCOPE")
os.Unsetenv("LDAP_FILTER")
os.Unsetenv("LDAP_CONNECT_TIMEOUT")
os.Unsetenv("TOKEN_EXPIRATION")
os.Unsetenv("HARBOR_ADMIN_PASSWORD")
os.Unsetenv("EXT_REG_URL")
os.Unsetenv("UI_SECRET")
os.Unsetenv("SECRET_KEY")
os.Unsetenv("SELF_REGISTRATION")
os.Unsetenv("CREATE_PROJECT_RESTRICTION")
os.Unsetenv("REGISTRY_URL")
os.Unsetenv("JOB_SERVICE_URL")
url := os.Getenv("ADMIN_SERVER_URL")
defer os.Setenv("ADMIN_SERVER_URL", url)
os.Exit(rc)
if err := os.Setenv("ADMIN_SERVER_URL", server.URL); err != nil {
t.Fatalf("failed to set env %s: %v", "ADMIN_SERVER_URL", err)
}
func TestAuth(t *testing.T) {
if AuthMode() != auth {
t.Errorf("Expected auth mode:%s, in fact: %s", auth, AuthMode())
}
if LDAP() != ldap {
t.Errorf("Expected ldap setting: %+v, in fact: %+v", ldap, LDAP())
}
if err := Init(); err != nil {
t.Fatalf("failed to initialize configurations: %v", err)
}
func TestTokenExpiration(t *testing.T) {
if TokenExpiration() != tokenExpRes {
t.Errorf("Expected token expiration: %d, in fact: %d", tokenExpRes, TokenExpiration())
}
if err := Load(); err != nil {
t.Fatalf("failed to load configurations: %v", err)
}
func TestURLs(t *testing.T) {
if InternalRegistryURL() != internalRegistryURL {
t.Errorf("Expected internal Registry URL: %s, in fact: %s", internalRegistryURL, InternalRegistryURL())
}
if InternalJobServiceURL() != jobServiceURL {
t.Errorf("Expected internal jobservice URL: %s, in fact: %s", jobServiceURL, InternalJobServiceURL())
}
if ExtRegistryURL() != externalRegURL {
t.Errorf("Expected External Registry URL: %s, in fact: %s", externalRegURL, ExtRegistryURL())
}
if err := Upload(map[string]string{}); err != nil {
t.Fatalf("failed to upload configurations: %v", err)
}
func TestSelfRegistration(t *testing.T) {
if SelfRegistration() {
t.Errorf("Expected Self Registration to be false")
}
if _, err := GetSystemCfg(); err != nil {
t.Fatalf("failed to get system configurations: %v", err)
}
func TestSecrets(t *testing.T) {
if SecretKey() != secretKey {
t.Errorf("Expected Secrect Key :%s, in fact: %s", secretKey, SecretKey())
}
if UISecret() != uiSecret {
t.Errorf("Expected UI Secret: %s, in fact: %s", uiSecret, UISecret())
mode, err := AuthMode()
if err != nil {
t.Fatalf("failed to get auth mode: %v", err)
}
if mode != "db_auth" {
t.Errorf("unexpected mode: %s != %s", mode, "db_auth")
}
func TestProjectCreationRestrict(t *testing.T) {
if !OnlyAdminCreateProject() {
t.Errorf("Expected OnlyAdminCreateProject to be true")
}
if _, err := LDAP(); err != nil {
t.Fatalf("failed to get ldap settings: %v", err)
}
func TestInitAdminPassword(t *testing.T) {
if InitialAdminPassword() != adminPassword {
t.Errorf("Expected adminPassword: %s, in fact: %s", adminPassword, InitialAdminPassword())
if _, err := TokenExpiration(); err != nil {
t.Fatalf("failed to get token expiration: %v", err)
}
if _, err := DomainName(); err != nil {
t.Fatalf("failed to get domain name: %v", err)
}
if _, err := SecretKey(); err != nil {
t.Fatalf("failed to get secret key: %v", err)
}
if _, err := SelfRegistration(); err != nil {
t.Fatalf("failed to get self registration: %v", err)
}
if _, err := RegistryURL(); err != nil {
t.Fatalf("failed to get registry URL: %v", err)
}
InternalJobServiceURL()
InitialAdminPassword()
if _, err := OnlyAdminCreateProject(); err != nil {
t.Fatalf("failed to get onldy admin create project: %v", err)
}
if _, err := VerifyRemoteCert(); err != nil {
t.Fatalf("failed to get verify remote cert: %v", err)
}
if _, err := Email(); err != nil {
t.Fatalf("failed to get email settings: %v", err)
}
if _, err := Database(); err != nil {
t.Fatalf("failed to get database: %v", err)
}
UISecret()
}

21
src/ui/controllers/base.go
View File

@ -103,7 +103,12 @@ func (b *BaseController) Prepare() {
b.Data["CurLang"] = curLang.Name
b.Data["RestLangs"] = restLangs
authMode := config.AuthMode()
authMode, err := config.AuthMode()
if err != nil {
log.Errorf("failed to get auth mode: %v", err)
b.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if authMode == "" {
authMode = "db_auth"
}
@ -120,9 +125,13 @@ func (b *BaseController) Prepare() {
b.UseCompressedJS = false
}
b.SelfRegistration = config.SelfRegistration()
b.SelfRegistration, err = config.SelfRegistration()
if err != nil {
log.Errorf("failed to get self registration: %v", err)
b.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
b.Data["SelfRegistration"] = config.SelfRegistration()
b.Data["SelfRegistration"] = b.SelfRegistration
sessionUserID := b.GetSession("userId")
if sessionUserID != nil {
@ -235,12 +244,13 @@ func (cc *CommonController) UserExists() {
}
func init() {
//conf/app.conf -> os.Getenv("config_path")
configPath := os.Getenv("CONFIG_PATH")
if len(configPath) != 0 {
log.Infof("Config path: %s", configPath)
beego.LoadAppConfig("ini", configPath)
if err := beego.LoadAppConfig("ini", configPath); err != nil {
log.Errorf("failed to load app config: %v", err)
}
}
beego.AddFuncMap("i18n", i18n.Tr)
@ -263,5 +273,4 @@ func init() {
log.Errorf("Fail to set message file: %s", err.Error())
}
}
}

6
src/ui/controllers/controllers_test.go
View File

@ -14,6 +14,8 @@ import (
"github.com/astaxie/beego"
//"github.com/dghubble/sling"
"github.com/stretchr/testify/assert"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/config"
)
//const (
@ -29,6 +31,9 @@ import (
//var admin *usrInfo
func init() {
if err := config.Init(); err != nil {
log.Fatalf("failed to initialize configurations: %v", err)
}
_, file, _, _ := runtime.Caller(1)
apppath, _ := filepath.Abs(filepath.Dir(filepath.Join(file, ".."+string(filepath.Separator))))
@ -63,7 +68,6 @@ func init() {
//Init user Info
//admin = &usrInfo{adminName, adminPwd}
}
// TestMain is a sample to run an endpoint test

18
src/ui/controllers/password.go
View File

@ -6,12 +6,12 @@ import (
"regexp"
"text/template"
"github.com/astaxie/beego"
"github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils"
email_util "github.com/vmware/harbor/src/common/utils/email"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/config"
)
type messageDetail struct {
@ -49,7 +49,11 @@ func (cc *CommonController) SendEmail() {
message := new(bytes.Buffer)
harborURL := config.ExtEndpoint()
harborURL, err := config.DomainName()
if err != nil {
log.Errorf("failed to get domain name: %v", err)
cc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if harborURL == "" {
harborURL = "localhost"
}
@ -65,14 +69,14 @@ func (cc *CommonController) SendEmail() {
cc.CustomAbort(http.StatusInternalServerError, "internal_error")
}
config, err := beego.AppConfig.GetSection("mail")
emailSettings, err := config.Email()
if err != nil {
log.Errorf("Can not load app.conf: %v", err)
log.Errorf("failed to get email configurations: %v", err)
cc.CustomAbort(http.StatusInternalServerError, "internal_error")
}
mail := utils.Mail{
From: config["from"],
mail := email_util.Mail{
From: emailSettings.From,
To: []string{email},
Subject: cc.Tr("reset_email_subject"),
Message: message.String()}

9
src/ui/controllers/project.go
View File

@ -1,6 +1,8 @@
package controllers
import (
"net/http"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/config"
@ -23,6 +25,11 @@ func (pc *ProjectController) Get() {
isSysAdmin = false
}
}
pc.Data["CanCreate"] = !config.OnlyAdminCreateProject() || isSysAdmin
onlyAdmin, err := config.OnlyAdminCreateProject()
if err != nil {
log.Errorf("failed to determine whether only admin can create projects: %v", err)
pc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
pc.Data["CanCreate"] = !onlyAdmin || isSysAdmin
pc.Forward("page_title_project", "project.htm")
}

11
src/ui/controllers/repository.go
View File

@ -1,6 +1,10 @@
package controllers
import (
"net/http"
"strings"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/config"
)
@ -11,6 +15,11 @@ type RepositoryController struct {
// Get renders repository page
func (rc *RepositoryController) Get() {
rc.Data["HarborRegUrl"] = config.ExtRegistryURL()
url, err := config.DomainName()
if err != nil {
log.Errorf("failed to get domain name: %v", err)
rc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
rc.Data["HarborRegUrl"] = strings.Split(url, "://")[1]
rc.Forward("page_title_repository", "repository.htm")
}

23
src/ui/main.go
View File

@ -64,7 +64,6 @@ func updateInitPassword(userID int, password string) error {
}
func main() {
beego.BConfig.WebConfig.Session.SessionOn = true
//TODO
redisURL := os.Getenv("_REDIS_URL")
@ -72,12 +71,28 @@ func main() {
beego.BConfig.WebConfig.Session.SessionProvider = "redis"
beego.BConfig.WebConfig.Session.SessionProviderConfig = redisURL
}
//
beego.AddTemplateExt("htm")
dao.InitDatabase()
log.Info("initializing configurations...")
if err := config.Init(); err != nil {
log.Fatalf("failed to initialize configurations: %v", err)
}
log.Info("configurations initialization completed")
if err := updateInitPassword(adminUserID, config.InitialAdminPassword()); err != nil {
database, err := config.Database()
if err != nil {
log.Fatalf("failed to get database configuration: %v", err)
}
if err := dao.InitDatabase(database); err != nil {
log.Fatalf("failed to initialize database: %v", err)
}
password, err := config.InitialAdminPassword()
if err != nil {
log.Fatalf("failed to get admin's initia password: %v", err)
}
if err := updateInitPassword(adminUserID, password); err != nil {
log.Error(err)
}
initRouters()

1
src/ui/router.go
View File

@ -84,6 +84,7 @@ func initRouters() {
beego.Router("/api/users/:id/sysadmin", &api.UserAPI{}, "put:ToggleUserAdminRole")
beego.Router("/api/repositories/top", &api.RepositoryAPI{}, "get:GetTopRepos")
beego.Router("/api/logs", &api.LogAPI{})
beego.Router("/api/configurations", &api.ConfigAPI{})
beego.Router("/api/systeminfo/volumes", &api.SystemInfoAPI{}, "get:GetVolumeInfo")
beego.Router("/api/systeminfo/getcert", &api.SystemInfoAPI{}, "get:GetCert")

19
src/ui/service/token/authutils.go
View File

@ -37,13 +37,6 @@ const (
privateKey = "/etc/ui/private_key.pem"
)
var expiration int //minutes
func init() {
expiration = config.TokenExpiration()
log.Infof("token expiration: %d minutes", expiration)
}
// GetResourceActions ...
func GetResourceActions(scopes []string) []*token.ResourceActions {
log.Debugf("scopes: %+v", scopes)
@ -91,7 +84,12 @@ func FilterAccess(username string, a *token.ResourceActions) {
repoLength := len(repoSplit)
if repoLength > 1 { //Only check the permission when the requested image has a namespace, i.e. project
var projectName string
registryURL := config.ExtRegistryURL()
registryURL, err := config.DomainName()
if err != nil {
log.Errorf("failed to get domain name: %v", err)
return
}
registryURL = strings.Split(registryURL, "://")[1]
if repoSplit[0] == registryURL {
projectName = repoSplit[1]
log.Infof("Detected Registry URL in Project Name. Assuming this is a notary request and setting Project Name as %s\n", projectName)
@ -153,6 +151,11 @@ func MakeToken(username, service string, access []*token.ResourceActions) (token
if err != nil {
return "", 0, nil, err
}
expiration, err := config.TokenExpiration()
if err != nil {
return "", 0, nil, err
}
tk, expiresIn, issuedAt, err := makeTokenCore(issuer, username, service, expiration, access, pk)
if err != nil {
return "", 0, nil, err

2
src/ui/ui_test.go
View File

@ -1,5 +1,6 @@
package main
/*
import (
"testing"
)
@ -7,3 +8,4 @@ import (
func TestMain(t *testing.T) {
}
*/

11
tests/docker-compose.test.yml
View File

@ -21,6 +21,17 @@ services:
- ./common/config/db/env
ports:
- 3306:3306
adminserver:
build:
context: ../
dockerfile: make/dev/adminserver/Dockerfile
env_file:
- ./common/config/adminserver/env
restart: always
volumes:
- /data/config/:/etc/harbor/
ports:
- 8888:80
ldap:
image: osixia/openldap:1.1.7
restart: always

5
tests/startuptest.go
View File

@ -25,12 +25,15 @@ func main() {
os.Exit(1)
}
// fmt.Printf("%s", b)
if strings.Contains(string(b), "Harbor") {
fmt.Printf("sucess!\n")
} else {
fmt.Println("the response does not contain \"Harbor\"!")
fmt.Println(string(b))
os.Exit(1)
}
}
}

6
tests/testprepare.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash
set -e
cp tests/docker-compose.test.yml make/.
mkdir /etc/ui
@ -7,3 +7,7 @@ cp make/common/config/ui/private_key.pem /etc/ui/.
mkdir conf
cp make/common/config/ui/app.conf conf/.
sed -i -r "s/MYSQL_HOST=mysql/MYSQL_HOST=127.0.0.1/" make/common/config/adminserver/env
sed -i -r "s|REGISTRY_URL=http://registry:5000|REGISTRY_URL=http://127.0.0.1:5000|" make/common/config/adminserver/env
sed -i -r "s/UI_SECRET=.*/UI_SECRET=$UI_SECRET/" make/common/config/adminserver/env