Merge pull request #3736 from yixingjia/HA

HA installation script
This commit is contained in:
yixingjia 2017-12-12 16:27:28 +08:00 committed by GitHub
commit ec269047c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 421 additions and 17 deletions

View File

@ -230,13 +230,14 @@ PACKAGE_OFFLINE_PARA=-zcvf harbor-offline-installer-$(GITTAGVERSION).tgz \
$(HARBORPKG)/prepare $(HARBORPKG)/NOTICE \
$(HARBORPKG)/upgrade $(HARBORPKG)/harbor_1_1_0_template \
$(HARBORPKG)/LICENSE $(HARBORPKG)/install.sh \
$(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME)
$(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
$(HARBORPKG)/ha
PACKAGE_ONLINE_PARA=-zcvf harbor-online-installer-$(GITTAGVERSION).tgz \
$(HARBORPKG)/common/templates $(HARBORPKG)/prepare \
$(HARBORPKG)/LICENSE $(HARBORPKG)/NOTICE \
$(HARBORPKG)/upgrade $(HARBORPKG)/harbor_1_1_0_template \
$(HARBORPKG)/install.sh $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
$(HARBORPKG)/harbor.cfg
$(HARBORPKG)/harbor.cfg $(HARBORPKG)/ha
DOCKERCOMPOSE_LIST=-f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
ifeq ($(NOTARYFLAG), true)
@ -327,7 +328,9 @@ build: build_$(BASEIMAGE)
modify_composefile:
@echo "preparing docker-compose file..."
@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
@cp $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME)
@$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
@$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME)
modify_sourcefiles:
@echo "change mode of source files."
@ -345,6 +348,8 @@ package_online: modify_composefile
@if [ -n "$(REGISTRYSERVER)" ] ; then \
$(SEDCMD) -i 's/image\: vmware/image\: $(REGISTRYSERVER)\/$(REGISTRYPROJECTNAME)/' \
$(HARBORPKG)/docker-compose.yml ; \
$(SEDCMD) -i 's/image\: vmware/image\: $(REGISTRYSERVER)\/$(REGISTRYPROJECTNAME)/' \
$(HARBORPKG)/ha/docker-compose.yml ; \
fi
@cp LICENSE $(HARBORPKG)/LICENSE
@cp NOTICE $(HARBORPKG)/NOTICE
@ -363,6 +368,7 @@ package_offline: compile build modify_sourcefiles modify_composefile
@cp NOTICE $(HARBORPKG)/NOTICE
@cp tools/migration/migration_cfg/upgrade $(HARBORPKG)/upgrade
@cp tools/migration/migration_cfg/harbor_1_1_0_template $(HARBORPKG)/harbor_1_1_0_template
@cp $(HARBORPKG)/common/db/registry.sql $(HARBORPKG)/ha/
@echo "pulling nginx and registry..."
@$(DOCKERPULL) vmware/registry:$(REGISTRYVERSION)

View File

@ -0,0 +1,45 @@
version: 0.1
log:
level: debug
fields:
service: registry
storage:
cache:
layerinfo: redis
Place_holder_for_Storage_configureation
maintenance:
uploadpurging:
enabled: false
delete:
enabled: true
redis:
addr: $redis_url
db: 0
dialtimeout: 10ms
readtimeout: 10ms
writetimeout: 10ms
pool:
maxidle: 16
maxactive: 64
idletimeout: 300s
http:
addr: :5000
secret: placeholder
debug:
addr: localhost:5001
auth:
token:
issuer: harbor-token-issuer
realm: $ui_url/service/token
rootcertbundle: /etc/registry/root.crt
service: harbor-registry
notifications:
endpoints:
- name: harbor
disabled: false
url: http://ui:8080/service/notifications
timeout: 3000ms
threshold: 5
backoff: 1s

View File

@ -5,3 +5,4 @@ JOBSERVICE_SECRET=$jobservice_secret
GODEBUG=netdns=cgo
ADMINSERVER_URL=http://adminserver:8080
UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem
_REDIS_URL=$redis_url

121
make/ha/docker-compose.tpl Normal file
View File

@ -0,0 +1,121 @@
version: '2'
services:
log:
image: vmware/harbor-log:__version__
container_name: harbor-log
restart: always
volumes:
- /var/log/harbor/:/var/log/docker/:z
- ./common/config/log/:/etc/logrotate.d/:z
ports:
- 127.0.0.1:1514:10514
networks:
- harbor
registry:
image: vmware/registry:2.6.2-photon
container_name: registry
restart: always
volumes:
- /data/registry:/storage:z
- ./common/config/registry/:/etc/registry/:z
networks:
- harbor
environment:
- GODEBUG=netdns=cgo
command:
["serve", "/etc/registry/config.yml"]
depends_on:
- log
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "registry"
adminserver:
image: vmware/harbor-adminserver:__version__
container_name: harbor-adminserver
env_file:
- ./common/config/adminserver/env
restart: always
volumes:
- /data/config/:/etc/adminserver/config/:z
- /data/secretkey:/etc/adminserver/key:z
- /data/:/data/:z
networks:
- harbor
depends_on:
- log
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "adminserver"
ui:
image: vmware/harbor-ui:__version__
container_name: harbor-ui
env_file:
- ./common/config/ui/env
restart: always
volumes:
- ./common/config/ui/app.conf:/etc/ui/app.conf:z
- ./common/config/ui/private_key.pem:/etc/ui/private_key.pem:z
- ./common/config/ui/certificates/:/etc/ui/certifates/
- /data/secretkey:/etc/ui/key:z
- /data/ca_download/:/etc/ui/ca/:z
- /data/psc/:/etc/ui/token/:z
networks:
- harbor
depends_on:
- log
- adminserver
- registry
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "ui"
jobservice:
image: vmware/harbor-jobservice:__version__
container_name: harbor-jobservice
env_file:
- ./common/config/jobservice/env
restart: always
volumes:
- /data/job_logs:/var/log/jobs:z
- ./common/config/jobservice/app.conf:/etc/jobservice/app.conf:z
- /data/secretkey:/etc/jobservice/key:z
networks:
- harbor
depends_on:
- ui
- adminserver
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "jobservice"
proxy:
image: vmware/nginx-photon:1.11.13
container_name: nginx
restart: always
volumes:
- ./common/config/nginx:/etc/nginx:z
networks:
- harbor
ports:
- 80:80
- 443:443
- 4443:4443
depends_on:
- registry
- ui
- log
logging:
driver: "syslog"
options:
syslog-address: "tcp://127.0.0.1:1514"
tag: "proxy"
networks:
harbor:
external: false

View File

@ -0,0 +1,80 @@
global_defs {
router_id haborlb
}
vrrp_sync_groups VG1 {
group {
VI_1
}
}
#Please change to ens160 to the interface name on you loadbalancer hosts.
vrrp_instance VI_1 {
interface ens160
track_interface {
ens160
}
state MASTER
virtual_router_id 51
priority 10
virtual_ipaddress {
VIP/32
}
advert_int 1
authentication {
auth_type PASS
auth_pass d0cker
}
}
#Please change VIP, harbor_node1_ip, harbor_node2_ip to real ip address
virtual_server VIP 80 {
delay_loop 15
lb_algo rr
lb_kind DR
protocol TCP
nat_mask 255.255.255.0
persistence_timeout 10
real_server harbor_node1_ip 80 {
weight 10
TCP_CHECK {
connect_timeout 3
connect_port 80
}
}
real_server harbor_node2_ip 80 {
weight 10
TCP_CHECK {
connect_timeout 3
connect_port 80
}
}
}
#Please uncomment the follow when harbor running under https
#virtual_server VIP 443 {
# delay_loop 15
# lb_algo rr
# lb_kind DR
# protocol TCP
# nat_mask 255.255.255.0
# persistence_timeout 10
#
# real_server harbor_node1_ip 443 {
# weight 10
# TCP_CHECK {
# connect_timeout 3
# connect_port 443
# }
# }
#
# real_server harbor_node2_ip 443 {
# weight 10
# TCP_CHECK {
# connect_timeout 3
# connect_port 443
# }
# }
#}

View File

@ -0,0 +1,7 @@
#!/bin/bash
http_code = `curl -s -o /dev/null -w "%{http_code}" 127.0.0.1`
if [ $http_code == 200 ] || [ $http_code == 301 ] ; then
exit 0
else
exit 1
fi

View File

@ -0,0 +1,36 @@
global_defs {
router_id haborlb
}
vrrp_script check_harbor {
script "/usr/local/bin/check_harbor.sh"
interval 15
fail 5
rise 2
}
vrrp_sync_groups VG1 {
group {
VI_1
}
}
#Please change to ens160 to the interface name on you loadbalancer hosts.
vrrp_instance VI_1 {
interface ens160
track_interface {
ens160
}
state MASTER
virtual_router_id 51
priority 10
virtual_ipaddress {
VIP/32
}
advert_int 1
authentication {
auth_type PASS
auth_pass d0cker
}
}

View File

@ -116,6 +116,8 @@ db_port = 3306
#The user name of mysql database
db_user = root
#The redis server address
redis_url =
#************************END INITIAL PROPERTIES************************
#The following attributes only need to be set when auth mode is uaa_auth
uaa_endpoint = uaa.mydomain.org

View File

@ -58,7 +58,8 @@ item=0
with_notary=$false
# clair is not enabled by default
with_clair=$false
# HA mode is not enabled by default
harbor_ha=$false
while [ $# -gt 0 ]; do
case $1 in
--help)
@ -67,7 +68,9 @@ while [ $# -gt 0 ]; do
--with-notary)
with_notary=true;;
--with-clair)
with_clair=true;;
with_clair=true;;
--ha)
harbor_ha=true;;
*)
note "$usage"
exit 1;;
@ -158,24 +161,28 @@ then
sed "s/^hostname = .*/hostname = $host/g" -i ./harbor.cfg
fi
prepare_para=
if [ $with_notary ]
if [ $with_notary ] && [ ! $harbor_ha ]
then
prepare_para="${prepare_para} --with-notary"
fi
if [ $with_clair ]
if [ $with_clair ] && [ ! $harbor_ha ]
then
prepare_para="${prepare_para} --with-clair"
fi
if [ $harbor_ha ]
then
prepare_para="${prepare_para} --ha"
fi
./prepare $prepare_para
echo ""
h2 "[Step $item]: checking existing instance of Harbor ..."; let item+=1
docker_compose_list='-f docker-compose.yml'
if [ $with_notary ]
if [ $with_notary ] && [ ! $harbor_ha ]
then
docker_compose_list="${docker_compose_list} -f docker-compose.notary.yml"
fi
if [ $with_clair ]
if [ $with_clair ] && [ ! $harbor_ha ]
then
docker_compose_list="${docker_compose_list} -f docker-compose.clair.yml"
fi
@ -188,6 +195,11 @@ fi
echo ""
h2 "[Step $item]: starting Harbor ..."
if [ $harbor_ha ]
then
mv docker-compose.yml docker-compose.yml.bak
cp ha/docker-compose.yml docker-compose.yml
fi
docker-compose $docker_compose_list up -d
protocol=http

View File

@ -20,6 +20,27 @@ if sys.version_info[:3][0] == 3:
import io as StringIO
def validate(conf, args):
if args.ha_mode:
db_host = rcp.get("configuration", "db_host")
if db_host == "mysql":
raise Exception("Error: In HA mode, db_host in harbor.cfg needs to point to an external DB address")
registry_config_path = os.path.join(templates_dir,"registry","config_ha.yml")
if check_storage_config(registry_config_path):
raise Exception("Error: In HA model shared storage configuration is required registry, refer HA installation guide for detail.")
redis_url = rcp.get("configuration", "redis_url")
if redis_url is None or len(redis_url) < 1:
raise Exception("Error: In HA mode redis is required redis_url need to point to an redis cluster")
if args.notary_mode or args.clair_mode:
raise Exception("Error: HA mode doesn't support clair and notary currently")
cert_path = rcp.get("configuration", "ssl_cert")
cert_key_path = rcp.get("configuration", "ssl_cert_key")
shared_cert_key = os.path.join(base_dir, "ha", os.path.basename(cert_key_path))
shared_cert_path = os.path.join(base_dir, "ha", os.path.basename(cert_path))
if os.path.isfile(shared_cert_key):
shutil.copy2(shared_cert_key, cert_key_path)
if os.path.isfile(shared_cert_path):
shutil.copy2(shared_cert_path, cert_path)
protocol = rcp.get("configuration", "ui_url_protocol")
if protocol != "https" and args.notary_mode:
raise Exception("Error: the protocol must be https when Harbor is deployed with Notary")
@ -39,6 +60,63 @@ def validate(conf, args):
if project_creation != "everyone" and project_creation != "adminonly":
raise Exception("Error invalid value for project_creation_restriction: %s" % project_creation)
def prepare_ha(conf, args):
#files under ha folder will have high prority
protocol = rcp.get("configuration", "ui_url_protocol")
if protocol == "https":
#copy nginx certificate
cert_path = rcp.get("configuration", "ssl_cert")
cert_key_path = rcp.get("configuration", "ssl_cert_key")
shared_cert_key = os.path.join(base_dir, "ha", os.path.basename(cert_key_path))
shared_cert_path = os.path.join(base_dir, "ha", os.path.basename(cert_path))
if os.path.isfile(shared_cert_key):
shutil.copy2(shared_cert_key, cert_key_path)
else:
if os.path.isfile(cert_key_path):
shutil.copy2(cert_key_path, shared_cert_key)
if os.path.isfile(shared_cert_path):
shutil.copy2(shared_cert_path, cert_path)
else:
if os.path.isfile(cert_path):
shutil.copy2(cert_path, shared_cert_path)
#check if ca exsit
cert_ca_path = "/data/ca_download/ca.crt"
shared_ca_path = os.path.join(base_dir, "ha", os.path.basename(cert_ca_path))
if os.path.isfile(shared_ca_path):
shutil.copy2(shared_ca_path, cert_ca_path)
else:
if os.path.isfile(cert_ca_path):
shutil.copy2(cert_ca_path, shared_ca_path)
#check root.crt and priviate_key.pem
private_key_pem = os.path.join(config_dir, "ui", "private_key.pem")
root_crt = os.path.join(config_dir, "registry", "root.crt")
shared_private_key_pem = os.path.join(base_dir, "ha", "private_key.pem")
shared_root_crt = os.path.join(base_dir, "ha", "root.crt")
if os.path.isfile(shared_private_key_pem):
shutil.copy2(shared_private_key_pem, private_key_pem)
else:
if os.path.isfile(private_key_pem):
shutil.copy2(private_key_pem, shared_private_key_pem)
if os.path.isfile(shared_root_crt):
shutil.copy2(shared_root_crt, root_crt)
else:
if os.path.isfile(root_crt):
shutil.copy2(root_crt, shared_root_crt)
#secretkey
shared_secret_key = os.path.join(base_dir, "ha", "secretkey")
secretkey_path = rcp.get("configuration", "secretkey_path")
secret_key = os.path.join(secretkey_path, "secretkey")
if os.path.isfile(shared_secret_key):
shutil.copy2(shared_secret_key, secret_key)
else:
if os.path.isfile(secret_key):
shutil.copy2(secret_key, shared_secret_key)
def check_storage_config(path):
if 'Place_holder_for_Storage_configureation' in open(path).read():
return True
return False
def get_secret_key(path):
secret_key = _get_secret(path, "secretkey")
if len(secret_key) != 16:
@ -96,6 +174,7 @@ parser = argparse.ArgumentParser()
parser.add_argument('--conf', dest='cfgfile', default=base_dir+'/harbor.cfg',type=str,help="the path of Harbor configuration file")
parser.add_argument('--with-notary', dest='notary_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with notary")
parser.add_argument('--with-clair', dest='clair_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with clair")
parser.add_argument('--ha', dest='ha_mode', default=False, action='store_true', help="the Harbor instance is to be deployed in HA mode")
args = parser.parse_args()
delfile(config_dir)
@ -106,7 +185,6 @@ conf.write(open(args.cfgfile).read())
conf.seek(0, os.SEEK_SET)
rcp = ConfigParser.RawConfigParser()
rcp.readfp(conf)
validate(rcp, args)
hostname = rcp.get("configuration", "hostname")
@ -141,8 +219,8 @@ ldap_scope = rcp.get("configuration", "ldap_scope")
ldap_timeout = rcp.get("configuration", "ldap_timeout")
db_password = rcp.get("configuration", "db_password")
db_host = rcp.get("configuration", "db_host")
db_port = rcp.get("configuration", "db_port")
db_user = rcp.get("configuration", "db_user")
db_port = rcp.get("configuration", "db_port")
self_registration = rcp.get("configuration", "self_registration")
if protocol == "https":
cert_path = rcp.get("configuration", "ssl_cert")
@ -161,9 +239,15 @@ uaa_endpoint = rcp.get("configuration", "uaa_endpoint")
uaa_clientid = rcp.get("configuration", "uaa_clientid")
uaa_clientsecret = rcp.get("configuration", "uaa_clientsecret")
uaa_ca_root = rcp.get("configuration", "uaa_ca_root")
secret_key = get_secret_key(secretkey_path)
log_rotate_count = rcp.get("configuration", "log_rotate_count")
log_rotate_size = rcp.get("configuration", "log_rotate_size")
if rcp.has_option("configuration", "redis_url"):
redis_url = rcp.get("configuration", "redis_url")
else:
redis_url = ""
########
ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))
@ -229,8 +313,8 @@ render(os.path.join(templates_dir, "adminserver", "env"),
ldap_timeout=ldap_timeout,
db_password=db_password,
db_host=db_host,
db_port=db_port,
db_user=db_user,
db_port=db_port,
email_host=email_host,
email_port=email_port,
email_usr=email_usr,
@ -258,12 +342,19 @@ render(os.path.join(templates_dir, "ui", "env"),
ui_conf_env,
ui_secret=ui_secret,
jobservice_secret=jobservice_secret,
redis_url = redis_url
)
render(os.path.join(templates_dir, "registry",
"config.yml"),
registry_conf,
ui_url=ui_url)
if args.ha_mode:
render(os.path.join(templates_dir, "registry",
"config_ha.yml"),
registry_conf,
ui_url=ui_url,
redis_url=redis_url)
else:
render(os.path.join(templates_dir, "registry",
"config.yml"),
registry_conf,
ui_url=ui_url)
render(os.path.join(templates_dir, "db", "env"),
db_conf_env,
@ -345,7 +436,7 @@ else:
shutil.copyfile(os.path.join(templates_dir, "ui", "private_key.pem"), os.path.join(ui_config_dir, "private_key.pem"))
print("Copied configuration file: %s" % registry_config_dir + "root.crt")
shutil.copyfile(os.path.join(templates_dir, "registry", "root.crt"), os.path.join(registry_config_dir, "root.crt"))
if args.notary_mode:
notary_config_dir = prep_conf_dir(config_dir, "notary")
notary_temp_dir = os.path.join(templates_dir, "notary")
@ -413,6 +504,9 @@ if args.clair_mode:
clair_conf = os.path.join(clair_config_dir, "config.yaml")
render(os.path.join(clair_temp_dir, "config.yaml"), clair_conf, password = pg_password)
if args.ha_mode:
prepare_ha(rcp, args)
FNULL.close()
print("The configuration files are ready, please use docker-compose to start the service.")