Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-21 17:55:30 +01:00
Code Issues Projects Releases Wiki Activity

Feat: auto install ca in registry

Browse Source 
Signed-off-by: DQ <dengq@vmware.com>
This commit is contained in:
DQ 2020-02-14 21:11:52 +08:00
parent b852605193
commit dcc6950af7
32 changed files with 404 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Makefile
View File

@ -85,6 +85,7 @@ NPM_REGISTRY=https://registry.npmjs.org
# enable/disable chart repo supporting
CHARTFLAG=false
BUILDTARGET=build
GEN_TLS=
# version prepare
# for docker image tag
@ -349,10 +350,16 @@ compile: check_environment versions_prepare compile_core compile_jobservice comp
update_prepare_version:
@echo "substitute the prepare version tag in prepare file..."
@$(SEDCMDI) -e 's/goharbor\/prepare:.*[[:space:]]\+/goharbor\/prepare:$(VERSIONTAG) /' $(MAKEPATH)/prepare ;
@$(SEDCMDI) -e 's/goharbor\/prepare:.*[[:space:]]\+/goharbor\/prepare:$(VERSIONTAG) prepare /' $(MAKEPATH)/prepare ;
gen_tls:
@$(DOCKERCMD) run --rm -v /:/hostfs:z goharbor/prepare:$(VERSIONTAG) gencert /etc/harbor/tls/internal
prepare: update_prepare_version
@echo "preparing..."
@if [ -n "$(GEN_TLS)" ] ; then \
$(DOCKERCMD) run --rm -v /:/hostfs:z goharbor/prepare:$(VERSIONTAG) gencert /etc/harbor/tls/internal; \
fi
@$(MAKEPATH)/$(PREPARECMD) $(PREPARECMD_PARA)
build:

2
make/harbor.yml.tmpl
View File

@ -17,7 +17,7 @@ https:
certificate: /your/certificate/path
private_key: /your/private/key/path
# internal_tls:
# internal_tls: /etc/harbor/tls/internal
# Uncomment external_url if you want to enable external proxy
# And when it enabled the hostname will no longer used

10
make/photon/clair-adapter/Dockerfile
View File

@ -1,10 +1,14 @@
ARG harbor_base_image_version
FROM goharbor/harbor-clair-adapter-base:${harbor_base_image_version}
COPY ./make/photon/common/install_cert.sh /home/clair-adapter
COPY ./make/photon/clair-adapter/entrypoint.sh /home/clair-adapter
COPY ./make/photon/clair-adapter/binary/harbor-scanner-clair /clair-adapter/clair-adapter
RUN chown -R 10000:10000 /clair-adapter \
&& chmod u+x /clair-adapter/clair-adapter
RUN chown -R clair-adapter:clair-adapter /etc/pki/tls/certs \
&& chown -R clair-adapter:clair-adapter /clair-adapter && chmod u+x /clair-adapter/clair-adapter \
&& chown clair-adapter:clair-adapter /home/clair-adapter/entrypoint.sh && chmod u+x /home/clair-adapter/entrypoint.sh \
&& chown clair-adapter:clair-adapter /home/clair-adapter/install_cert.sh && chmod u+x /home/clair-adapter/install_cert.sh
EXPOSE 8080
@ -12,4 +16,4 @@ HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:8080
USER clair-adapter
ENTRYPOINT ["/clair-adapter/clair-adapter"]
ENTRYPOINT ["/home/clair-adapter/entrypoint.sh"]

7
make/photon/clair-adapter/entrypoint.sh Normal file
View File

@ -0,0 +1,7 @@
#!/bin/sh
set -e
/home/clair-adapter/install_cert.sh
/clair-adapter/clair-adapter

4
make/photon/clair/Dockerfile
View File

@ -10,10 +10,10 @@ VOLUME /config
EXPOSE 6060 6061
RUN chown -R clair:clair /etc/pki/tls/certs \
&& chown -R clair:clair /home/clair \
RUN chown -R clair:clair /etc/pki/tls/certs && chown -R clair:clair /home/clair \
&& chmod u+x /home/clair/clair \
&& chmod u+x /home/clair/docker-entrypoint.sh \
&& chmod u+x /home/clair/install_cert.sh \
&& chmod +x /home/clair/dumb-init
HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:6061/health || exit 1

0
make/photon/prepare/commands/__init__.py Normal file
View File

26
make/photon/prepare/commands/gencerts.py Normal file
View File

@ -0,0 +1,26 @@
import os
import click
import pathlib
from subprocess import check_call, PIPE, STDOUT
from utils.cert import openssl_installed
from utils.misc import get_realpath
gen_tls_script = pathlib.Path(__file__).parent.parent.joinpath('scripts/gencert.sh').absolute()
@click.command()
@click.argument('path')
def gencert(path):
path = get_realpath(path)
click.echo('Check openssl ...')
if not openssl_installed():
raise(Exception('openssl not installed'))
click.echo("start generate internal tls certs")
if not os.path.exists(path):
click.echo('path {} not exist, create it...'.format(path))
os.makedirs(path, exist_ok=True)
shell_stat = check_call([gen_tls_script], stdout=PIPE, stderr=STDOUT, cwd=path)
if shell_stat != 0:
click.echo('Can not generate internal tls certs')

78
make/photon/prepare/commands/prepare.py Normal file
View File

@ -0,0 +1,78 @@
# pylint: disable=no-value-for-parameter
import sys
import logging
import click
from utils.misc import delfile
from utils.configs import validate, parse_yaml_config
from utils.cert import prepare_registry_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key
from utils.db import prepare_db
from utils.jobservice import prepare_job_service
from utils.registry import prepare_registry
from utils.registry_ctl import prepare_registry_ctl
from utils.core import prepare_core
from utils.notary import prepare_notary
from utils.log import prepare_log_configs
from utils.clair import prepare_clair
from utils.clair_adapter import prepare_clair_adapter
from utils.chart import prepare_chartmuseum
from utils.docker_compose import prepare_docker_compose
from utils.nginx import prepare_nginx, nginx_confd_dir
from utils.redis import prepare_redis
from utils.internal_tls import prepare_tls
from utils.trivy_adapter import prepare_trivy_adapter
from g import (config_dir, input_config_path, private_key_pem_path, root_crt_path, secret_key_dir,
old_private_key_pem_path, old_crt_path)
@click.command()
@click.option('--conf', default=input_config_path, help="the path of Harbor configuration file")
@click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary")
@click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair")
@click.option('--with-trivy', is_flag=True, help="the Harbor instance is to be deployed with Trivy")
@click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting")
def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
delfile(config_dir)
config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
try:
validate(config_dict, notary_mode=with_notary)
except Exception as e:
click.echo('Error happened in config validation...')
logging.error(e)
sys.exit(-1)
prepare_log_configs(config_dict)
prepare_nginx(config_dict)
prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
prepare_registry(config_dict)
prepare_registry_ctl(config_dict)
prepare_db(config_dict)
prepare_job_service(config_dict)
prepare_redis(config_dict)
prepare_tls(config_dict)
get_secret_key(secret_key_dir)
# If Customized cert enabled
prepare_registry_ca(
private_key_pem_path=private_key_pem_path,
root_crt_path=root_crt_path,
old_private_key_pem_path=old_private_key_pem_path,
old_crt_path=old_crt_path)
if with_notary:
prepare_notary(config_dict, nginx_confd_dir, SSL_CERT_PATH, SSL_CERT_KEY_PATH)
if with_clair:
prepare_clair(config_dict)
prepare_clair_adapter(config_dict)
if with_trivy:
prepare_trivy_adapter(config_dict)
if with_chartmuseum:
prepare_chartmuseum(config_dict)
prepare_docker_compose(config_dict, with_clair, with_trivy, with_notary, with_chartmuseum)

83
make/photon/prepare/main.py
View File

@ -1,80 +1,13 @@
# pylint: disable=no-value-for-parameter
import sys
import logging
from commands.prepare import prepare
from commands.gencerts import gencert
import click
from utils.misc import delfile
from utils.configs import validate, parse_yaml_config
from utils.cert import prepare_registry_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key
from utils.db import prepare_db
from utils.jobservice import prepare_job_service
from utils.registry import prepare_registry
from utils.registry_ctl import prepare_registry_ctl
from utils.core import prepare_core
from utils.notary import prepare_notary
from utils.log import prepare_log_configs
from utils.clair import prepare_clair
from utils.clair_adapter import prepare_clair_adapter
from utils.trivy_adapter import prepare_trivy_adapter
from utils.chart import prepare_chartmuseum
from utils.docker_compose import prepare_docker_compose
from utils.nginx import prepare_nginx, nginx_confd_dir
from utils.redis import prepare_redis
from utils.internal_tls import prepare_tls
from g import (config_dir, input_config_path, private_key_pem_path, root_crt_path, secret_key_dir,
old_private_key_pem_path, old_crt_path)
# Main function
@click.command()
@click.option('--conf', default=input_config_path, help="the path of Harbor configuration file")
@click.option('--with-notary', is_flag=True, help="the Harbor instance is to be deployed with notary")
@click.option('--with-clair', is_flag=True, help="the Harbor instance is to be deployed with clair")
@click.option('--with-trivy', is_flag=True, help="the Harbor instance is to be deployed with Trivy")
@click.option('--with-chartmuseum', is_flag=True, help="the Harbor instance is to be deployed with chart repository supporting")
def main(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
@click.group()
def cli():
pass
delfile(config_dir)
config_dict = parse_yaml_config(conf, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
try:
validate(config_dict, notary_mode=with_notary)
except Exception as e:
logging.info('Error happened in config validation...')
logging.error(e)
sys.exit(-1)
prepare_log_configs(config_dict)
prepare_nginx(config_dict)
prepare_core(config_dict, with_notary=with_notary, with_clair=with_clair, with_trivy=with_trivy, with_chartmuseum=with_chartmuseum)
prepare_registry(config_dict)
prepare_registry_ctl(config_dict)
prepare_db(config_dict)
prepare_job_service(config_dict)
prepare_redis(config_dict)
prepare_tls(config_dict)
get_secret_key(secret_key_dir)
# If Customized cert enabled
prepare_registry_ca(
private_key_pem_path=private_key_pem_path,
root_crt_path=root_crt_path,
old_private_key_pem_path=old_private_key_pem_path,
old_crt_path=old_crt_path)
if with_notary:
prepare_notary(config_dict, nginx_confd_dir, SSL_CERT_PATH, SSL_CERT_KEY_PATH)
if with_clair:
prepare_clair(config_dict)
prepare_clair_adapter(config_dict)
if with_trivy:
prepare_trivy_adapter(config_dict)
if with_chartmuseum:
prepare_chartmuseum(config_dict)
prepare_docker_compose(config_dict, with_clair, with_trivy, with_notary, with_chartmuseum)
cli.add_command(prepare)
cli.add_command(gencert)
if __name__ == '__main__':
main()
cli()

2
make/photon/prepare/models.py
View File

@ -4,7 +4,7 @@ from pathlib import Path
from shutil import copytree, rmtree
from g import internal_tls_dir, DEFAULT_GID, DEFAULT_UID, PG_GID, PG_UID
from utils.misc import check_permission, owner_can_read, other_can_read, get_realpath, owner_can_read
from utils.misc import check_permission, owner_can_read, get_realpath
class InternalTLS:

118
make/photon/prepare/scripts/gencert.sh Executable file
View File

@ -0,0 +1,118 @@
#! /bin/bash
# CA key and certificate
openssl req -x509 -nodes -days 365 -newkey rsa:4096 \
-keyout "harbor_internal_ca.key" \
-out "harbor_internal_ca.crt" \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware"
# generate proxy key and csr
openssl req -new -newkey rsa:4096 -nodes -sha256 \
-keyout proxy.key \
-out proxy.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=proxy"
# Sign proxy
openssl x509 -req -days 365 -sha256 -in proxy.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out proxy.crt
# generate core key and csr
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout core.key \
-out core.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=core"
# Sign core csr with CA certificate and key
openssl x509 -req -days 365 -sha256 -in core.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out core.crt
# job_service key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout job_service.key \
-out job_service.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=jobservice"
# sign job_service csr with CA certificate and key
openssl x509 -req -days 365 -sha256 -in job_service.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out job_service.crt
# generate registry key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout registry.key \
-out registry.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=registry"
# sign registry csr with CA certificate and key
openssl x509 -req -days 365 -sha256 -in registry.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out registry.crt
# generate registryctl key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout registryctl.key \
-out registryctl.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=registryctl"
# sign registryctl csr with CA certificate and key
openssl x509 -req -days 365 -sha256 -in registryctl.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out registryctl.crt
# generate clair_adapter key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout clair_adapter.key \
-out clair_adapter.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=clair_adapter"
# sign clair_adapter csr with CA certificate and key
openssl x509 -req -days 365 -sha256 -in clair_adapter.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out clair_adapter.crt
# generate clair key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout clair.key \
-out clair.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=clair"
# sign clair csr with CA certificate and key
openssl x509 -req -days 365 -sha256 -in clair.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out clair.crt
# generate notary_signer key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout notary_signer.key \
-out notary_signer.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=notary_signer"
# sign notary_signer csr with CA certificate and key
openssl x509 -req -days 365 -sha256 -in notary_signer.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out notary_signer.crt
# generate notary_server key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout notary_server.key \
-out notary_server.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=notary_server"
# sign notary_server csr with CA certificate and key
openssl x509 -req -days 365 -sha256 -in notary_server.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out notary_server.crt
# generate chartmuseum key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout chartmuseum.key \
-out chartmuseum.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=chartmuseum"
# sign chartmuseum csr with CA certificate and key
openssl x509 -req -days 365 -sha256 -in chartmuseum.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out chartmuseum.crt
# generate harbor_db key
openssl req -new \
-newkey rsa:4096 -nodes -sha256 -keyout harbor_db.key \
-out harbor_db.csr \
-subj "/C=CN/ST=Beijing/L=Beijing/O=VMware/CN=harbor_db"
# sign harbor_db csr with CA certificate and key
openssl x509 -req -days 365 -sha256 -in harbor_db.csr -CA harbor_internal_ca.crt -CAkey harbor_internal_ca.key -CAcreateserial -out harbor_db.crt

38
make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja
View File

@ -4,7 +4,7 @@ services:
image: goharbor/harbor-log:{{version}}
container_name: harbor-log
restart: always
dns_search: ""
dns_search: .
cap_drop:
- ALL
cap_add:
@ -42,9 +42,12 @@ services:
target: /etc/registry/gcs.key
{% endif %}
{%if internal_tls.enabled %}
- type: bind
source: {{internal_tls.core_key_path}}
target: /harbor_cust_cert/core.crt
- type: bind
source: {{internal_tls.harbor_internal_ca_crt_path}}
target: /etc/harbor/tls/harbor_internal_ca.crt
target: /harbor_cust_cert/harbor_internal_ca.crt
- type: bind
source: {{internal_tls.registry_crt_path}}
target: /etc/harbor/tls/registry.crt
@ -57,7 +60,7 @@ services:
{% if with_clair %}
- harbor-clair
{% endif %}
dns_search: ""
dns_search: .
depends_on:
- log
logging:
@ -96,7 +99,7 @@ services:
{% endif %}
networks:
- harbor
dns_search: ""
dns_search: .
depends_on:
- log
logging:
@ -141,7 +144,7 @@ services:
aliases:
- harbor-db
{% endif %}
dns_search: ""
dns_search: .
env_file:
- ./common/config/db/env
depends_on:
@ -207,7 +210,7 @@ services:
aliases:
- harbor-core
{% endif %}
dns_search: ""
dns_search: .
depends_on:
- log
- registry
@ -235,7 +238,7 @@ services:
- NET_BIND_SERVICE
networks:
- harbor
dns_search: ""
dns_search: .
depends_on:
- log
logging:
@ -277,7 +280,7 @@ services:
{% if with_clair %}
- harbor-clair
{% endif %}
dns_search: ""
dns_search: .
depends_on:
- core
logging:
@ -310,7 +313,7 @@ services:
aliases:
- redis
{% endif %}
dns_search: ""
dns_search: .
depends_on:
- log
logging:
@ -351,7 +354,7 @@ services:
{% if with_notary %}
- harbor-notary
{% endif %}
dns_search: ""
dns_search: .
ports:
- {{http_port}}:8080
{% if protocol == 'https' %}
@ -378,7 +381,7 @@ services:
networks:
- notary-sig
- harbor-notary
dns_search: ""
dns_search: .
volumes:
- ./common/config/notary:/etc/notary:z
- type: bind
@ -419,7 +422,7 @@ services:
notary-sig:
aliases:
- notarysigner
dns_search: ""
dns_search: .
volumes:
- ./common/config/notary:/etc/notary:z
- type: bind
@ -466,7 +469,7 @@ services:
- SETGID
- SETUID
cpu_quota: 50000
dns_search: ""
dns_search: .
depends_on:
- log
{% if external_database == False %}
@ -479,7 +482,7 @@ services:
{%if internal_tls.enabled %}
- type: bind
source: {{internal_tls.harbor_internal_ca_crt_path}}
target: /etc/harbor/ssl/harbor_internal_ca.crt
target: /harbor_cust_cert/harbor_internal_ca.crt
- type: bind
source: {{internal_tls.clair_crt_path}}
target: /etc/harbor/ssl/clair.crt
@ -507,16 +510,17 @@ services:
- SETGID
- SETUID
cpu_quota: 50000
dns_search: ""
dns_search: .
depends_on:
- clair
{% if external_redis == False %}
- redis
{% endif %}
{%if internal_tls.enabled %}
volumes:
- type: bind
source: {{internal_tls.harbor_internal_ca_crt_path}}
target: /etc/harbor/ssl/harbor_internal_ca.crt
target: /harbor_cust_cert/harbor_internal_ca.crt
- type: bind
source: {{internal_tls.clair_adapter_crt_path}}
target: /etc/harbor/ssl/clair_adapter.crt
@ -575,7 +579,7 @@ services:
- SETUID
networks:
- harbor-chartmuseum
dns_search: ""
dns_search: .
depends_on:
- log
volumes:

2
make/photon/prepare/templates/registry/config.yml.jinja
View File

@ -34,7 +34,7 @@ http:
certificate: /etc/harbor/tls/registry.crt
key: /etc/harbor/tls/registry.key
clientcas:
- /etc/harbor/tls/harbor_internal_ca.crt
- /harbor_cust_cert/harbor_internal_ca.crt
{% endif %}
auth:

2
make/photon/prepare/utils/cert.py
View File

@ -14,8 +14,6 @@ from .misc import (
SSL_CERT_PATH = os.path.join("/etc/cert", "server.crt")
SSL_CERT_KEY_PATH = os.path.join("/etc/cert", "server.key")
secret_keys_dir = '/secret/keys'
def _get_secret(folder, filename, length=16):
key_file = os.path.join(folder, filename)
if os.path.isfile(key_file):

36
make/photon/prepare/utils/configs.py
View File

@ -106,7 +106,6 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
configs = yaml.load(f)
config_dict = {
'adminserver_url': 'http://adminserver:8080',
'registry_url': 'http://registry:5000',
'registry_controller_url': 'http://registryctl:8080',
'core_url': 'http://core:8080',
@ -337,25 +336,24 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
config_dict['registry_password'] = generate_random_string(32)
# TLS related configs
if configs.get('internal_tls'):
config_dict['internal_tls'] = InternalTLS(
configs['internal_tls'],
configs['data_volume'],
with_notary=with_notary,
with_clair=with_clair,
with_chartmuseum=with_chartmuseum,
external_database=config_dict['external_database'])
config_dict['internal_tls'] = InternalTLS(
configs.get('internal_tls') or '',
configs['data_volume'],
with_notary=with_notary,
with_clair=with_clair,
with_chartmuseum=with_chartmuseum,
external_database=config_dict['external_database'])
if config_dict['internal_tls'].enabled:
config_dict['registry_url'] = 'https://registry:5443'
config_dict['registry_controller_url'] = 'https://registryctl:8443'
config_dict['core_url'] = 'https://core:8443'
config_dict['core_local_url'] = 'https://127.0.0.1:8443'
config_dict['token_service_url'] = 'https://core:8443/service/token'
config_dict['jobservice_url'] = 'https://jobservice:8443'
# config_dict['clair_adapter_url'] = 'https://clair-adapter:8443'
# config_dict['notary_url'] = 'https://notary-server:4443'
config_dict['chart_repository_url'] = 'https://chartmuseum:9443'
if config_dict['internal_tls'].enabled:
config_dict['registry_url'] = 'https://registry:5443'
config_dict['registry_controller_url'] = 'https://registryctl:8443'
config_dict['core_url'] = 'https://core:8443'
config_dict['core_local_url'] = 'https://core:8443'
config_dict['token_service_url'] = 'https://core:8443/service/token'
config_dict['jobservice_url'] = 'https://jobservice:8443'
# config_dict['clair_adapter_url'] = 'https://clair-adapter:8443'
# config_dict['notary_url'] = 'http://notary-server:4443'
config_dict['chart_repository_url'] = 'https://chartmuseum:9443'
return config_dict

1
make/photon/prepare/utils/docker_compose.py
View File

@ -1,5 +1,4 @@
import os
from functools import reduce
from g import templates_dir
from .configs import parse_versions

4
make/photon/prepare/utils/notary.py
View File

@ -1,5 +1,5 @@
import os, shutil, pathlib
from g import templates_dir, config_dir, root_crt_path, secret_key_dir,DEFAULT_UID, DEFAULT_GID
from g import templates_dir, config_dir, root_crt_path, secret_key_dir, secret_dir, DEFAULT_UID, DEFAULT_GID
from .cert import openssl_installed, create_cert, create_root_cert, get_alias
from .jinja import render_jinja
from .misc import mark_file, prepare_dir
@ -25,7 +25,7 @@ def prepare_env_notary(nginx_config_dir):
old_signer_key_secret_path = pathlib.Path(os.path.join(config_dir, 'notary-signer.key'))
old_signer_ca_cert_secret_path = pathlib.Path(os.path.join(config_dir, 'notary-signer-ca.crt'))
notary_secret_dir = prepare_dir('/secret/notary')
notary_secret_dir = prepare_dir(secret_dir ,'notary')
signer_cert_secret_path = pathlib.Path(os.path.join(notary_secret_dir, 'notary-signer.crt'))
signer_key_secret_path = pathlib.Path(os.path.join(notary_secret_dir, 'notary-signer.key'))
signer_ca_cert_secret_path = pathlib.Path(os.path.join(notary_secret_dir, 'notary-signer-ca.crt'))

2
make/prepare
View File

@ -56,7 +56,7 @@ docker run --rm -v $input_dir:/input:z \
-v $harbor_prepare_path:/compose_location:z \
-v $config_dir:/config:z \
-v /:/hostfs:z \
goharbor/prepare:dev $@
goharbor/prepare:dev prepare $@
echo "Clean up the input dir"
# Clean up input dir

2
src/chartserver/reverse_proxy.go
View File

@ -17,6 +17,7 @@ import (
"github.com/goharbor/harbor/src/common"
"github.com/goharbor/harbor/src/common/api"
commonhttp "github.com/goharbor/harbor/src/common/http"
hlog "github.com/goharbor/harbor/src/common/utils/log"
n_event "github.com/goharbor/harbor/src/pkg/notifier/event"
"github.com/goharbor/harbor/src/replication"
@ -57,6 +58,7 @@ func NewProxyEngine(target *url.URL, cred *Credential, middlewares ...func(http.
director(target, cred, req)
},
ModifyResponse: modifyResponse,
Transport: commonhttp.GetHTTPTransport(commonhttp.InternalTransport),
}
if len(middlewares) > 0 {

7
src/core/config/config.go
View File

@ -229,6 +229,11 @@ func InternalJobServiceURL() string {
return strings.TrimSuffix(cfgMgr.Get(common.JobServiceURL).GetString(), "/")
}
// GetCoreURL returns the url of core from env
func GetCoreURL() string {
return os.Getenv("CORE_URL")
}
// InternalCoreURL returns the local harbor core url
func InternalCoreURL() string {
return strings.TrimSuffix(cfgMgr.Get(common.CoreURL).GetString(), "/")
@ -409,7 +414,7 @@ func GetPortalURL() string {
// GetRegistryCtlURL returns the URL of registryctl
func GetRegistryCtlURL() string {
url := os.Getenv("REGISTRYCTL_URL")
url := os.Getenv("REGISTRY_CONTROLLER_URL")
if len(url) == 0 {
return common.DefaultRegistryCtlURL
}

5
src/core/main.go
View File

@ -169,7 +169,10 @@ func main() {
iTLSCertPath := os.Getenv("INTERNAL_TLS_CERT_PATH")
iTrustCA := os.Getenv("INTERNAL_TLS_TRUST_CA_PATH")
log.Infof("load client key: %s client cert: %s", iTLSKeyPath, iTLSCertPath)
log.Infof("load client key: %s client cert: %s client TrustCA %s", iTLSKeyPath, iTLSCertPath, iTrustCA)
// uncomment following if harbor2 is ready
// beego.BConfig.Listen.EnableMutualHTTPS = true
// beego.BConfig.Listen.TrustCaFile = iTrustCA
beego.BConfig.Listen.EnableHTTPS = true
beego.BConfig.Listen.HTTPSPort = 8443
beego.BConfig.Listen.HTTPSKeyFile = iTLSKeyPath

43
src/internal/transport.go
View File

@ -1,43 +0,0 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
import (
"crypto/tls"
"net/http"
)
var (
secureHTTPTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: false,
},
}
insecureHTTPTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
)
// GetHTTPTransport returns the HTTP transport based on insecure configuration
func GetHTTPTransport(insecure ...bool) *http.Transport {
if len(insecure) > 0 && insecure[0] {
return insecureHTTPTransport
}
return secureHTTPTransport
}

12
src/jobservice/hook/hook_client.go
View File

@ -42,10 +42,6 @@ type basicClient struct {
// NewClient return the ptr of the new hook client
func NewClient(ctx context.Context) Client {
tlsConfig, err := commonhttp.GetInternalTLSConfig()
if err != nil {
panic(err)
}
// Create transport
transport := &http.Transport{
MaxIdleConns: 20,
@ -58,7 +54,13 @@ func NewClient(ctx context.Context) Client {
ResponseHeaderTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: tlsConfig,
}
if commonhttp.InternalTLSEnabled() {
tlsConfig, err := commonhttp.GetInternalTLSConfig()
if err != nil {
panic(err)
}
transport.TLSClientConfig = tlsConfig
}
client := &http.Client{

3
src/jobservice/job/impl/replication/scheduler.go
View File

@ -16,7 +16,6 @@ package replication
import (
"fmt"
"github.com/goharbor/harbor/src/internal"
"net/http"
"os"
@ -61,7 +60,7 @@ func (s *Scheduler) Run(ctx job.Context, params job.Parameters) error {
policyID := (int64)(params["policy_id"].(float64))
cred := auth.NewSecretAuthorizer(os.Getenv("JOBSERVICE_SECRET"))
client := common_http.NewClient(&http.Client{
Transport: internal.GetHTTPTransport(true),
Transport: common_http.GetHTTPTransport(common_http.InternalTransport),
}, cred)
if err := client.Post(url, struct {
PolicyID int64 `json:"policy_id"`

14
src/pkg/registry/auth/authorizer.go
View File

@ -16,25 +16,27 @@ package auth
import (
"fmt"
"net/http"
"net/url"
"strings"
"sync"
"github.com/docker/distribution/registry/client/auth/challenge"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/internal"
"github.com/goharbor/harbor/src/pkg/registry/auth/basic"
"github.com/goharbor/harbor/src/pkg/registry/auth/bearer"
"github.com/goharbor/harbor/src/pkg/registry/auth/null"
"net/http"
"net/url"
"strings"
"sync"
)
// NewAuthorizer creates an authorizer that can handle different auth schemes
func NewAuthorizer(username, password string, insecure bool) internal.Authorizer {
func NewAuthorizer(username, password string, trType uint) internal.Authorizer {
return &authorizer{
username: username,
password: password,
client: &http.Client{
Transport: internal.GetHTTPTransport(insecure),
Transport: commonhttp.GetHTTPTransport(trType),
},
}
}

36
src/pkg/registry/client.go
View File

@ -27,10 +27,10 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/manifestlist"
// register oci manifest unmarshal function
_ "github.com/docker/distribution/manifest/ocischema"
_ "github.com/docker/distribution/manifest/ocischema" // register oci manifest unmarshal function
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/internal"
ierror "github.com/goharbor/harbor/src/internal/error"
@ -54,6 +54,13 @@ var (
schema2.MediaTypeManifest,
schema1.MediaTypeSignedManifest,
}
localRegistryURL = map[string]bool{
"http://registry:5000": true,
"https://registry:5443": true,
"http://core:8080": true,
"https://core:10443": true,
}
)
// const definition
@ -99,22 +106,41 @@ type Client interface {
// of the registry automatically and calls the corresponding underlying authorizers(basic/bearer) to
// do the auth work. If a customized authorizer is needed, use "NewClientWithAuthorizer" instead
func NewClient(url, username, password string, insecure bool) Client {
var transportType uint
if insecure {
transportType = commonhttp.InsecureTransport
} else {
transportType = commonhttp.SecureTransport
}
if _, ok := localRegistryURL[strings.TrimRight(url, "/")]; ok {
transportType = commonhttp.InternalTransport
}
return &client{
url: url,
authorizer: auth.NewAuthorizer(username, password, insecure),
authorizer: auth.NewAuthorizer(username, password, transportType),
client: &http.Client{
Transport: internal.GetHTTPTransport(insecure),
Transport: commonhttp.GetHTTPTransport(transportType),
},
}
}
// NewClientWithAuthorizer creates a registry client with the provided authorizer
func NewClientWithAuthorizer(url string, authorizer internal.Authorizer, insecure bool) Client {
var transportType uint
if insecure {
transportType = commonhttp.InsecureTransport
} else {
transportType = commonhttp.SecureTransport
}
if _, ok := localRegistryURL[strings.TrimRight(url, "/")]; ok {
transportType = commonhttp.InternalTransport
}
return &client{
url: url,
authorizer: authorizer,
client: &http.Client{
Transport: internal.GetHTTPTransport(insecure),
Transport: commonhttp.GetHTTPTransport(transportType),
},
}
}

6
src/pkg/signature/notary/helper.go
View File

@ -17,17 +17,17 @@ package notary
import (
"encoding/hex"
"fmt"
"github.com/goharbor/harbor/src/internal"
model2 "github.com/goharbor/harbor/src/pkg/signature/notary/model"
"net/http"
"os"
"path"
"strings"
"github.com/docker/distribution/registry/auth/token"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/core/config"
tokenutil "github.com/goharbor/harbor/src/core/service/token"
model2 "github.com/goharbor/harbor/src/pkg/signature/notary/model"
"github.com/theupdateframework/notary"
"github.com/theupdateframework/notary/client"
"github.com/theupdateframework/notary/trustpinning"
@ -82,7 +82,7 @@ func GetTargets(notaryEndpoint string, username string, fqRepo string) ([]model2
authorizer := &notaryAuthorizer{
token: t.Token,
}
tr := NewTransport(internal.GetHTTPTransport(), authorizer)
tr := NewTransport(commonhttp.GetHTTPTransport(commonhttp.DefaultTransport), authorizer)
gun := data.GUN(fqRepo)
notaryRepo, err := client.NewFileCachedRepository(notaryCachePath, gun, notaryEndpoint, tr, mockRetriever, trustPin)
if err != nil {

15
src/replication/adapter/aliacr/adapter.go
View File

@ -4,9 +4,7 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/docker/distribution/registry/client/auth/challenge"
"github.com/goharbor/harbor/src/internal"
"github.com/goharbor/harbor/src/pkg/registry/auth/bearer"
"net/http"
"path/filepath"
"regexp"
@ -14,8 +12,11 @@ import (
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
"github.com/aliyun/alibaba-cloud-sdk-go/services/cr"
"github.com/docker/distribution/registry/client/auth/challenge"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/registry/auth/bearer"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
"github.com/goharbor/harbor/src/replication/model"
@ -69,9 +70,13 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
}
func ping(registry *model.Registry) (string, string, error) {
client := &http.Client{
Transport: internal.GetHTTPTransport(registry.Insecure),
client := &http.Client{}
if registry.Insecure {
client.Transport = commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
} else {
client.Transport = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
}
resp, err := client.Get(registry.URL + "/v2/")
if err != nil {
return "", "", err

20
src/replication/adapter/awsecr/adapter.go
View File

@ -16,7 +16,6 @@ package awsecr
import (
"errors"
"github.com/goharbor/harbor/src/internal"
"net/http"
"regexp"
@ -25,6 +24,7 @@ import (
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils/log"
adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/adapter/native"
@ -245,11 +245,18 @@ func (a *adapter) createRepository(repository string) error {
if a.region == "" {
return errors.New("no region parsed")
}
var tr *http.Transport
if a.registry.Insecure {
tr = commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
} else {
tr = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
}
config := &aws.Config{
Credentials: cred,
Region: &a.region,
HTTPClient: &http.Client{
Transport: internal.GetHTTPTransport(a.registry.Insecure),
Transport: tr,
},
}
if a.forceEndpoint != nil {
@ -287,11 +294,18 @@ func (a *adapter) DeleteManifest(repository, reference string) error {
if a.region == "" {
return errors.New("no region parsed")
}
var tr *http.Transport
if a.registry.Insecure {
tr = commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
} else {
tr = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
}
config := &aws.Config{
Credentials: cred,
Region: &a.region,
HTTPClient: &http.Client{
Transport: internal.GetHTTPTransport(a.registry.Insecure),
Transport: tr,
},
}
if a.forceEndpoint != nil {

20
src/replication/adapter/awsecr/auth.go
View File

@ -18,18 +18,19 @@ import (
"encoding/base64"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
commonhttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/http/modifier"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/internal"
"net/http"
"net/url"
"strings"
"time"
)
// Credential ...
@ -96,11 +97,18 @@ func (a *awsAuthCredential) getAuthorization() (string, string, string, *time.Ti
a.accessKey,
a.accessSecret,
"")
var tr *http.Transport
if a.insecure {
tr = commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
} else {
tr = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
}
config := &aws.Config{
Credentials: cred,
Region: &a.region,
HTTPClient: &http.Client{
Transport: internal.GetHTTPTransport(a.insecure),
Transport: tr,
},
}
if a.forceEndpoint != nil {

2
src/replication/adapter/native/adapter.go
View File

@ -19,8 +19,6 @@ import (
"fmt"
"sync"
"sync"
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/internal"

6
src/replication/event/handler.go
View File

@ -18,14 +18,14 @@ import (
"errors"
"fmt"
"github.com/goharbor/harbor/src/replication/util"
commonthttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/replication/config"
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/operation"
"github.com/goharbor/harbor/src/replication/policy"
"github.com/goharbor/harbor/src/replication/registry"
"github.com/goharbor/harbor/src/replication/util"
)
// Handler is the handler to handle event
@ -191,6 +191,6 @@ func GetLocalRegistry() *model.Registry {
// use secret to do the auth for the local Harbor
AccessSecret: config.Config.JobserviceSecret,
},
Insecure: true,
Insecure: !commonthttp.InternalTLSEnabled(),
}
}