Enhance: Create shared to store shared ca

this shared ca will mount to all harbor components

Signed-off-by: DQ <dengq@vmware.com>

make/photon/prepare/commands/prepare.py

@@ -7,7 +7,7 @@ import click
 
 from utils.misc import delfile
 from utils.configs import validate, parse_yaml_config
-from utils.cert import prepare_registry_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key
+from utils.cert import prepare_registry_ca, SSL_CERT_KEY_PATH, SSL_CERT_PATH, get_secret_key, prepare_trust_ca
 from utils.db import prepare_db
 from utils.jobservice import prepare_job_service
 from utils.registry import prepare_registry
@@ -52,6 +52,7 @@ def prepare(conf, with_notary, with_clair, with_trivy, with_chartmuseum):
     prepare_job_service(config_dict)
     prepare_redis(config_dict)
     prepare_tls(config_dict)
+    prepare_trust_ca(config_dict)
 
     get_secret_key(secret_key_dir)
 

make/photon/prepare/g.py

@@ -26,6 +26,7 @@ trust_ca_dir = secret_dir.joinpath('keys', 'trust_ca')
 internal_tls_dir = secret_dir.joinpath('tls')
 
 storage_ca_bundle_filename = 'storage_ca_bundle.crt'
+internal_ca_filename = 'harbor_internal_ca.crt'
 
 old_private_key_pem_path = Path('/config/core/private_key.pem')
 old_crt_path = Path('/config/registry/root.crt')
@@ -39,6 +40,7 @@ versions_file_path = Path('/usr/src/app/versions')
 
 cert_dir = config_dir.joinpath("nginx", "cert")
 core_cert_dir = config_dir.joinpath("core", "certificates")
+shared_cert_dir = config_dir.joinpath("shared", "trust-certificates")
 
 INTERNAL_NO_PROXY_DN = {
     '127.0.0.1',

make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja

@@ -40,6 +40,9 @@ services:
       - type: bind
         source: {{data_volume}}/secret/registry/root.crt
         target: /etc/registry/root.crt
+      - type: bind
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
 {% if gcs_keyfile %}
       - type: bind
         source: {{gcs_keyfile}}
@@ -47,11 +50,8 @@ services:
 {% endif %}
 {%if internal_tls.enabled %}
       - type: bind
-        source: {{internal_tls.core_key_path}}
+        source: {{internal_tls.core_crt_path}}
         target: /harbor_cust_cert/core.crt
-      - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /harbor_cust_cert/harbor_internal_ca.crt
       - type: bind
         source: {{internal_tls.registry_crt_path}}
         target: /etc/harbor/tls/registry.crt
@@ -90,10 +90,10 @@ services:
       - type: bind
         source: ./common/config/registryctl/config.yml
         target: /etc/registryctl/config.yml
-{%if internal_tls.enabled %}
       - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /harbor_cust_cert/harbor_internal_ca.crt
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
+{%if internal_tls.enabled %}
       - type: bind
         source: {{internal_tls.registryctl_crt_path}}
         target: /etc/harbor/ssl/registryctl.crt
@@ -172,15 +172,15 @@ services:
       - type: bind
         source: {{data_volume}}/secret/keys/secretkey
         target: /etc/core/key
+      - type: bind
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
 {% if uaa_ca_file %}
       - type: bind
         source: {{uaa_ca_file}}
         target: /etc/core/certificates/uaa_ca.pem
 {% endif %}
 {%if internal_tls.enabled %}
-      - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /harbor_cust_cert/harbor_internal_ca.crt 
       - type: bind
         source: {{internal_tls.core_crt_path}}
         target: /etc/harbor/ssl/core.crt
@@ -257,21 +257,16 @@ services:
       - type: bind
         source: ./common/config/jobservice/config.yml
         target: /etc/jobservice/config.yml
-{%if internal_tls.enabled %}
       - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /harbor_cust_cert/harbor_internal_ca.crt
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
+{%if internal_tls.enabled %}
       - type: bind
         source: {{internal_tls.job_service_crt_path}}
         target: /etc/harbor/ssl/job_service.crt
       - type: bind
         source: {{internal_tls.job_service_key_path}}
         target: /etc/harbor/ssl/job_service.key
-{% endif %}
-{% if protocol == 'https' %}
-      - type: bind
-        source: {{data_volume}}/secret/cert/server.crt
-        target: /harbor_cust_cert/harbor_ca.crt
 {% endif %}
     networks:
       - harbor
@@ -336,10 +331,10 @@ services:
 {% if protocol == 'https' %}
       - {{data_volume}}/secret/cert:/etc/cert:z
 {% endif %}
-{%if internal_tls.enabled %}
       - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /etc/harbor/tls/harbor_internal_ca.crt
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
+{%if internal_tls.enabled %}
       - type: bind
         source: {{internal_tls.proxy_crt_path}}
         target: /etc/harbor/tls/proxy.crt
@@ -388,10 +383,10 @@ services:
       - type: bind
         source: {{data_volume}}/secret/registry/root.crt
         target: /etc/notary/root.crt
-{%if internal_tls.enabled %}
       - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /etc/harbor/ssl/harbor_internal_ca.crt
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
+{%if internal_tls.enabled %}
       - type: bind
         source: {{internal_tls.notary_server_crt_path}}
         target: /etc/harbor/ssl/notary_server.crt
@@ -429,10 +424,10 @@ services:
       - type: bind
         source: {{data_volume}}/secret/notary/notary-signer.key
         target: /etc/notary/notary-signer.key
-{%if internal_tls.enabled %}
       - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /etc/harbor/ssl/harbor_internal_ca.crt
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
+{%if internal_tls.enabled %}
       - type: bind
         source: {{internal_tls.notary_signer_crt_path}}
         target: /etc/harbor/ssl/notary_signer.crt
@@ -477,10 +472,10 @@ services:
       - type: bind
         source: ./common/config/clair/config.yaml
         target: /etc/clair/config.yaml
-{%if internal_tls.enabled %}
       - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /harbor_cust_cert/harbor_internal_ca.crt
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
+{%if internal_tls.enabled %}
       - type: bind
         source: {{internal_tls.clair_crt_path}}
         target: /etc/harbor/ssl/clair.crt
@@ -514,11 +509,11 @@ services:
 {% if external_redis == False %}
       - redis
 {% endif %}
-{%if internal_tls.enabled %}
     volumes:
       - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /harbor_cust_cert/harbor_internal_ca.crt 
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
+{%if internal_tls.enabled %}
       - type: bind
         source: {{internal_tls.clair_adapter_crt_path}}
         target: /etc/harbor/ssl/clair_adapter.crt
@@ -555,15 +550,10 @@ services:
       - type: bind
         source: {{data_volume}}/trivy-adapter/reports
         target: /home/scanner/.cache/reports
-{% if protocol == 'https' %}
       - type: bind
-        source: {{data_volume}}/secret/cert/server.crt
-        target: /harbor_cust_cert/harbor_ca.crt
-{% endif %}
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
 {% if internal_tls.enabled %}
-      - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /harbor_cust_cert/harbor_internal_ca.crt
       - type: bind
         source: {{internal_tls.trivy_adapter_crt_path}}
         target: /etc/harbor/ssl/trivy_adapter.crt
@@ -599,10 +589,10 @@ services:
     volumes:
       - {{data_volume}}/chart_storage:/chart_storage:z
       - ./common/config/chartserver:/etc/chartserver:z
-{%if internal_tls.enabled %}
       - type: bind
-        source: {{internal_tls.harbor_internal_ca_crt_path}}
-        target: /harbor_cust_cert/harbor_internal_ca.crt
+        source: ./common/config/shared/trust-certificates
+        target: /harbor_cust_cert
+{%if internal_tls.enabled %}
       - type: bind
         source: {{internal_tls.chartmuseum_crt_path}}
         target: /etc/harbor/ssl/chartmuseum.crt

make/photon/prepare/templates/nginx/nginx.https.conf.jinja

@@ -89,7 +89,7 @@ http {
 
       proxy_ssl_certificate         /etc/harbor/tls/proxy.crt;
       proxy_ssl_certificate_key     /etc/harbor/tls/proxy.key;
-      proxy_ssl_trusted_certificate /etc/harbor/tls/harbor_internal_ca.crt;
+      proxy_ssl_trusted_certificate /harbor_cust_cert/harbor_internal_ca.crt;
       proxy_ssl_verify_depth 2;
       proxy_ssl_verify        on;
       proxy_ssl_session_reuse on;
@@ -115,7 +115,7 @@ http {
 
       proxy_ssl_certificate         /etc/harbor/tls/proxy.crt;
       proxy_ssl_certificate_key     /etc/harbor/tls/proxy.key;
-      proxy_ssl_trusted_certificate /etc/harbor/tls/harbor_internal_ca.crt;
+      proxy_ssl_trusted_certificate /harbor_cust_cert/harbor_internal_ca.crt;
       proxy_ssl_verify_depth 2;
       proxy_ssl_verify        on;
       proxy_ssl_session_reuse on;
@@ -141,7 +141,7 @@ http {
 
       proxy_ssl_certificate         /etc/harbor/tls/proxy.crt;
       proxy_ssl_certificate_key     /etc/harbor/tls/proxy.key;
-      proxy_ssl_trusted_certificate /etc/harbor/tls/harbor_internal_ca.crt;
+      proxy_ssl_trusted_certificate /harbor_cust_cert/harbor_internal_ca.crt;
       proxy_ssl_verify_depth 2;
       proxy_ssl_verify        on;
       proxy_ssl_session_reuse on;
@@ -171,7 +171,7 @@ http {
 
       proxy_ssl_certificate         /etc/harbor/tls/proxy.crt;
       proxy_ssl_certificate_key     /etc/harbor/tls/proxy.key;
-      proxy_ssl_trusted_certificate /etc/harbor/tls/harbor_internal_ca.crt;
+      proxy_ssl_trusted_certificate /harbor_cust_cert/harbor_internal_ca.crt;
       proxy_ssl_verify_depth 2;
       proxy_ssl_verify        on;
       proxy_ssl_session_reuse on;
@@ -194,7 +194,7 @@ http {
 
       proxy_ssl_certificate         /etc/harbor/tls/proxy.crt;
       proxy_ssl_certificate_key     /etc/harbor/tls/proxy.key;
-      proxy_ssl_trusted_certificate /etc/harbor/tls/harbor_internal_ca.crt;
+      proxy_ssl_trusted_certificate /harbor_cust_cert/harbor_internal_ca.crt;
       proxy_ssl_verify_depth 2;
       proxy_ssl_verify        on;
       proxy_ssl_session_reuse on;

make/photon/prepare/utils/cert.py

@@ -2,8 +2,9 @@
 import os, subprocess, shutil
 from pathlib import Path
 from subprocess import DEVNULL
+import logging
 
-from g import DEFAULT_GID, DEFAULT_UID, trust_ca_dir, storage_ca_bundle_filename
+from g import DEFAULT_GID, DEFAULT_UID, shared_cert_dir, storage_ca_bundle_filename, internal_tls_dir, internal_ca_filename
 from .misc import (
     mark_file,
     generate_random_string,
@@ -99,33 +100,32 @@ def prepare_registry_ca(
         os.chown(private_key_pem_path, DEFAULT_UID, DEFAULT_GID)
 
 
-def prepare_trust_ca(**kwargs):
-    def f(path: str, file_name: str):
+def prepare_trust_ca(config_dict):
+    if shared_cert_dir.exists():
+        shutil.rmtree(shared_cert_dir)
+    shared_cert_dir.mkdir(parents=True, exist_ok=True)
 
+    internal_ca_src = internal_tls_dir.joinpath(internal_ca_filename)
+    ca_bundle_src = config_dict.get('registry_custom_ca_bundle_path')
+    for src_path, dst_filename in (
+        (internal_ca_src, internal_ca_filename),
+        (ca_bundle_src, storage_ca_bundle_filename)):
+        logging.info('copy {} to shared trust ca dir as name {} ...'.format(src_path, dst_filename))
         # check if source file valied
-        src_path = kwargs.get(path)
         if not src_path:
-            return
-        real_path = get_realpath(src_path)
-        if not real_path.exists():
-            raise Exception('ca file {} is not exist'.format(real_path))
-        if not real_path.is_file():
-            raise Exception('{} is not file'.format(real_path))
+            continue
+        real_src_path = get_realpath(str(src_path))
+        if not real_src_path.exists():
+            logging.info('ca file {} is not exist'.format(real_src_path))
+            continue
+        if not real_src_path.is_file():
+            logging.info('{} is not file'.format(real_src_path))
+            continue
 
-        dst_path = trust_ca_dir.joinpath(file_name)
-        # check destination dir exist
-        if not trust_ca_dir.exists():
-            trust_ca_dir.mkdir(parents=True)
-        else:
-            os.remove(dst_path)
+        dst_path = shared_cert_dir.joinpath(dst_filename)
 
         # copy src to dst
-        shutil.copy(src_path, dst_path)
+        shutil.copy2(real_src_path, dst_path)
 
         # change ownership and permission
-        mark_file(dst_path)
-
-    for p in (
-        ('internal_https_ca_path', 'harbor_internal_ca.crt'),
-        ('registry_custom_ca_bundle_path', storage_ca_bundle_filename)):
-        f(*p)
+        mark_file(dst_path, mode=0o644)

make/photon/prepare/utils/misc.py

@@ -175,5 +175,5 @@ def get_realpath(path: str) -> Path:
     """
 
     if os.path.isdir(host_root_dir):
-        return os.path.join(host_root_dir, path.lstrip('/'))
+        return host_root_dir.joinpath(path.lstrip('/'))
     return Path(path)