Merge pull request #8301 from ninjadq/external_endpoint_support

Add supoort for external endpoint

make/harbor.yml

@@ -72,14 +72,25 @@ chart:
 log:
   # options are debug, info, warning, error, fatal
   level: info
-  # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
+  # configs for logs in local storage
-  rotate_count: 50
+  local:
-  # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
+    # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
-  # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
+    rotate_count: 50
-  # are all valid.
+    # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
-  rotate_size: 200M
+    # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
-  # The directory on your host that store log
+    # are all valid.
-  location: /var/log/harbor
+    rotate_size: 200M
+    # The directory on your host that store log
+    location: /var/log/harbor
+
+  # Uncomment following lines to enable external syslog endpoint.
+  # external_endpoint:
+  #   # protocol used to transmit log to external endpoint, options is tcp or udp
+  #   protocol: tcp
+  #   # The host of external endpoint
+  #   host: localhost
+  #   # Port of external endpoint
+  #   port: 5140
 
 #This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
 _version: 1.8.0

make/photon/log/rsyslog_docker.conf

@@ -1,8 +1,5 @@
 #  Rsyslog configuration file for docker.
-
+template(name="DynaFile" type="string" string="/var/log/docker/%programname%.log")
-template(name="DynaFile" type="string"
+if $programname != "rsyslogd" then {
-	string="/var/log/docker/%syslogtag:R,ERE,0,DFLT:[^[]*--end:secpath-replace%.log"
+      action(type="omfile" dynaFile="DynaFile")
-)
+}
-#if $programname == "docker" then ?DynaFile
-if $programname != "rsyslogd" then -?DynaFile
-

make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja

@@ -14,7 +14,8 @@ services:
       - SETUID
     volumes:
       - {{log_location}}/:/var/log/docker/:z
-      - ./common/config/log/:/etc/logrotate.d/:z
+      - ./common/config/log/logrotate.conf:/etc/logrotate.d/logrotate.conf:z
+      - ./common/config/log/rsyslog_docker.conf:/etc/rsyslog.d/rsyslog_docker.conf:z
     ports:
       - 127.0.0.1:1514:10514
     networks:

make/photon/prepare/templates/log/rsyslog_docker.conf.jinja

@@ -0,0 +1,11 @@
+# Rsyslog configuration file for docker.
+
+template(name="DynaFile" type="string" string="/var/log/docker/%programname%.log")
+
+if $programname != "rsyslogd" then {
+{%if log_external %}
+	action(type="omfwd" Target="{{log_ep_host}}" Port="{{log_ep_port}}" Protocol="{{log_ep_protocol}}" Template="RSYSLOG_SyslogProtocol23Format")
+{% else %}
+	action(type="omfile" dynaFile="DynaFile")
+{% endif %}
+}

make/photon/prepare/utils/configs.py

@@ -13,6 +13,14 @@ def validate(conf, **kwargs):
         if not conf.get("cert_key_path"):
             raise Exception("Error: The protocol is https but attribute ssl_cert_key is not set")
 
+    # log endpoint validate
+    if ('log_ep_host' in conf) and not conf['log_ep_host']:
+        raise Exception('Error: must set log endpoint host to enable external host')
+    if ('log_ep_port' in conf) and not conf['log_ep_port']:
+        raise Exception('Error: must set log endpoint port to enable external host')
+    if ('log_ep_protocol' in conf) and (conf['log_ep_protocol'] not in ['udp', 'tcp']):
+        raise Exception("Protocol in external log endpoint must be one of 'udp' or 'tcp' ")
+
     # Storage validate
     valid_storage_drivers = ["filesystem", "azure", "gcs", "s3", "swift", "oss"]
     storage_provider_name = conf.get("storage_provider_name")
@@ -183,14 +191,27 @@ def parse_yaml_config(config_file_path):
     # Log configs
     allowed_levels = ['debug', 'info', 'warning', 'error', 'fatal']
     log_configs = configs.get('log') or {}
-    config_dict['log_location'] = log_configs["location"]
+
-    config_dict['log_rotate_count'] = log_configs["rotate_count"]
-    config_dict['log_rotate_size'] = log_configs["rotate_size"]
     log_level = log_configs['level']
     if log_level not in allowed_levels:
         raise Exception('log level must be one of debug, info, warning, error, fatal')
     config_dict['log_level'] = log_level.lower()
 
+    # parse local log related configs
+    local_logs = log_configs.get('local') or {}
+    if local_logs:
+        config_dict['log_location'] = local_logs.get('location') or '/var/log/harbor'
+        config_dict['log_rotate_count'] = local_logs.get('rotate_count') or 50
+        config_dict['log_rotate_size'] = local_logs.get('rotate_size') or '200M'
+
+    # parse external log endpoint related configs
+    if log_configs.get('external_endpoint'):
+        config_dict['log_external'] = True
+        config_dict['log_ep_protocol'] = log_configs['external_endpoint']['protocol']
+        config_dict['log_ep_host'] = log_configs['external_endpoint']['host']
+        config_dict['log_ep_port'] = log_configs['external_endpoint']['port']
+    else:
+        config_dict['log_external'] = False
 
     # external DB, optional, if external_db enabled, it will cover the database config
     external_db_configs = configs.get('external_database') or {}
@@ -202,7 +223,7 @@ def parse_yaml_config(config_file_path):
         config_dict['harbor_db_username'] = external_db_configs['harbor']['username']
         config_dict['harbor_db_password'] = external_db_configs['harbor']['password']
         config_dict['harbor_db_sslmode'] = external_db_configs['harbor']['ssl_mode']
-        # clari db
+        # clair db
         config_dict['clair_db_host'] = external_db_configs['clair']['host']
         config_dict['clair_db_port'] = external_db_configs['clair']['port']
         config_dict['clair_db_name'] = external_db_configs['clair']['db_name']

make/photon/prepare/utils/docker_compose.py

@@ -33,17 +33,25 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
         'with_chartmuseum': with_chartmuseum
     }
 
+    # for gcs
     storage_config = configs.get('storage_provider_config') or {}
     if storage_config.get('keyfile') and configs['storage_provider_name'] == 'gcs':
         rendering_variables['gcs_keyfile'] = storage_config['keyfile']
 
+    # for http
     if configs['protocol'] == 'https':
         rendering_variables['cert_key_path'] = configs['cert_key_path']
         rendering_variables['cert_path'] = configs['cert_path']
         rendering_variables['https_port'] = configs['https_port']
 
+    # for uaa
     uaa_config = configs.get('uaa') or {}
     if uaa_config.get('ca_file'):
         rendering_variables['uaa_ca_file'] = uaa_config['ca_file']
 
+    # for log
+    log_ep_host = configs.get('log_ep_host')
+    if log_ep_host:
+        rendering_variables['external_log_endpoint'] = True
+
     render_jinja(docker_compose_template_path, docker_compose_yml_path, **rendering_variables)

make/photon/prepare/utils/log.py

@@ -5,9 +5,15 @@ from utils.misc import prepare_config_dir
 from utils.jinja import render_jinja
 
 log_config_dir = os.path.join(config_dir, "log")
+
+# logrotate config file
 logrotate_template_path = os.path.join(templates_dir, "log", "logrotate.conf.jinja")
 log_rotate_config = os.path.join(config_dir, "log", "logrotate.conf")
 
+# syslog docker config file
+log_syslog_docker_template_path = os.path.join(templates_dir, 'log', 'rsyslog_docker.conf.jinja')
+log_syslog_docker_config = os.path.join(config_dir, 'log', 'rsyslog_docker.conf')
+
 def prepare_log_configs(config_dict):
     prepare_config_dir(log_config_dir)
 
@@ -17,4 +23,13 @@ def prepare_log_configs(config_dict):
         log_rotate_config,
         uid=DEFAULT_UID,
         gid=DEFAULT_GID,
-        **config_dict)
+        **config_dict)
+
+   # Render syslog docker config
+    render_jinja(
+        log_syslog_docker_template_path,
+        log_syslog_docker_config,
+        uid=DEFAULT_UID,
+        gid=DEFAULT_GID,
+        **config_dict
+   )