From 66b9699ac24e4416f0cd35c2713809bdbaaee027 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 8 Nov 2017 13:07:27 +0800 Subject: [PATCH] Improve log rotation configurability --- make/common/templates/log/env | 1 - make/common/templates/log/logrotate.conf | 8 ++++++ make/docker-compose.tpl | 3 +-- make/harbor.cfg | 8 ++++-- make/photon/log/Dockerfile | 10 +++---- make/photon/log/rotate.sh | 33 ------------------------ make/photon/log/rsyslog_docker.conf | 2 +- make/prepare | 12 +++++---- 8 files changed, 28 insertions(+), 49 deletions(-) delete mode 100644 make/common/templates/log/env create mode 100644 make/common/templates/log/logrotate.conf delete mode 100755 make/photon/log/rotate.sh diff --git a/make/common/templates/log/env b/make/common/templates/log/env deleted file mode 100644 index d0c9d994f..000000000 --- a/make/common/templates/log/env +++ /dev/null @@ -1 +0,0 @@ -LOG_ROTATE_DAYS=$log_rotate_days \ No newline at end of file diff --git a/make/common/templates/log/logrotate.conf b/make/common/templates/log/logrotate.conf new file mode 100644 index 000000000..bc63e78de --- /dev/null +++ b/make/common/templates/log/logrotate.conf @@ -0,0 +1,8 @@ +/var/log/docker/*.log { + rotate $log_rotate_count + size $log_rotate_size + copytruncate + compress + missingok + nodateext +} \ No newline at end of file diff --git a/make/docker-compose.tpl b/make/docker-compose.tpl index 18c8d6cf7..0bf52032a 100644 --- a/make/docker-compose.tpl +++ b/make/docker-compose.tpl @@ -3,11 +3,10 @@ services: log: image: vmware/harbor-log:__version__ container_name: harbor-log - env_file: - - ./common/config/log/env restart: always volumes: - /var/log/harbor/:/var/log/docker/:z + - ./common/config/log/:/etc/logrotate.d/:z ports: - 127.0.0.1:1514:10514 networks: diff --git a/make/harbor.cfg b/make/harbor.cfg index 34af8b960..f926c0429 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -34,8 +34,12 @@ admiral_url = NA #Please update it before deployment, subsequent update will cause Clair's API server and Harbor unable to access Clair's database. clair_db_password = password -#The logs n days before will be compressed -log_rotate_days = 3 +#Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. +log_rotate_count = 50 +#Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. +#If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G +#are all valid. +log_rotate_size = 200M #NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES #only take effect in the first boot, the subsequent changes of these properties diff --git a/make/photon/log/Dockerfile b/make/photon/log/Dockerfile index c82bf07f8..7b31c75b5 100644 --- a/make/photon/log/Dockerfile +++ b/make/photon/log/Dockerfile @@ -1,7 +1,7 @@ FROM vmware/photon:1.0 RUN tdnf distro-sync -y || echo \ - && tdnf install -y cronie rsyslog shadow tar gzip sudo net-tools\ + && tdnf install -y cronie rsyslog logrotate shadow tar gzip sudo net-tools\ && mkdir /etc/rsyslog.d/ \ && mkdir /var/spool/rsyslog \ && groupadd -r -g 10000 syslog && useradd --no-log-init -r -g 10000 -u 10000 syslog \ @@ -9,19 +9,19 @@ RUN tdnf distro-sync -y || echo \ COPY rsyslog.conf /etc/rsyslog.conf -# notes: file name cannot contain dot, or the script will not run -COPY rotate.sh /etc/cron.daily/rotate - # rsyslog configuration file for docker COPY rsyslog_docker.conf /etc/rsyslog.d/ +# run logrotate hourly +RUN mv /etc/cron.daily/logrotate /etc/cron.hourly/logrotate + COPY start.sh /usr/local/bin/ RUN chmod +x /usr/local/bin/start.sh && \ chown -R 10000:10000 /run HEALTHCHECK CMD netstat -ltu|grep 10514 -VOLUME /var/log/docker/ /run/ +VOLUME /var/log/docker/ /run/ /etc/logrotate.d/ EXPOSE 10514 diff --git a/make/photon/log/rotate.sh b/make/photon/log/rotate.sh deleted file mode 100755 index 5f102dffc..000000000 --- a/make/photon/log/rotate.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -e -echo "Log rotate starting..." - -#The logs n days before will be compressed. -n=$LOG_ROTATE_DAYS -if [ -z "$n" ] -then - n=3 -fi - -echo "logs rotate days: $n" - -path=/var/log/docker - -list="" -n_days_before=$(($(date +%s) - 3600*24*$n)) -for dir in $(ls $path | grep -v "tar.gz"); -do - if [ $(date --date=$dir +%s) -lt $n_days_before ] - then - echo "$dir will be compressed" - list="$list $dir" - fi -done - -if [ -n "$list" ] -then - cd $path - tar --remove-files -zcvf $(date -d @$n_days_before +%F)-.tar.gz $list -fi - -echo "Log rotate finished." diff --git a/make/photon/log/rsyslog_docker.conf b/make/photon/log/rsyslog_docker.conf index ce565c705..a21cc5078 100644 --- a/make/photon/log/rsyslog_docker.conf +++ b/make/photon/log/rsyslog_docker.conf @@ -1,7 +1,7 @@ # Rsyslog configuration file for docker. template(name="DynaFile" type="string" - string="/var/log/docker/%$now%/%syslogtag:R,ERE,0,DFLT:[^[]*--end:secpath-replace%.log" + string="/var/log/docker/%syslogtag:R,ERE,0,DFLT:[^[]*--end:secpath-replace%.log" ) #if $programname == "docker" then ?DynaFile if $programname != "rsyslogd" then -?DynaFile diff --git a/make/prepare b/make/prepare index 3702bea46..5fb7c3380 100755 --- a/make/prepare +++ b/make/prepare @@ -162,7 +162,8 @@ uaa_clientid = rcp.get("configuration", "uaa_clientid") uaa_clientsecret = rcp.get("configuration", "uaa_clientsecret") uaa_ca_root = rcp.get("configuration", "uaa_ca_root") secret_key = get_secret_key(secretkey_path) -log_rotate_days = rcp.get("configuration", "log_rotate_days") +log_rotate_count = rcp.get("configuration", "log_rotate_count") +log_rotate_size = rcp.get("configuration", "log_rotate_size") ######## ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16)) @@ -190,7 +191,7 @@ db_conf_env = os.path.join(config_dir, "db", "env") job_conf_env = os.path.join(config_dir, "jobservice", "env") nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf") cert_dir = os.path.join(config_dir, "nginx", "cert") -log_conf_env = os.path.join(config_dir, "log", "env") +log_rotate_config = os.path.join(config_dir, "log", "logrotate.conf") if protocol == "https": target_cert_path = os.path.join(cert_dir, os.path.basename(cert_path)) @@ -273,9 +274,10 @@ render(os.path.join(templates_dir, "jobservice", "env"), ui_secret=ui_secret, jobservice_secret=jobservice_secret) -render(os.path.join(templates_dir, "log", "env"), - log_conf_env, - log_rotate_days=log_rotate_days) +render(os.path.join(templates_dir, "log", "logrotate.conf"), + log_rotate_config, + log_rotate_count=log_rotate_count, + log_rotate_size=log_rotate_size) print("Generated configuration file: %s" % jobservice_conf) shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf)