Merge pull request #3571 from ywk253100/171107_log_rotate

Improve log rotation configurability

make/common/templates/log/env

@@ -1 +0,0 @@
-LOG_ROTATE_DAYS=$log_rotate_days

make/common/templates/log/logrotate.conf

@@ -0,0 +1,8 @@
+/var/log/docker/*.log {
+        rotate $log_rotate_count
+        size $log_rotate_size
+        copytruncate
+        compress
+        missingok
+        nodateext
+}

make/docker-compose.tpl

@@ -3,11 +3,10 @@ services:
   log:
     image: vmware/harbor-log:__version__
     container_name: harbor-log 
-    env_file:
-      - ./common/config/log/env
     restart: always
     volumes:
       - /var/log/harbor/:/var/log/docker/:z
+      - ./common/config/log/:/etc/logrotate.d/:z
     ports:
       - 127.0.0.1:1514:10514
     networks:

make/harbor.cfg

@@ -34,8 +34,12 @@ admiral_url = NA
 #Please update it before deployment, subsequent update will cause Clair's API server and Harbor unable to access Clair's database.
 clair_db_password = password
 
-#The logs n days before will be compressed
+#Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
-log_rotate_days = 3
+log_rotate_count = 50
+#Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
+#If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
+#are all valid.
+log_rotate_size = 200M
 
 #NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
 #only take effect in the first boot, the subsequent changes of these properties 

make/photon/log/Dockerfile

@@ -1,7 +1,7 @@
 FROM vmware/photon:1.0
 
 RUN tdnf distro-sync -y || echo \
-    && tdnf install -y cronie rsyslog shadow tar gzip sudo net-tools\
+    && tdnf install -y cronie rsyslog logrotate shadow tar gzip sudo net-tools\
     && mkdir /etc/rsyslog.d/ \
     && mkdir /var/spool/rsyslog \
     && groupadd -r -g 10000 syslog && useradd --no-log-init -r -g 10000 -u 10000 syslog \
@@ -9,19 +9,19 @@ RUN tdnf distro-sync -y || echo \
 
 COPY rsyslog.conf /etc/rsyslog.conf
 
-# notes: file name cannot contain dot, or the script will not run
-COPY rotate.sh /etc/cron.daily/rotate
-
 # rsyslog configuration file for docker
 COPY rsyslog_docker.conf /etc/rsyslog.d/
 
+# run logrotate hourly
+RUN mv /etc/cron.daily/logrotate /etc/cron.hourly/logrotate
+
 COPY start.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/start.sh && \
     chown -R 10000:10000 /run 
 
 HEALTHCHECK CMD netstat -ltu|grep 10514
 
-VOLUME /var/log/docker/ /run/
+VOLUME /var/log/docker/ /run/ /etc/logrotate.d/
 
 EXPOSE 10514
 

make/photon/log/rotate.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-set -e
-echo "Log rotate starting..."
-
-#The logs n days before will be compressed.
-n=$LOG_ROTATE_DAYS
-if [ -z "$n" ]
-then
-	n=3
-fi
-
-echo "logs rotate days: $n"
-
-path=/var/log/docker
-
-list=""
-n_days_before=$(($(date +%s) - 3600*24*$n))
-for dir in $(ls $path | grep -v "tar.gz");
-do
-	if [ $(date --date=$dir +%s) -lt $n_days_before ]
-	then
-		echo "$dir will be compressed"
-		list="$list $dir"
-	fi
-done
-
-if [ -n "$list" ]
-then
-	cd $path
-	tar --remove-files -zcvf $(date -d @$n_days_before +%F)-.tar.gz $list
-fi
-
-echo "Log rotate finished."

make/photon/log/rsyslog_docker.conf

@@ -1,7 +1,7 @@
 #  Rsyslog configuration file for docker.
 
 template(name="DynaFile" type="string"
-	string="/var/log/docker/%$now%/%syslogtag:R,ERE,0,DFLT:[^[]*--end:secpath-replace%.log"
+	string="/var/log/docker/%syslogtag:R,ERE,0,DFLT:[^[]*--end:secpath-replace%.log"
 )
 #if $programname == "docker" then ?DynaFile
 if $programname != "rsyslogd" then -?DynaFile

make/prepare

@@ -162,7 +162,8 @@ uaa_clientid = rcp.get("configuration", "uaa_clientid")
 uaa_clientsecret = rcp.get("configuration", "uaa_clientsecret")
 uaa_ca_root = rcp.get("configuration", "uaa_ca_root")
 secret_key = get_secret_key(secretkey_path)
-log_rotate_days = rcp.get("configuration", "log_rotate_days")
+log_rotate_count = rcp.get("configuration", "log_rotate_count")
+log_rotate_size = rcp.get("configuration", "log_rotate_size")
 ########
 
 ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))  
@@ -190,7 +191,7 @@ db_conf_env = os.path.join(config_dir, "db", "env")
 job_conf_env = os.path.join(config_dir, "jobservice", "env")
 nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf")
 cert_dir = os.path.join(config_dir, "nginx", "cert")
-log_conf_env = os.path.join(config_dir, "log", "env") 
+log_rotate_config = os.path.join(config_dir, "log", "logrotate.conf") 
 
 if protocol == "https":
     target_cert_path = os.path.join(cert_dir, os.path.basename(cert_path))
@@ -273,9 +274,10 @@ render(os.path.join(templates_dir, "jobservice", "env"),
         ui_secret=ui_secret,
         jobservice_secret=jobservice_secret)
 		
-render(os.path.join(templates_dir, "log", "env"),
+render(os.path.join(templates_dir, "log", "logrotate.conf"),
-        log_conf_env,
+        log_rotate_config,
-        log_rotate_days=log_rotate_days)
+        log_rotate_count=log_rotate_count,
+		log_rotate_size=log_rotate_size)
 
 print("Generated configuration file: %s" % jobservice_conf)
 shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf)