From 17d8b7b813a6beb4b9bdefaafb99c703011553cf Mon Sep 17 00:00:00 2001 From: stonezdj Date: Thu, 16 Dec 2021 16:00:07 +0800 Subject: [PATCH] Add upload purge config to registry/config.yml Enable the uploadpurging by default Fixes #15641 Signed-off-by: stonezdj --- make/harbor.yml.tmpl | 9 ++ make/photon/prepare/models.py | 30 +++++++ .../templates/registry/config.yml.jinja | 11 +++ make/photon/prepare/test_purgeuploads.py | 88 +++++++++++++++++++ make/photon/prepare/utils/configs.py | 9 +- 5 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 make/photon/prepare/test_purgeuploads.py diff --git a/make/harbor.yml.tmpl b/make/harbor.yml.tmpl index 4e7fc92fa..759cdc6a9 100644 --- a/make/harbor.yml.tmpl +++ b/make/harbor.yml.tmpl @@ -228,3 +228,12 @@ proxy: # # compression: false # # insecure: true # # timeout: 10s + +# enable purge _upload directories +upload_purging: + enabled: true + # remove files in _upload directories which exist for a period of time, default is one week. + age: 168h + # the interval of the purge operations + interval: 24h + dryrun: false diff --git a/make/photon/prepare/models.py b/make/photon/prepare/models.py index 833b36508..38f75a69b 100644 --- a/make/photon/prepare/models.py +++ b/make/photon/prepare/models.py @@ -204,3 +204,33 @@ class Trace: self.jaeger.validate() elif self.otel.enabled: self.otel.validate() + +class PurgeUpload: + def __init__(self, config: dict): + if not config: + self.enabled = False + self.enabled = config.get('enabled') + self.age = config.get('age') or '168h' + self.interval = config.get('interval') or '24h' + self.dryrun = config.get('dryrun') or False + return + + def validate(self): + if not self.enabled: + return + # age should end with h + if not isinstance(self.age, str) or not self.age.endswith('h'): + raise Exception('purge upload age should set with with nh, n is the number of hour') + # interval should larger than 2h + age = self.age[:-1] + if not age.isnumeric() or int(age) < 2 : + raise Exception('purge upload age should set with with nh, n is the number of hour and n should not be less than 2') + + # interval should end with h + if not isinstance(self.interval, str) or not self.interval.endswith('h'): + raise Exception('purge upload interval should set with with nh, n is the number of hour') + # interval should larger than 2h + interval = self.interval[:-1] + if not interval.isnumeric() or int(interval) < 2 : + raise Exception('purge upload interval should set with with nh, n is the number of hour and n should not beless than 2') + return diff --git a/make/photon/prepare/templates/registry/config.yml.jinja b/make/photon/prepare/templates/registry/config.yml.jinja index 3823edab0..19e195d9d 100644 --- a/make/photon/prepare/templates/registry/config.yml.jinja +++ b/make/photon/prepare/templates/registry/config.yml.jinja @@ -9,7 +9,18 @@ storage: {{storage_provider_info}} maintenance: uploadpurging: +{% if purge_upload.enabled %} + enabled: true + age: {{ purge_upload.age }} + interval: {{ purge_upload.interval }} + {% if purge_upload.dryrun %} + dryrun: true + {% else %} + dryrun: false + {% endif %} +{% else %} enabled: false +{% endif %} delete: enabled: true {% if storage_redirect_disabled %} diff --git a/make/photon/prepare/test_purgeuploads.py b/make/photon/prepare/test_purgeuploads.py new file mode 100644 index 000000000..5fa76ebab --- /dev/null +++ b/make/photon/prepare/test_purgeuploads.py @@ -0,0 +1,88 @@ + +import unittest +from models import PurgeUpload + +class TestPurgeUploadsDefault(unittest.TestCase): + def test_validate_config(self): + purge_config = dict([ + ('enabled',True), + ('age','168h'), + ('interval','24h'), + ('dryrun', False), + ]) + cfg = PurgeUpload(purge_config) + cfg.validate() + + def test_validate_config(self): + purge_config = dict([ + ('enabled','false'), + ('age','168h'), + ('interval','24h'), + ('dryrun', 'false'), + ]) + cfg = PurgeUpload(purge_config) + cfg.validate() + + def test_validate_config_2hour(self): + purge_config = dict([ + ('enabled',True), + ('age','2h'), + ('interval','2h'), + ('dryrun', False), + ]) + cfg = PurgeUpload(purge_config) + cfg.validate() + + def test_validate_config_1hour(self): + purge_config = dict([ + ('enabled',True), + ('age','1h'), + ('interval','1h'), + ('dryrun', False), + ]) + cfg = PurgeUpload(purge_config) + with self.assertRaises(Exception): + cfg.validate() + + def test_validate_config_invalid_format(self): + purge_config = dict([ + ('enabled',True), + ('age','1s'), + ('interval','1s'), + ('dryrun', False), + ]) + cfg = PurgeUpload(purge_config) + with self.assertRaises(Exception): + cfg.validate() + + def test_validate_config_invalid_format(self): + purge_config = dict([ + ('enabled',True), + ('age',168), + ('interval',24), + ('dryrun', False), + ]) + cfg = PurgeUpload(purge_config) + with self.assertRaises(Exception): + cfg.validate() + + def test_validate_config_disabled_invalid_format(self): + purge_config = dict([ + ('enabled',"false"), + ('age','ssh'), + ('interval','ssh'), + ('dryrun', False), + ]) + cfg = PurgeUpload(purge_config) + cfg.validate() + + def test_validate_config_invalid_string(self): + purge_config = dict([ + ('enabled',True), + ('age','ssh'), + ('interval','ssh'), + ('dryrun', False), + ]) + cfg = PurgeUpload(purge_config) + with self.assertRaises(Exception): + cfg.validate() diff --git a/make/photon/prepare/utils/configs.py b/make/photon/prepare/utils/configs.py index 3b8506d46..77a0a5c92 100644 --- a/make/photon/prepare/utils/configs.py +++ b/make/photon/prepare/utils/configs.py @@ -3,7 +3,7 @@ import os import yaml from urllib.parse import urlencode from g import versions_file_path, host_root_dir, DEFAULT_UID, INTERNAL_NO_PROXY_DN -from models import InternalTLS, Metric, Trace +from models import InternalTLS, Metric, Trace, PurgeUpload from utils.misc import generate_random_string, owner_can_read, other_can_read default_db_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns @@ -78,6 +78,9 @@ def validate(conf: dict, **kwargs): if conf.get('trace'): conf['trace'].validate() + + if conf.get('purge_upload'): + conf['purge_upload'].validate() def parse_versions(): if not versions_file_path.is_file(): @@ -346,6 +349,10 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu # config_dict['notary_url'] = 'http://notary-server:4443' config_dict['chart_repository_url'] = 'https://chartmuseum:9443' + # purge upload configs + purge_upload_config = configs.get('upload_purging') + config_dict['purge_upload'] = PurgeUpload(purge_upload_config or {}) + return config_dict