diff --git a/tests/apitests/python/test_audit_log_forward.py b/tests/apitests/python/test_audit_log_forward.py index a73748068..304b58ce8 100644 --- a/tests/apitests/python/test_audit_log_forward.py +++ b/tests/apitests/python/test_audit_log_forward.py @@ -1,8 +1,11 @@ from __future__ import absolute_import import unittest +import requests +import json +import time -from testutils import ADMIN_CLIENT, LOG_PATH, harbor_server, suppress_urllib3_warning +from testutils import ADMIN_CLIENT, harbor_server, SYSLOG_ENDPOINT, ES_ENDPOINT, suppress_urllib3_warning from library.audit_log import Audit_Log from library.user import User from library.project import Project @@ -23,16 +26,13 @@ class TestAuditLogForword(unittest.TestCase, object): self.audit_log = Audit_Log() self.image = "hello-world" self.tag = "latest" - self.audit_log_path = LOG_PATH + "audit.log" - self.audit_log_forward_endpoint = "harbor-log:10514" + self.tag2 = "test" # 1. Reset audit log forword self.config.set_configurations_of_audit_log_forword("", False) def tearDown(self): # 1. Reset audit log forword self.config.set_configurations_of_audit_log_forword("", False) - # 2. Close audit log file - TestAuditLogForword.audit_log_file.close def testAuditLogForword(self): """ @@ -47,7 +47,7 @@ class TestAuditLogForword(unittest.TestCase, object): 6. Verify that the Audit Log should be in the log database; 7. Verify that the Audit Log should be in the audit.log; 8. Enable Skip Audit Log Database; - 9. Delete image(IA); + 9. Create a tag; 10. Verify that the Audit Log should not be in log database; 11. Verify that the Audit Log should be in the audit.log; 12. Verify that Skip Audit Log Database cannot be enabled without Audit Log Forward; @@ -70,10 +70,10 @@ class TestAuditLogForword(unittest.TestCase, object): self.config.set_configurations_of_audit_log_forword(skip_audit_log_database=True, expect_status_code=400) # 4 Enable Audit Log Forward - self.config.set_configurations_of_audit_log_forword(audit_log_forward_endpoint=self.audit_log_forward_endpoint, expect_status_code=200) + self.config.set_configurations_of_audit_log_forword(audit_log_forward_endpoint=SYSLOG_ENDPOINT, expect_status_code=200) # 4.1 Verify configuration configurations = self.config.get_configurations() - self.assertEqual(configurations.audit_log_forward_endpoint.value, self.audit_log_forward_endpoint) + self.assertEqual(configurations.audit_log_forward_endpoint.value, SYSLOG_ENDPOINT) self.assertFalse(configurations.skip_audit_log_database.value) # 5 Push a new image(IA) in project(PA) by user(UA) @@ -88,23 +88,17 @@ class TestAuditLogForword(unittest.TestCase, object): self.assertIsNotNone(first_audit_log.op_time) # 7. Verify that the Audit Log should be in the audit.log - TestAuditLogForword.audit_log_file = open(self.audit_log_path, "r") - latest_line = TestAuditLogForword.audit_log_file.readlines()[-1] - self.assertIn('operator="{}"'.format(user_name), latest_line) - self.assertIn('resourceType="artifact"', latest_line) - self.assertIn('action:create', latest_line) - self.assertIn('resource:{}:{}'.format(repo_name, tag), latest_line) - self.assertIn('time="20', latest_line) + self.assertTrue(self.verifyLogInSyslogService(user_name, "{}:{}".format(repo_name, tag), "artifact", "create")) # 8.1 Enable Skip Audit Log Database self.config.set_configurations_of_audit_log_forword(skip_audit_log_database=True) # 8.1 Verify configuration configurations = self.config.get_configurations() - self.assertEqual(configurations.audit_log_forward_endpoint.value, self.audit_log_forward_endpoint) + self.assertEqual(configurations.audit_log_forward_endpoint.value, SYSLOG_ENDPOINT) self.assertTrue(configurations.skip_audit_log_database.value) - # 9. Delete image(IA) - self.artifact.delete_artifact(project_name, self.image, self.tag, **user_client) + # 9. Create a tag + self.artifact.create_tag(project_name, self.image, self.tag, self.tag2, **user_client) # 10. Verify that the Audit Log should not be in log database second_audit_log = self.audit_log.get_latest_audit_log() @@ -115,15 +109,32 @@ class TestAuditLogForword(unittest.TestCase, object): self.assertEqual(first_audit_log.op_time, second_audit_log.op_time) # 11. Verify that the Audit Log should be in the audit.log - latest_line = TestAuditLogForword.audit_log_file.readlines()[-1] - self.assertIn('operator="{}"'.format(user_name), latest_line) - self.assertIn('resourceType="artifact"', latest_line) - self.assertIn('action:delete', latest_line) - self.assertIn('resource:{}'.format(repo_name), latest_line) - self.assertIn('time="20', latest_line) + self.assertTrue(self.verifyLogInSyslogService(user_name, "{}:{}".format(repo_name, self.tag2), "tag", "create")) # 12 Verify that Skip Audit Log Database cannot be enabled without Audit Log Forward self.config.set_configurations_of_audit_log_forword(audit_log_forward_endpoint="", expect_status_code=400) + + def verifyLogInSyslogService(self, username, resource, resource_type, operation, expected_count=1): + url = ES_ENDPOINT + "/_count" + payload = json.dumps({ + "query": { + "match": { + "message": { + "query": "operator=\"{}\" resource:{} resourceType=\"{}\" action:{}".format(username, resource, resource_type, operation), + "operator": "and" + } + } + } + }) + headers = { 'Content-Type': 'application/json' } + for _ in range(5): + response = requests.request("GET", url, headers=headers, data=payload) + self.assertEqual(response.status_code, 200) + response_json = response.json() + if response_json["count"] == expected_count: + return True + time.sleep(5) + return False if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/apitests/python/testutils.py b/tests/apitests/python/testutils.py index 48dbefd6d..78dcca78e 100644 --- a/tests/apitests/python/testutils.py +++ b/tests/apitests/python/testutils.py @@ -32,7 +32,8 @@ notary_url = os.environ.get('NOTARY_URL', 'https://'+harbor_server+':4443') DOCKER_USER = os.environ.get('DOCKER_USER', '') DOCKER_PWD = os.environ.get('DOCKER_PWD', '') METRIC_URL = os.environ.get('METRIC_URL', 'http://'+harbor_server+':9090') -LOG_PATH = os.environ.get('LOG_PATH', '/var/log/harbor/') +SYSLOG_ENDPOINT = os.environ.get('SYSLOG_ENDPOINT', '') +ES_ENDPOINT = os.environ.get('ES_ENDPOINT', '') BASE_IMAGE = dict(name='busybox', tag='latest') BASE_IMAGE_ABS_PATH_NAME = '/' + BASE_IMAGE['name'] + '.tar' diff --git a/tests/ci/api_run.sh b/tests/ci/api_run.sh index 6b4844984..83f60ffd8 100755 --- a/tests/ci/api_run.sh +++ b/tests/ci/api_run.sh @@ -21,7 +21,7 @@ set +e docker ps # run db auth api cases if [ "$1" = 'DB' ]; then - docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -v /var/log/harbor/:/var/log/harbor/ -w /drone $E2E_IMAGE robot --exclude proxy_cache -v DOCKER_USER:${DOCKER_USER} -v DOCKER_PWD:${DOCKER_PWD} -v ip:$2 -v ip1: -v http_get_ca:false -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_DB.robot + docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone $E2E_IMAGE robot --exclude proxy_cache -v DOCKER_USER:${DOCKER_USER} -v DOCKER_PWD:${DOCKER_PWD} -v ip:$2 -v ip1: -v http_get_ca:false -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_DB.robot elif [ "$1" = 'PROXY_CACHE' ]; then docker run -i --privileged -v $DIR/../../:/drone -v $DIR/../:/ca -w /drone $E2E_IMAGE robot --include setup --include proxy_cache -v DOCKER_USER:${DOCKER_USER} -v DOCKER_PWD:${DOCKER_PWD} -v ip:$2 -v ip1: -v http_get_ca:false -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot /drone/tests/robot-cases/Group0-BAT/API_DB.robot elif [ "$1" = 'LDAP' ]; then diff --git a/tests/resources/APITest-Util.robot b/tests/resources/APITest-Util.robot index 6b93e2466..1ccb4aa41 100644 --- a/tests/resources/APITest-Util.robot +++ b/tests/resources/APITest-Util.robot @@ -12,10 +12,16 @@ Setup API Test Retry Keyword N Times When Error 10 Make Swagger Client Harbor API Test - [Arguments] ${testcase_name} + [Arguments] ${testcase_name} &{param} ${current_dir}= Run pwd ${prev_lvl} Set Log Level NONE - ${rc} ${output}= Run And Return Rc And Output SWAGGER_CLIENT_PATH=${current_dir}/harborclient HARBOR_HOST=${ip} DOCKER_USER=${DOCKER_USER} DOCKER_PWD=${DOCKER_PWD} python ${testcase_name} + ${param_str}= Set Variable + IF &{param} != {} + FOR ${key} IN @{param.keys()} + ${param_str}= Set Variable ${param_str} ${key}=${param['${key}']} + END + END + ${rc} ${output}= Run And Return Rc And Output SWAGGER_CLIENT_PATH=${current_dir}/harborclient HARBOR_HOST=${ip} DOCKER_USER=${DOCKER_USER} DOCKER_PWD=${DOCKER_PWD} ${param_str} python ${testcase_name} ${prev_lvl} Set Log Level ${prev_lvl} Log ${output} Should Be Equal As Integers ${rc} 0 \ No newline at end of file diff --git a/tests/resources/Harbor-Pages/Configuration.robot b/tests/resources/Harbor-Pages/Configuration.robot index ba59b855f..55a5f7ad9 100644 --- a/tests/resources/Harbor-Pages/Configuration.robot +++ b/tests/resources/Harbor-Pages/Configuration.robot @@ -363,7 +363,7 @@ Edit A Distribution Set Audit Log Forward [Arguments] ${syslog_endpoint} ${expected_msg} Switch To System Settings - Run Keyword If '${syslog_endpoint}' == '${null}' Retry Clear Element Text By Press Keys ${audit_log_forward_syslog_endpoint_input_id} + Run Keyword If '${syslog_endpoint}' == '${null}' Press Keys ${audit_log_forward_syslog_endpoint_input_id} CTRL+a BACKSPACE ... ELSE Retry Text Input ${audit_log_forward_syslog_endpoint_input_id} ${syslog_endpoint} Retry Double Keywords When Error Retry Element Click ${config_save_button_xpath} Retry Wait Until Page Contains ${expected_msg} diff --git a/tests/resources/Harbor-Pages/Logs.robot b/tests/resources/Harbor-Pages/Logs.robot index ec1ed74ab..0d687dfe6 100644 --- a/tests/resources/Harbor-Pages/Logs.robot +++ b/tests/resources/Harbor-Pages/Logs.robot @@ -35,10 +35,10 @@ Verify Log Should Be Equal ${real_resource_type} ${resource_type} Should Be Equal ${real_operation} ${operation} -Verify Log In File - [Arguments] ${username} ${resource} ${resource_type} ${operation} ${audit_log_path}=${log_path}/audit.log - ${contents}= OperatingSystem.Get File ${audit_log_path} - @{lines}= Split to lines ${contents} - Log ${lines}[-1] - @{items} Create List operator="${username}" resource:${resource} resourceType="${resource_type}" action:${operation} time="20 - Should Contain Any ${lines}[-1] @{items} \ No newline at end of file +Verify Log In Syslog Service + [Arguments] ${username} ${resource} ${resource_type} ${operation} ${expected_count}=1 + ${data_raw}= Set Variable {"query": {"match": {"message": {"query": "operator=\\"${username}\\" resource:${resource} resourceType=\\"${resource_type}\\" action:${operation}","operator": "and"}}}} + ${cmd}= Set Variable curl -s -k -X GET '${ES_ENDPOINT}/_count' -H 'Content-Type: application/json' -d '${data_raw}' + ${json}= Run Curl And Return Json ${cmd} + ${count}= Set Variable ${json["count"]} + Should Be Equal As Integers ${count} ${expected_count} \ No newline at end of file diff --git a/tests/resources/SeleniumUtil.robot b/tests/resources/SeleniumUtil.robot index 9e273f1d0..72cf3d586 100644 --- a/tests/resources/SeleniumUtil.robot +++ b/tests/resources/SeleniumUtil.robot @@ -19,7 +19,6 @@ Resource Util.robot *** Variables *** ${download_directory} /drone/download ${files_directory} ${EXECDIR}/tests/files -${log_path} /var/log/harbor/ *** Keywords *** Start Selenium Standalone Server Locally diff --git a/tests/resources/Util.robot b/tests/resources/Util.robot index 13ee72778..bb733d8b0 100644 --- a/tests/resources/Util.robot +++ b/tests/resources/Util.robot @@ -144,16 +144,6 @@ Retry Clear Element Text @{param} Create List ${element_xpath} Retry Action Keyword Clear Element Text @{param} -Retry Clear Element Text By Press Keys - [Arguments] ${element_xpath} - ${value}= Get Value ${element_xpath} - ${value_length}= Get length ${value} - ${keys}= Create List - FOR ${idx} IN RANGE ${value_length} - Append To List ${keys} BACK_SPACE - END - Press Keys ${element_xpath} @{keys} - Retry Link Click [Arguments] ${element_xpath} @{param} Create List ${element_xpath} @@ -310,11 +300,9 @@ Retry File Should Not Exist Run Curl And Return Json [Arguments] ${curl_cmd} - ${json_data_file}= Set Variable ${CURDIR}${/}cur_user_info.json ${rc} ${output}= Run And Return Rc And Output ${curl_cmd} Should Be Equal As Integers 0 ${rc} - Create File ${json_data_file} ${output} - ${json}= Load Json From File ${json_data_file} + ${json}= Convert String To Json ${output} [Return] ${json} Log All diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index 0956d3309..46145ca2f 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -16,7 +16,6 @@ ${SERVER_API_ENDPOINT} ${SERVER_URL}/api &{SERVER_CONFIG} endpoint=${SERVER_API_ENDPOINT} verify_ssl=False *** Test Cases *** - Test Case - Garbage Collection [Tags] gc Harbor API Test ./tests/apitests/python/test_garbage_collection.py @@ -183,7 +182,10 @@ Test Case - Log Rotation Test Case - Log Forward [Tags] log_forward - Harbor API Test ./tests/apitests/python/test_audit_log_forward.py + ${SYSLOG_ENDPOINT_VALUE}= Get Variable Value ${SYSLOG_ENDPOINT} ${EMPTY} + ${ES_ENDPOINT_VALUE}= Get Variable Value ${ES_ENDPOINT} ${EMPTY} + Skip If '${SYSLOG_ENDPOINT_VALUE}' == '${EMPTY}' or '${ES_ENDPOINT_VALUE}' == '${EMPTY}' + Harbor API Test ./tests/apitests/python/test_audit_log_forward.py SYSLOG_ENDPOINT=${SYSLOG_ENDPOINT_VALUE} ES_ENDPOINT=${ES_ENDPOINT_VALUE} Test Case - Scan Data Export [Tags] scan_data_export diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index 5f7c01dbc..17e6baabf 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -928,11 +928,12 @@ Test Case - Audit Log And Purge Close Browser Test Case - Audit Log Forward - [Tags] audit_log_forward + [Tags] audit_log_forward need_syslog_endpoint + ${SYSLOG_ENDPOINT_VALUE}= Get Variable Value ${SYSLOG_ENDPOINT} ${EMPTY} + ${ES_ENDPOINT_VALUE}= Get Variable Value ${ES_ENDPOINT} ${EMPTY} + Skip If '${SYSLOG_ENDPOINT_VALUE}' == '${EMPTY}' or '${ES_ENDPOINT_VALUE}' == '${EMPTY}' Init Chrome Driver ${d}= Get Current Date result_format=%m%s - ${audit_log_path}= Set Variable ${log_path}/audit.log - ${syslog_endpoint}= Set Variable harbor-log:10514 ${test_endpoint}= Set Variable test.endpoint ${image}= Set Variable alpine ${tag1}= Set Variable 3.10 @@ -945,13 +946,13 @@ Test Case - Audit Log Forward Retry Wait Element Should Be Disabled ${skip_audit_log_database_checkbox} Set Audit Log Forward ${test_endpoint} bad request: could not connect to the audit endpoint: ${test_endpoint} # Set Audit Log Forward - Set Audit Log Forward ${syslog_endpoint} Configuration has been successfully saved. + Set Audit Log Forward ${SYSLOG_ENDPOINT} Configuration has been successfully saved. Wait Until Element Is Enabled ${skip_audit_log_database_checkbox} # create artifact Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} ${tag1} ${tag1} Switch To Logs Verify Log ${HARBOR_ADMIN} project${d}/${image}:${tag1} artifact create - Verify Log In File ${HARBOR_ADMIN} project${d}/${image}:${tag1} artifact create + Retry Action Keyword Verify Log In Syslog Service ${HARBOR_ADMIN} project${d}/${image}:${tag1} artifact create # Enable Skip Audit Log Database Enable Skip Audit Log Database Go Into Project project${d} @@ -961,7 +962,7 @@ Test Case - Audit Log Forward Add A New Tag ${tag2} Switch To Logs Verify Log ${HARBOR_ADMIN} project${d}/${image}:${tag1} artifact create - Verify Log In File ${HARBOR_ADMIN} project${d}/${image}:${tag2} artifact create + Retry Action Keyword Verify Log In Syslog Service ${HARBOR_ADMIN} project${d}/${image}:${tag2} tag create Set Audit Log Forward ${null} Configuration has been successfully saved. Retry Wait Element Should Be Disabled ${skip_audit_log_database_checkbox} Checkbox Should Not Be Selected ${skip_audit_log_database_checkbox} @@ -972,7 +973,7 @@ Test Case - Audit Log Forward Delete A Tag ${tag2} Switch To Logs Verify Log ${HARBOR_ADMIN} project${d}/${image}:${tag2} tag delete - Verify Log In File ${HARBOR_ADMIN} project${d}/${image}:${tag2} artifact create + Retry Action Keyword Verify Log In Syslog Service ${HARBOR_ADMIN} project${d}/${image}:${tag2} tag delete 0 Close Browser Test Case - Export CVE