Merge pull request #12138 from danfengliu/add-oras-py-test

add oras cli py-test

.github/workflows/CI.yml

@@ -143,6 +143,10 @@ jobs:
           go list
           make build
           sudo mv bin/cnab-to-oci /usr/local/bin
+          curl -LO https://github.com/deislabs/oras/releases/download/v0.8.1/oras_0.8.1_linux_amd64.tar.gz
+          mkdir -p oras-install/
+          tar -zxf oras_0.8.1_*.tar.gz -C oras-install/
+          sudo mv oras-install/oras /usr/local/bin/
       - name: install
         run: |
           cd src/github.com/goharbor/harbor

tests/apitests/python/library/oras.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+import os
+import base
+from datetime import datetime
+
+oras_cmd = "oras"
+file_artifact = "artifact.txt"
+file_readme = "readme.md"
+file_config = "config.json"
+
+def oras_push(harbor_server, user, password, project, repo, tag):
+    oras_login(harbor_server, user, password)
+    fo = open(file_artifact, "w")
+    fo.write( "hello artifact" )
+    fo.close()
+    md5_artifact = base.run_command( ["md5sum", file_artifact] )
+    fo = open(file_readme, "w")
+    fo.write( r"Docs on this artifact" )
+    fo.close()
+    md5_readme = base.run_command( [ "md5sum", file_readme] )
+    fo = open(file_config, "w")
+    fo.write( "{\"doc\":\"readme.md\"}" )
+    fo.close()
+    ret = base.run_command( [oras_cmd, "push", harbor_server + "/" + project + "/" + repo+":"+ tag,
+                             "--manifest-config", "config.json:application/vnd.acme.rocket.config.v1+json", \
+                             file_artifact+":application/vnd.acme.rocket.layer.v1+txt", \
+                             file_readme +":application/vnd.acme.rocket.docs.layer.v1+json"] )
+    return md5_artifact.split(' ')[0], md5_readme.split(' ')[0]
+
+def oras_login(harbor_server, user, password):
+     ret = base.run_command([oras_cmd, "login", "-u", user, "-p", password, harbor_server])
+
+def oras_pull(harbor_server, user, password, project, repo, tag):
+    try:
+        cwd = os.getcwd()
+        cwd= cwd + r"/tmp" + datetime.now().strftime(r'%m%s')
+        if os.path.exists(cwd):
+          os.rmdir(cwd)
+        os.makedirs(cwd)
+        os.chdir(cwd)
+        print "Tmp dir:", cwd
+    except Exception as e:
+        raise Exception('Error: Exited with error {}',format(e))
+    ret = base.run_command([oras_cmd, "pull", harbor_server + "/" + project + "/" + repo+":"+ tag, "-a"])
+    assert os.path.exists(file_artifact)
+    assert os.path.exists(file_readme)
+    return base.run_command( ["md5sum", file_artifact] ).split(' ')[0], base.run_command( [ "md5sum", file_readme] ).split(' ')[0]

tests/apitests/python/library/sign.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 import subprocess
+from testutils import notary_url
 
 def sign_image(registry_ip, project_name, image, tag):
     try:
-        ret = subprocess.check_output(["./tests/apitests/python/sign_image.sh", registry_ip, project_name, image, tag], shell=False)
+        ret = subprocess.check_output(["./tests/apitests/python/sign_image.sh", registry_ip, project_name, image, tag, notary_url], shell=False)
         print "sign_image return: ", ret
     except subprocess.CalledProcessError, exc:
         raise Exception("Failed to sign image error is {} {}.".format(exc.returncode, exc.output))

tests/apitests/python/sign_image.sh

@@ -1,11 +1,12 @@
 #!/bin/sh
 IP=$1
+NOTARY_URL=$5
 PASSHRASE='Harbor12345'
 
 echo $IP
 
 export DOCKER_CONTENT_TRUST=1
-export DOCKER_CONTENT_TRUST_SERVER=https://$IP:4443
+export DOCKER_CONTENT_TRUST_SERVER=$NOTARY_URL
 
 export NOTARY_ROOT_PASSPHRASE=$PASSHRASE
 export NOTARY_TARGETS_PASSPHRASE=$PASSHRASE

tests/apitests/python/test_oras_cli.py

@@ -0,0 +1,84 @@
+from __future__ import absolute_import
+import unittest
+import urllib
+
+import library.oras
+from library.sign import sign_image
+from testutils import ADMIN_CLIENT
+from testutils import harbor_server
+from testutils import TEARDOWN
+from library.user import User
+from library.project import Project
+from library.repository import Repository
+from library.artifact import Artifact
+
+
+class TestProjects(unittest.TestCase):
+    @classmethod
+    def setUp(self):
+        self.project = Project()
+        self.user = User()
+        self.artifact = Artifact()
+        self.repo = Repository()
+        self.repo_name = "hello-artifact"
+        self.tag = "test_v2"
+
+    @classmethod
+    def tearDown(self):
+        print "Case completed"
+
+    @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
+    def test_ClearData(self):
+        #1. Delete user(UA);
+        self.user.delete_user(TestProjects.user_sign_image_id, **ADMIN_CLIENT)
+
+    def testOrasCli(self):
+        """
+        Test case:
+            Push Artifact With ORAS CLI
+        Test step and expected result:
+            1. Create user-001
+            2. Create a new private project(PA) by user(UA);
+            3. ORAS CLI push artifacts;
+            4. Get repository from Harbor successfully, and verfiy repository name is repo pushed by ORAS CLI;;
+            5. Get and verify artifacts by tag;
+            6. ORAS CLI pull artifacts index by tag;
+            7. Verfiy MD5 between artifacts pushed by ORAS and artifacts pulled by ORAS;
+        Tear down:
+            NA
+        """
+        url = ADMIN_CLIENT["endpoint"]
+        user_001_password = "Aa123456"
+
+        #1. Create user-001
+        TestProjects.user_sign_image_id, user_name = self.user.create_user(user_password = user_001_password, **ADMIN_CLIENT)
+
+        TestProjects.USER_CLIENT=dict(with_signature = True, endpoint = url, username = user_name, password = user_001_password)
+
+        #2. Create a new private project(PA) by user(UA);
+        TestProjects.project_id, TestProjects.project_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CLIENT)
+
+        #3. ORAS CLI push artifacts;
+        md5_list_push = library.oras.oras_push(harbor_server, user_name, user_001_password, TestProjects.project_name, self.repo_name, self.tag)
+        print "md5_list_push:",md5_list_push
+
+        #4. Get repository from Harbor successfully, and verfiy repository name is repo pushed by ORAS CLI;;
+        repo_data = self.repo.get_repository(TestProjects.project_name, self.repo_name, **TestProjects.USER_CLIENT)
+        print "repo_data:", repo_data
+        self.assertEqual(repo_data.name, TestProjects.project_name + "/" + self.repo_name)
+
+        #5. Get and verify artifacts by tag;
+        artifact = self.artifact.get_reference_info(TestProjects.project_name, self.repo_name, self.tag, **TestProjects.USER_CLIENT)
+        print "artifact:", artifact
+        self.assertEqual(artifact[0].tags[0].name, self.tag)
+
+        #6. ORAS CLI pull artifacts index by tag;
+        md5_list_pull = library.oras.oras_pull(harbor_server, user_name, user_001_password, TestProjects.project_name, self.repo_name, self.tag)
+        print "md5_list_pull:",md5_list_pull
+
+        #7. Verfiy MD5 between artifacts pushed by ORAS and artifacts pulled by ORAS;
+        if set(md5_list_push) != set(md5_list_pull):
+            raise Exception(r"MD5 check failed with {} and {}.".format(str(md5_list_push), str(md5_list_pull)))
+
+if __name__ == '__main__':
+    unittest.main()

tests/apitests/python/test_push_image_with_special_name.py

@@ -36,7 +36,7 @@ class TestProjects(unittest.TestCase):
         #3. Delete user(UA);
         self.user.delete_user(TestProjects.user_sign_image_id, **ADMIN_CLIENT)
 
-    def testSignImage(self):
+    def testPushImageWithSpecialName(self):
         """
         Test case:
             Push Image With Special Name

tests/apitests/python/testutils.py

@@ -17,6 +17,7 @@ ADMIN_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://
 CHART_API_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://"+harbor_server+"/api", username = admin_user, password =  admin_pwd)
 USER_ROLE=dict(admin=0,normal=1)
 TEARDOWN = os.environ.get('TEARDOWN', 'true').lower() in ('true', 'yes')
+notary_url = os.environ.get('NOTARY_URL', 'https://'+harbor_server+':4443')
 
 def GetProductApi(username, password, harbor_server= os.environ["HARBOR_HOST"]):
 

tests/resources/Docker-Util.robot

@@ -67,7 +67,7 @@ Cannot Pull Image
     [Arguments]  ${ip}  ${user}  ${pwd}  ${project}  ${image}  ${tag}=${null}  ${err_msg}=${null}
     ${image_with_tag}=  Set Variable If  '${tag}'=='${null}'  ${image}  ${image}:${tag}
     Wait Unitl Command Success  docker login -u ${user} -p ${pwd} ${ip}
-    :FOR  ${idx}  IN RANGE  0  4
+    :FOR  ${idx}  IN RANGE  0  30
     \   ${out}  Run Keyword And Ignore Error  Command Should be Failed  docker pull ${ip}/${project}/${image_with_tag}
     \   Exit For Loop If  '${out[0]}'=='PASS'
     \   Sleep  3

tests/resources/Util.robot

@@ -224,6 +224,7 @@ Command Should be Failed
     [Arguments]  ${cmd}
     ${rc}  ${output}=  Run And Return Rc And Output  ${cmd}
     Should Not Be Equal As Strings  '${rc}'  '0'
+    Log  ${output}
     [Return]  ${output}
 
 Retry Keyword N Times When Error

tests/robot-cases/Group0-BAT/API_DB.robot

@@ -98,3 +98,6 @@ Test Case - Registry API
 Test Case - Push Image With Special Name
     [Tags]  special_repo_name
     Harbor API Test  ./tests/apitests/python/test_push_image_with_special_name.py
+Test Case - Push Artifact With ORAS CLI
+    [Tags]  oras
+    Harbor API Test  ./tests/apitests/python/test_oras_cli.py

tests/robot-cases/Group1-Nightly/Notary.robot

@@ -37,7 +37,7 @@ Test Case - Project Level Policy Content Trust
     # Verify
     # Unsigned image can not be pulled
     Content Trust Should Be Selected
-    Cannot Pull Unsigned Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  hello-world:latest
+    Cannot Pull Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  hello-world:latest  err_msg=The image is not signed in Notary
     # Signed image can be pulled
     Body Of Admin Push Signed Image  image=redis  project=project${d}
     Pull image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  redis  tag=latest