From e020b90bf05a3a2b277cff989c0568a74b51b084 Mon Sep 17 00:00:00 2001 From: danfengliu Date: Thu, 4 Jun 2020 18:17:26 +0800 Subject: [PATCH] add oras cli py-test 1. Add oras cli py-test; 2. Add env for notary url, allow to input different notary port instead of solid 4443; 3. Add retry for keyword Cannot Pull Image and make it longer during retry. Signed-off-by: danfengliu --- .github/workflows/CI.yml | 4 + tests/apitests/python/library/oras.py | 48 +++++++++++ tests/apitests/python/library/sign.py | 3 +- tests/apitests/python/sign_image.sh | 3 +- tests/apitests/python/test_oras_cli.py | 84 +++++++++++++++++++ .../test_push_image_with_special_name.py | 2 +- tests/apitests/python/testutils.py | 1 + tests/resources/Docker-Util.robot | 2 +- tests/resources/Util.robot | 1 + tests/robot-cases/Group0-BAT/API_DB.robot | 3 + tests/robot-cases/Group1-Nightly/Notary.robot | 2 +- 11 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 tests/apitests/python/library/oras.py create mode 100644 tests/apitests/python/test_oras_cli.py diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 244c0b99b..6f16b6257 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -143,6 +143,10 @@ jobs: go list make build sudo mv bin/cnab-to-oci /usr/local/bin + curl -LO https://github.com/deislabs/oras/releases/download/v0.8.1/oras_0.8.1_linux_amd64.tar.gz + mkdir -p oras-install/ + tar -zxf oras_0.8.1_*.tar.gz -C oras-install/ + sudo mv oras-install/oras /usr/local/bin/ - name: install run: | cd src/github.com/goharbor/harbor diff --git a/tests/apitests/python/library/oras.py b/tests/apitests/python/library/oras.py new file mode 100644 index 000000000..2e9751196 --- /dev/null +++ b/tests/apitests/python/library/oras.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +import os +import base +from datetime import datetime + +oras_cmd = "oras" +file_artifact = "artifact.txt" +file_readme = "readme.md" +file_config = "config.json" + +def oras_push(harbor_server, user, password, project, repo, tag): + oras_login(harbor_server, user, password) + fo = open(file_artifact, "w") + fo.write( "hello artifact" ) + fo.close() + md5_artifact = base.run_command( ["md5sum", file_artifact] ) + fo = open(file_readme, "w") + fo.write( r"Docs on this artifact" ) + fo.close() + md5_readme = base.run_command( [ "md5sum", file_readme] ) + fo = open(file_config, "w") + fo.write( "{\"doc\":\"readme.md\"}" ) + fo.close() + ret = base.run_command( [oras_cmd, "push", harbor_server + "/" + project + "/" + repo+":"+ tag, + "--manifest-config", "config.json:application/vnd.acme.rocket.config.v1+json", \ + file_artifact+":application/vnd.acme.rocket.layer.v1+txt", \ + file_readme +":application/vnd.acme.rocket.docs.layer.v1+json"] ) + return md5_artifact.split(' ')[0], md5_readme.split(' ')[0] + +def oras_login(harbor_server, user, password): + ret = base.run_command([oras_cmd, "login", "-u", user, "-p", password, harbor_server]) + +def oras_pull(harbor_server, user, password, project, repo, tag): + try: + cwd = os.getcwd() + cwd= cwd + r"/tmp" + datetime.now().strftime(r'%m%s') + if os.path.exists(cwd): + os.rmdir(cwd) + os.makedirs(cwd) + os.chdir(cwd) + print "Tmp dir:", cwd + except Exception as e: + raise Exception('Error: Exited with error {}',format(e)) + ret = base.run_command([oras_cmd, "pull", harbor_server + "/" + project + "/" + repo+":"+ tag, "-a"]) + assert os.path.exists(file_artifact) + assert os.path.exists(file_readme) + return base.run_command( ["md5sum", file_artifact] ).split(' ')[0], base.run_command( [ "md5sum", file_readme] ).split(' ')[0] diff --git a/tests/apitests/python/library/sign.py b/tests/apitests/python/library/sign.py index cc449c537..856b5bc58 100644 --- a/tests/apitests/python/library/sign.py +++ b/tests/apitests/python/library/sign.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- import subprocess +from testutils import notary_url def sign_image(registry_ip, project_name, image, tag): try: - ret = subprocess.check_output(["./tests/apitests/python/sign_image.sh", registry_ip, project_name, image, tag], shell=False) + ret = subprocess.check_output(["./tests/apitests/python/sign_image.sh", registry_ip, project_name, image, tag, notary_url], shell=False) print "sign_image return: ", ret except subprocess.CalledProcessError, exc: raise Exception("Failed to sign image error is {} {}.".format(exc.returncode, exc.output)) diff --git a/tests/apitests/python/sign_image.sh b/tests/apitests/python/sign_image.sh index 00635cdab..6ac3325ca 100755 --- a/tests/apitests/python/sign_image.sh +++ b/tests/apitests/python/sign_image.sh @@ -1,11 +1,12 @@ #!/bin/sh IP=$1 +NOTARY_URL=$5 PASSHRASE='Harbor12345' echo $IP export DOCKER_CONTENT_TRUST=1 -export DOCKER_CONTENT_TRUST_SERVER=https://$IP:4443 +export DOCKER_CONTENT_TRUST_SERVER=$NOTARY_URL export NOTARY_ROOT_PASSPHRASE=$PASSHRASE export NOTARY_TARGETS_PASSPHRASE=$PASSHRASE diff --git a/tests/apitests/python/test_oras_cli.py b/tests/apitests/python/test_oras_cli.py new file mode 100644 index 000000000..790aedcb8 --- /dev/null +++ b/tests/apitests/python/test_oras_cli.py @@ -0,0 +1,84 @@ +from __future__ import absolute_import +import unittest +import urllib + +import library.oras +from library.sign import sign_image +from testutils import ADMIN_CLIENT +from testutils import harbor_server +from testutils import TEARDOWN +from library.user import User +from library.project import Project +from library.repository import Repository +from library.artifact import Artifact + + +class TestProjects(unittest.TestCase): + @classmethod + def setUp(self): + self.project = Project() + self.user = User() + self.artifact = Artifact() + self.repo = Repository() + self.repo_name = "hello-artifact" + self.tag = "test_v2" + + @classmethod + def tearDown(self): + print "Case completed" + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def test_ClearData(self): + #1. Delete user(UA); + self.user.delete_user(TestProjects.user_sign_image_id, **ADMIN_CLIENT) + + def testOrasCli(self): + """ + Test case: + Push Artifact With ORAS CLI + Test step and expected result: + 1. Create user-001 + 2. Create a new private project(PA) by user(UA); + 3. ORAS CLI push artifacts; + 4. Get repository from Harbor successfully, and verfiy repository name is repo pushed by ORAS CLI;; + 5. Get and verify artifacts by tag; + 6. ORAS CLI pull artifacts index by tag; + 7. Verfiy MD5 between artifacts pushed by ORAS and artifacts pulled by ORAS; + Tear down: + NA + """ + url = ADMIN_CLIENT["endpoint"] + user_001_password = "Aa123456" + + #1. Create user-001 + TestProjects.user_sign_image_id, user_name = self.user.create_user(user_password = user_001_password, **ADMIN_CLIENT) + + TestProjects.USER_CLIENT=dict(with_signature = True, endpoint = url, username = user_name, password = user_001_password) + + #2. Create a new private project(PA) by user(UA); + TestProjects.project_id, TestProjects.project_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CLIENT) + + #3. ORAS CLI push artifacts; + md5_list_push = library.oras.oras_push(harbor_server, user_name, user_001_password, TestProjects.project_name, self.repo_name, self.tag) + print "md5_list_push:",md5_list_push + + #4. Get repository from Harbor successfully, and verfiy repository name is repo pushed by ORAS CLI;; + repo_data = self.repo.get_repository(TestProjects.project_name, self.repo_name, **TestProjects.USER_CLIENT) + print "repo_data:", repo_data + self.assertEqual(repo_data.name, TestProjects.project_name + "/" + self.repo_name) + + #5. Get and verify artifacts by tag; + artifact = self.artifact.get_reference_info(TestProjects.project_name, self.repo_name, self.tag, **TestProjects.USER_CLIENT) + print "artifact:", artifact + self.assertEqual(artifact[0].tags[0].name, self.tag) + + #6. ORAS CLI pull artifacts index by tag; + md5_list_pull = library.oras.oras_pull(harbor_server, user_name, user_001_password, TestProjects.project_name, self.repo_name, self.tag) + print "md5_list_pull:",md5_list_pull + + #7. Verfiy MD5 between artifacts pushed by ORAS and artifacts pulled by ORAS; + if set(md5_list_push) != set(md5_list_pull): + raise Exception(r"MD5 check failed with {} and {}.".format(str(md5_list_push), str(md5_list_pull))) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/apitests/python/test_push_image_with_special_name.py b/tests/apitests/python/test_push_image_with_special_name.py index 3d5955c9b..7077ed161 100644 --- a/tests/apitests/python/test_push_image_with_special_name.py +++ b/tests/apitests/python/test_push_image_with_special_name.py @@ -36,7 +36,7 @@ class TestProjects(unittest.TestCase): #3. Delete user(UA); self.user.delete_user(TestProjects.user_sign_image_id, **ADMIN_CLIENT) - def testSignImage(self): + def testPushImageWithSpecialName(self): """ Test case: Push Image With Special Name diff --git a/tests/apitests/python/testutils.py b/tests/apitests/python/testutils.py index 14fb16bd7..d991906c3 100644 --- a/tests/apitests/python/testutils.py +++ b/tests/apitests/python/testutils.py @@ -17,6 +17,7 @@ ADMIN_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ ":// CHART_API_CLIENT=dict(endpoint = os.environ.get("HARBOR_HOST_SCHEMA", "https")+ "://"+harbor_server+"/api", username = admin_user, password = admin_pwd) USER_ROLE=dict(admin=0,normal=1) TEARDOWN = os.environ.get('TEARDOWN', 'true').lower() in ('true', 'yes') +notary_url = os.environ.get('NOTARY_URL', 'https://'+harbor_server+':4443') def GetProductApi(username, password, harbor_server= os.environ["HARBOR_HOST"]): diff --git a/tests/resources/Docker-Util.robot b/tests/resources/Docker-Util.robot index a0ba1781a..5cc63e9b1 100644 --- a/tests/resources/Docker-Util.robot +++ b/tests/resources/Docker-Util.robot @@ -67,7 +67,7 @@ Cannot Pull Image [Arguments] ${ip} ${user} ${pwd} ${project} ${image} ${tag}=${null} ${err_msg}=${null} ${image_with_tag}= Set Variable If '${tag}'=='${null}' ${image} ${image}:${tag} Wait Unitl Command Success docker login -u ${user} -p ${pwd} ${ip} - :FOR ${idx} IN RANGE 0 4 + :FOR ${idx} IN RANGE 0 30 \ ${out} Run Keyword And Ignore Error Command Should be Failed docker pull ${ip}/${project}/${image_with_tag} \ Exit For Loop If '${out[0]}'=='PASS' \ Sleep 3 diff --git a/tests/resources/Util.robot b/tests/resources/Util.robot index 493379a77..b678303b1 100644 --- a/tests/resources/Util.robot +++ b/tests/resources/Util.robot @@ -224,6 +224,7 @@ Command Should be Failed [Arguments] ${cmd} ${rc} ${output}= Run And Return Rc And Output ${cmd} Should Not Be Equal As Strings '${rc}' '0' + Log ${output} [Return] ${output} Retry Keyword N Times When Error diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index e0f2548e7..425487d80 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -98,3 +98,6 @@ Test Case - Registry API Test Case - Push Image With Special Name [Tags] special_repo_name Harbor API Test ./tests/apitests/python/test_push_image_with_special_name.py +Test Case - Push Artifact With ORAS CLI + [Tags] oras + Harbor API Test ./tests/apitests/python/test_oras_cli.py diff --git a/tests/robot-cases/Group1-Nightly/Notary.robot b/tests/robot-cases/Group1-Nightly/Notary.robot index 3de07e6e5..2434d3abc 100644 --- a/tests/robot-cases/Group1-Nightly/Notary.robot +++ b/tests/robot-cases/Group1-Nightly/Notary.robot @@ -37,7 +37,7 @@ Test Case - Project Level Policy Content Trust # Verify # Unsigned image can not be pulled Content Trust Should Be Selected - Cannot Pull Unsigned Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest + Cannot Pull Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest err_msg=The image is not signed in Notary # Signed image can be pulled Body Of Admin Push Signed Image image=redis project=project${d} Pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} redis tag=latest