add Test Case for SBOM feature (#20797)

Signed-off-by: Shengwen Yu <yshengwen@vmware.com>
This commit is contained in:
Shengwen YU 2024-08-01 16:05:39 +08:00 committed by GitHub
parent e0b94aa7d7
commit 295e075568
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 359 additions and 9 deletions

View File

@ -30,6 +30,8 @@ class Artifact(base.Base, object):
if "with_scan_overview" in kwargs:
params["with_scan_overview"] = kwargs["with_scan_overview"]
params["x_accept_vulnerabilities"] = ",".join(report_mime_types)
if "with_sbom_overview" in kwargs:
params["with_sbom_overview"] = kwargs["with_sbom_overview"]
if "with_immutable_status" in kwargs:
params["with_immutable_status"] = kwargs["with_immutable_status"]
if "with_accessory" in kwargs:
@ -140,6 +142,29 @@ class Artifact(base.Base, object):
return
raise Exception("Scan image result is {}, not as expected {}.".format(scan_status, expected_scan_status))
def check_image_sbom_generation_result(self, project_name, repo_name, reference, expected_scan_status = "Success", **kwargs):
timeout_count = 30
scan_status=""
while True:
time.sleep(5)
timeout_count = timeout_count - 1
if (timeout_count == 0):
break
artifact = self.get_reference_info(project_name, repo_name, reference, **kwargs)
if expected_scan_status in ["Not Scanned", "No SBOM Overview"]:
if artifact.sbom_overview is None:
if (timeout_count > 24):
continue
print("artifact SBOM is not generated.")
return
else:
raise Exception("Artifact SBOM should not be generated {}.".format(artifact.sbom_overview))
scan_status = artifact.sbom_overview.scan_status
if scan_status == expected_scan_status:
return
raise Exception("Generate image SBOM result is {}, not as expected {}.".format(scan_status, expected_scan_status))
def check_reference_exist(self, project_name, repo_name, reference, ignore_not_found = False, **kwargs):
artifact = self.get_reference_info( project_name, repo_name, reference, ignore_not_found=ignore_not_found, **kwargs)
return {

View File

@ -21,3 +21,18 @@ class Scan(base.Base, object):
base._assert_status_code(expect_status_code, status_code)
return data
def sbom_generation_of_artifact(self, project_name, repo_name, reference, expect_status_code = 202, expect_response_body = None, **kwargs):
try:
req_param = dict(scan_type = {"scan_type":"sbom"})
data, status_code, _ = self._get_client(**kwargs).scan_artifact_with_http_info(project_name, repo_name, reference, **req_param)
except ApiException as e:
base._assert_status_code(expect_status_code, e.status)
if expect_response_body is not None:
base._assert_status_body(expect_response_body, e.body)
return
base._assert_status_code(expect_status_code, status_code)
return data

View File

@ -22,4 +22,19 @@ class StopScan(base.Base, object):
base._assert_status_code(expect_status_code, status_code)
return data
def stop_sbom_generation_of_artifact(self, project_name, repo_name, reference, expect_status_code = 202, expect_response_body = None, **kwargs):
try:
scanType = v2_swagger_client.ScanType()
scanType.scan_type = "sbom"
data, status_code, _ = self._get_client(**kwargs).stop_scan_artifact_with_http_info(project_name, repo_name, reference, scanType)
except ApiException as e:
base._assert_status_code(expect_status_code, e.status)
if expect_response_body is not None:
base._assert_status_body(expect_response_body, e.body)
return
base._assert_status_code(expect_status_code, status_code)
return data

View File

@ -73,7 +73,7 @@ list_metadata = Permission("{}/projects/{}/metadatas".format(harbor_base_url, pr
read_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "GET", 200, metadata_payload)
metadata_payload_for_update = { "auto_scan": "false" }
update_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "PUT", 200, metadata_payload_for_update)
delete_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "DELETE", 200, metadata_payload)
delete_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "DELETE", 200, metadata_payload_for_update)
# 4. Resource: repository actions: ['read', 'list', 'update', 'delete', 'pull', 'push']
# note: pull and push are for docker cli, no API needs them
@ -89,12 +89,17 @@ copy_artifact = Permission("{}/projects/{}/repositories/target_repo/artifacts?fr
delete_artifact = Permission("{}/projects/{}/repositories/target_repo/artifacts/{}".format(harbor_base_url, project_name, source_artifact_tag), "DELETE", 200)
# 6. Resource scan actions: ['read', 'create', 'stop']
stop_scan_payload = {
vulnerability_scan_payload = {
"scan_type": "vulnerability"
}
create_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202)
stop_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, stop_scan_payload)
create_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, vulnerability_scan_payload)
stop_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, vulnerability_scan_payload)
read_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/83be44fd-1234-5678-b49f-4b6d6e8f5730/log".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "get", 404)
sbom_gen_payload = {
"scan_type": "sbom"
}
create_sbom_generation = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, sbom_gen_payload)
stop_sbom_generation = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, sbom_gen_payload)
# 7. Resource tag actions: ['list', 'create', 'delete']
tag_payload = { "name": "test-{}".format(int(random.randint(1000, 9999))) }
@ -240,7 +245,7 @@ resource_permissions = {
"metadata": [create_metadata, list_metadata, read_metadata, update_metadata, delete_metadata],
"repository": [list_repo, read_repo, update_repo, delete_repo],
"artifact": [list_artifact, read_artifact, copy_artifact, delete_artifact],
"scan": [create_scan, stop_scan, read_scan],
"scan": [create_scan, stop_scan, read_scan, create_sbom_generation, stop_sbom_generation],
"tag": [create_tag, list_tag, delete_tag],
"accessory": [list_accessory],
"artifact-addition": [read_artifact_addition_vul, read_artifact_addition_dependencies],

View File

@ -0,0 +1,87 @@
from __future__ import absolute_import
import unittest
import sys
from testutils import harbor_server, suppress_urllib3_warning
from testutils import TEARDOWN
from testutils import ADMIN_CLIENT, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME
from library.project import Project
from library.user import User
from library.repository import Repository
from library.repository import push_self_build_image_to_project
from library.artifact import Artifact
from library.scan import Scan
class TestSBOMGeneration(unittest.TestCase):
@suppress_urllib3_warning
def setUp(self):
self.project= Project()
self.user= User()
self.artifact = Artifact()
self.repo = Repository()
self.scan = Scan()
self.url = ADMIN_CLIENT["endpoint"]
self.user_password = "Aa123456"
self.project_id, self.project_name, self.user_id, self.user_name, self.repo_name1 = [None] * 5
self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT)
self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password, with_sbom_overview = True)
#2. Create a new private project(PA) by user(UA);
self.project_id, self.project_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT)
#3. Add user(UA) as a member of project(PA) with project-admin role;
self.project.add_project_members(self.project_id, user_id = self.user_id, **ADMIN_CLIENT)
@unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
def do_tearDown(self):
#1. Delete repository(RA) by user(UA);
self.repo.delete_repository(self.project_name, self.repo_name1.split('/')[1], **self.USER_CLIENT)
#2. Delete project(PA);
self.project.delete_project(self.project_id, **self.USER_CLIENT)
#3. Delete user(UA);
self.user.delete_user(self.user_id, **ADMIN_CLIENT)
def testGenerateSBOMOfImageArtifact(self):
"""
Test case:
Generate an SBOM of An Image Artifact
Test step and expected result:
1. Create a new user(UA);
2. Create a new private project(PA) by user(UA);
3. Add user(UA) as a member of project(PA) with project-admin role;
4. Get private project of user(UA), user(UA) can see only one private project which is project(PA);
5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA);
6. Send sbom generation of an image command and get tag(TA) information to check sbom generation result, it should be finished;
Tear down:
1. Delete repository(RA) by user(UA);
2. Delete project(PA);
3. Delete user(UA);
"""
#4. Get private project of user(UA), user(UA) can see only one private project which is project(PA);
self.project.projects_should_exist(dict(public=False), expected_count = 1,
expected_project_id = self.project_id, **self.USER_CLIENT)
#Note: Please make sure that this Image has never been pulled before by any other cases,
# so it is a not-scanned image right after repository creation.
image = "docker"
src_tag = "1.13"
#5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA);
self.repo_name1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag)
#6. Send sbom generation of an image command and get tag(TA) information to check sbom generation result, it should be finished;
self.scan.sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT)
self.artifact.check_image_sbom_generation_result(self.project_name, image, tag, **self.USER_CLIENT)
self.do_tearDown()
if __name__ == '__main__':
suite = unittest.TestSuite(unittest.makeSuite(TestSBOMGeneration))
result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite)
if not result.wasSuccessful():
raise Exception(r"SBOM generation test failed: {}".format(result))

View File

@ -0,0 +1,91 @@
from __future__ import absolute_import
import unittest
import sys
from testutils import harbor_server, suppress_urllib3_warning
from testutils import TEARDOWN
from testutils import ADMIN_CLIENT, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME
from library.project import Project
from library.user import User
from library.repository import Repository
from library.repository import push_self_build_image_to_project
from library.artifact import Artifact
from library.scan import Scan
from library.scan_stop import StopScan
class TestStopSBOMGeneration(unittest.TestCase):
@suppress_urllib3_warning
def setUp(self):
self.project= Project()
self.user= User()
self.artifact = Artifact()
self.repo = Repository()
self.scan = Scan()
self.stop_scan = StopScan()
self.url = ADMIN_CLIENT["endpoint"]
self.user_password = "Aa123456"
self.project_id, self.project_name, self.user_id, self.user_name, self.repo_name1 = [None] * 5
self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT)
self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password, with_sbom_overview = True)
#2. Create a new private project(PA) by user(UA);
self.project_id, self.project_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT)
#3. Add user(UA) as a member of project(PA) with project-admin role;
self.project.add_project_members(self.project_id, user_id = self.user_id, **ADMIN_CLIENT)
@unittest.skipIf(TEARDOWN == False, "Test data won't be erased.")
def do_tearDown(self):
#1. Delete repository(RA) by user(UA);
self.repo.delete_repository(self.project_name, self.repo_name1.split('/')[1], **self.USER_CLIENT)
#2. Delete project(PA);
self.project.delete_project(self.project_id, **self.USER_CLIENT)
#3. Delete user(UA);
self.user.delete_user(self.user_id, **ADMIN_CLIENT)
def testStopSBOMGenerationOfImageArtifact(self):
"""
Test case:
Stop SBOM Generation Of An Image Artifact
Test step and expected result:
1. Create a new user(UA);
2. Create a new private project(PA) by user(UA);
3. Add user(UA) as a member of project(PA) with project-admin role;
4. Get private project of user(UA), user(UA) can see only one private project which is project(PA);
5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA);
6. Send SBOM generation of an image command;
7. Send stop SBOM generation of an image command.
Tear down:
1. Delete repository(RA) by user(UA);
2. Delete project(PA);
3. Delete user(UA);
"""
#4. Get private project of user(UA), user(UA) can see only one private project which is project(PA);
self.project.projects_should_exist(dict(public=False), expected_count = 1,
expected_project_id = self.project_id, **self.USER_CLIENT)
#Note: Please make sure that this Image has never been pulled before by any other cases,
# so it is a not-scanned image right after repository creation.
image = "docker"
src_tag = "1.13"
#5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA);
self.repo_name1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag)
#6. Send SBOM generation of an image command;
self.scan.sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT)
#7. Send stop SBOM generation of an image command.
self.stop_scan.stop_sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT)
self.do_tearDown()
if __name__ == '__main__':
suite = unittest.TestSuite(unittest.makeSuite(TestStopSBOMGeneration))
result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite)
if not result.wasSuccessful():
raise Exception(r"Stop SBOM generation test failed: {}".format(result))

View File

@ -41,9 +41,20 @@ Stop Scan Artifact
Retry Element Click ${stop_scan_artifact_btn}
Check Scan Artifact Job Status Is Stopped
Wait Until Element Is Visible ${stopped_label}
${job_status}= Get Text ${stopped_label}
Should Be Equal As Strings '${job_status}' 'Scan stopped'
Wait Until Element Is Visible ${scan_stopped_label}
Generate Artifact SBOM
[Arguments] ${project} ${repo} ${label_xpath}=//clr-dg-row//label[contains(@class,'clr-control-label')][1]
Go Into Repo ${project} ${repo}
Retry Element Click ${label_xpath}
Retry Element Click ${gen_artifact_sbom_btn}
Stop Gen Artifact SBOM
Retry Element Click ${artifact_action_xpath}
Retry Element Click ${stop_gen_artifact_sbom_btn}
Check Gen Artifact SBOM Job Status Is Stopped
Wait Until Element Is Visible ${gen_sbom_stopped_label}
Refresh Repositories
Retry Element Click ${refresh_repositories_xpath}

View File

@ -24,5 +24,8 @@ ${build_history_data} //clr-dg-row
${push_image_command_btn} //hbr-push-image-button//button
${scan_artifact_btn} //button[@id='scan-btn']
${stop_scan_artifact_btn} //button[@id='stop-scan']
${stopped_label} //span[@class='label stopped']
${scan_stopped_label} //span[normalize-space()='Scan stopped']
${gen_sbom_stopped_label} //span[normalize-space()='Generation stopped']
${gen_artifact_sbom_btn} //button[@id='generate-sbom-btn']
${stop_gen_artifact_sbom_btn} //button[@id='stop-sbom-btn']
${refresh_repositories_xpath} //hbr-repository-gridview//span[contains(@class,'refresh-btn')]

View File

@ -59,6 +59,41 @@ Enable Scan On Push
Checkbox Should Be Selected //clr-checkbox-wrapper[@id='scan-image-on-push-wrapper']//input
Retry Element Click ${project_config_save_btn}
Generate Repo SBOM
[Arguments] ${tagname} ${status}
Retry Element Click //clr-dg-row[contains(.,'${tagname}')]//label[contains(@class,'clr-control-label')]
Retry Element Click //button[@id='generate-sbom-btn']
Run Keyword If '${status}' == 'Succeed' Wait Until Element Is Visible //a[@title='SBOM details'] 300
Checkout And Review SBOM Details
[Arguments] ${tagname}
Retry Element Click //clr-dg-row[contains(.,'${tagname}')]//a[@title='SBOM details']
# Download SBOM file
Retry Element Click //button[@id='sbom-btn']
${sbom_artifact_short_sha256}= Get Text //span[@class='margin-left-10px']
${sbom_filename_raw}= Get Text //clr-dg-cell[contains(text(),'${sbom_artifact_short_sha256}')]
${sbom_filename}= Replace String ${sbom_filename_raw} : _ count=-1
${sbom_filename}= Replace String ${sbom_filename} / _ count=-1
${sbom_json_path}= Set Variable ${download_directory}/${sbom_filename}.json
Retry File Should Exist ${sbom_json_path}
# Load the downloaded SBOM json file and verify the first N package records
${sbom_json_content}= Load Json From File ${sbom_json_path}
${items}= Get Value From JSON ${sbom_json_content} packages
${items_length}= Get Length ${items}
${first_n_records}= Evaluate min(5, ${items_length})
FOR ${idx} IN RANGE 1 ${first_n_records}
${item}= Get From List ${items} ${idx}
Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.name}']
Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.versionInfo}']
Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.licenseConcluded}']
END
Enable Generating SBOM On Push
Checkbox Should Not Be Selected //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//input
Retry Element Click //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//label[contains(@class,'clr-control-label')]
Checkbox Should Be Selected //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//input
Retry Element Click ${project_config_save_btn}
Vulnerability Not Ready Project Hint
Sleep 2
${element}= Set Variable xpath=//span[contains(@class, 'db-status-warning')]

View File

@ -417,6 +417,49 @@ Stop Scan All
Stop Scan All Artifact
Retry Action Keyword Check Scan All Artifact Job Status Is Stopped
Body Of Generate SBOM of An Image In The Repo
[Arguments] ${image_argument} ${tag_argument}
Init Chrome Driver
${d}= get current date result_format=%m%s
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Project And Go Into Project project${d}
Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_argument}:${tag_argument}
Go Into Repo project${d} ${image_argument}
Generate Repo SBOM ${tag_argument} Succeed
Checkout And Review SBOM Details ${tag_argument}
Close Browser
Body Of Generate Image SBOM On Push
Init Chrome Driver
${d}= get current date result_format=%m%s
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Project And Go Into Project project${d}
Goto Project Config
Enable Generating SBOM On Push
Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} memcached
Go Into Repo project${d} memcached
Checkout And Review SBOM Details latest
Close Browser
Body Of Stop SBOM Manual Generation
Init Chrome Driver
${d}= get current date result_format=%m%s
${repo}= Set Variable goharbor/harbor-e2e-engine
${tag}= Set Variable test-ui
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Project And Go Into Project project${d}
Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${repo} ${tag} ${tag}
# stop generate sbom of an artifact
Retry Action Keyword Stop SBOM Generation project${d} ${repo}
Close Browser
Stop SBOM Generation
[Arguments] ${project_name} ${repo}
Generate Artifact SBOM ${project_name} ${repo}
Stop Gen Artifact SBOM
Retry Action Keyword Check Gen Artifact SBOM Job Status Is Stopped
Prepare Image Package Test Files
[Arguments] ${files_path}
${rc} ${output}= Run And Return Rc And Output bash tests/robot-cases/Group0-Util/prepare_imgpkg_test_files.sh ${files_path}

View File

@ -120,6 +120,14 @@ Test Case - Stop Scan All Images
[Tags] stop_scan_all
Harbor API Test ./tests/apitests/python/test_system_level_stop_scan_all.py
Test Case - Generate SBOM Of An Image
[Tags] generate_sbom
Harbor API Test ./tests/apitests/python/test_sbom_generation_of_image_artifact.py
Test Case - Stop Generating SBOM Of An Image
[Tags] stop_generating_sbom
Harbor API Test ./tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py
Test Case - Registry API
[Tags] reg_api
Harbor API Test ./tests/apitests/python/test_registry_api.py

View File

@ -164,6 +164,18 @@ Test Case - Stop Scan And Stop Scan All
[Tags] stop_scan_job
Body Of Stop Scan And Stop Scan All
Test Case - Verify SBOM Manual Generation
[Tags] sbom_manual_gen
Body Of Generate SBOM of An Image In The Repo alpine 3.10
Test Case - Generate Image SBOM On Push
[Tags] run-once
Body Of Generate Image SBOM On Push
Test Case - Stop SBOM Manual Generation
[Tags] stop_sbom_gen
Body Of Stop SBOM Manual Generation
Test Case - External Scanner CRUD
[Tags] external_scanner_crud need_scanner_endpoint
${SCANNER_ENDPOINT_VALUE}= Get Variable Value ${SCANNER_ENDPOINT} ${EMPTY}