From 6e93089845269b302c3f93413e09ebf937fd0cb2 Mon Sep 17 00:00:00 2001 From: Yang Jiao Date: Mon, 17 Jan 2022 07:40:49 +0000 Subject: [PATCH] Add cosign UI test case Signed-off-by: Yang Jiao --- tests/resources/Cosign_Util.robot | 27 +++++++++++++++++++ .../Project-Artifact-Elements.robot | 1 + .../Harbor-Pages/Project-Config.robot | 11 ++++++-- tests/resources/Harbor-Pages/Project.robot | 18 +++++++++++++ tests/resources/Util.robot | 1 + tests/robot-cases/Group1-Nightly/Common.robot | 27 +++++++++++++++++++ tests/robot-cases/Group1-Nightly/Notary.robot | 4 +-- tests/test-engine-image/Dockerfile.common | 3 +++ tests/test-engine-image/Dockerfile.ui_test | 5 ++-- 9 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 tests/resources/Cosign_Util.robot diff --git a/tests/resources/Cosign_Util.robot b/tests/resources/Cosign_Util.robot new file mode 100644 index 000000000..7f9b1c2b9 --- /dev/null +++ b/tests/resources/Cosign_Util.robot @@ -0,0 +1,27 @@ +# Copyright Project Harbor Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +*** Settings *** +Documentation This resource provides helper functions for docker operations +Library OperatingSystem +Library Process + +*** Keywords *** +Cosign Generate Key Pair + Remove Files cosign.key cosign.pub + Wait Unitl Command Success cosign generate-key-pair + +Cosign Sign + [Arguments] ${artifact} + Wait Unitl Command Success cosign sign --allow-insecure-registry --key cosign.key ${artifact} \ No newline at end of file diff --git a/tests/resources/Harbor-Pages/Project-Artifact-Elements.robot b/tests/resources/Harbor-Pages/Project-Artifact-Elements.robot index 3074a9179..e565def71 100644 --- a/tests/resources/Harbor-Pages/Project-Artifact-Elements.robot +++ b/tests/resources/Harbor-Pages/Project-Artifact-Elements.robot @@ -22,6 +22,7 @@ ${artifact_action_copy_xpath} //clr-dropdown-menu//div[contains(.,'Copy') and @ ${artifact_achieve_icon} //artifact-list-tab//clr-datagrid//clr-dg-row[contains(.,'sha256')]//clr-dg-cell[1]//clr-tooltip//a ${artifact_rows} //artifact-list-tab//clr-datagrid//clr-dg-row[contains(.,'sha256')] ${archive_rows} //artifact-list-tab//clr-datagrid//clr-dg-row[contains(.,'sha256')]//clr-dg-cell[1]//clr-tooltip//a +${artifact_list_refresh_btn} //artifact-list-tab//div//span[@class='refresh-btn'] ${artifact_list_spinner} xpath=//clr-datagrid//clr-spinner ${artifact_tag_component} xpath=//artifact-tag diff --git a/tests/resources/Harbor-Pages/Project-Config.robot b/tests/resources/Harbor-Pages/Project-Config.robot index 72e59d3a7..bf892006b 100644 --- a/tests/resources/Harbor-Pages/Project-Config.robot +++ b/tests/resources/Harbor-Pages/Project-Config.robot @@ -17,10 +17,14 @@ Click Project Public Mouse Down //hbr-project-policy-config//input[@name='public'] Mouse Up //hbr-project-policy-config//input[@name='public'] -Click Notary Deployment security +Click Notary Deployment Security Mouse Down //input[@id='content-trust'] Mouse Up //input[@id='content-trust'] +Click Cosign Deployment Security + Mouse Down //input[@id='content-trust-cosign'] + Mouse Up //input[@id='content-trust-cosign'] + Click Prevent Running Mouse Down //hbr-project-policy-config//input[@name='prevent-vulnerability-image'] Mouse Up //hbr-project-policy-config//input[@name='prevent-vulnerability-image'] @@ -46,9 +50,12 @@ Project Should Be Public [Arguments] ${projectName} Retry Wait Until Page Contains Element //clr-dg-row[contains(.,'${projectName}')]//clr-dg-cell[contains(.,'Public')] -Content Trust Should Be Selected +Content Notary Deployment security Be Selected Checkbox Should Be Selected //input[@id='content-trust'] +Content Cosign Deployment security Be Selected + Checkbox Should Be Selected //input[@id='content-trust-cosign'] + Prevent Running Should Be Selected Checkbox Should Be Selected //hbr-project-policy-config//input[@name='prevent-vulnerability-image'] diff --git a/tests/resources/Harbor-Pages/Project.robot b/tests/resources/Harbor-Pages/Project.robot index 6feba666b..ecd372b14 100644 --- a/tests/resources/Harbor-Pages/Project.robot +++ b/tests/resources/Harbor-Pages/Project.robot @@ -376,3 +376,21 @@ Select Storage Quota unit [Arguments] ${unit} Select From List By Value ${project_add_storage_quota_unit_id} ${unit} +Should Not Be Signed By Cosign + [Arguments] ${tag} + Retry Wait Element Visible //clr-dg-row[contains(.,'latest')]//clr-icon[contains(@class,'color-red')] + +Should Be Signed By Cosign + [Arguments] ${tag} + Retry Wait Element Visible //clr-dg-row[contains(.,'${tag}')]// clr-icon[contains(@class,'signed')] + +Delete Accessory + [Arguments] ${tag} + Retry Button Click //clr-dg-row[contains(.,'${tag}')]//button[contains(@class,'datagrid-expandable-caret-button')] + Retry Button Click //clr-dg-row[contains(.,'${tag}')]//button[contains(@class,'datagrid-action-toggle')] + Retry Button Click //div[@id='clr-action-menu1']/button[@class='action-item'] + Retry Button Click //div[contains(@class,'modal-content')]//button[contains(@class,'btn-danger')] + +Should be Accessory deleted + [Arguments] ${tag} + Retry Wait Until Page Not Contains Element //clr-dg-row[contains(.,'${tag}')]//button[contains(@class,'datagrid-expandable-caret-button')] \ No newline at end of file diff --git a/tests/resources/Util.robot b/tests/resources/Util.robot index ed01df5ee..2cd0898e4 100644 --- a/tests/resources/Util.robot +++ b/tests/resources/Util.robot @@ -74,6 +74,7 @@ Resource Cert-Util.robot Resource SeleniumUtil.robot Resource Nightly-Util.robot Resource APITest-Util.robot +Resource Cosign_Util.robot Resource TestCaseBody.robot *** Keywords *** diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index 1866a1f9c..3f0a7ab91 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -863,3 +863,30 @@ Test Case - Carvel Imgpkg Push And Pull To Harbor Wait Unitl Command Success docker logout ${ip} Retry File Should Exist ${out_path}/.imgpkg/bundle.yml Retry File Should Exist ${out_path}/.imgpkg/images.yml + +Test Case - Cosign And Cosign Deployment Security Policy + [Tags] cosign + Init Chrome Driver + ${user}= Set Variable user006 + ${pwd}= Set Variable Test1@34 + ${d}= Get Current Date result_format=%m%s + ${image}= Set Variable hello-world + ${tag}= Set Variable latest + Sign In Harbor ${HARBOR_URL} ${user} ${pwd} + Create An New Project And Go Into Project project${d} + Goto Project Config + Click Cosign Deployment Security + Save Project Config + Content Cosign Deployment security Be Selected + + Push Image With Tag ${ip} ${user} ${pwd} project${d} ${image} ${tag} + Go Into Project project${d} + Retry Double Keywords When Error Go Into Repo project${d}/${image} Should Not Be Signed By Cosign ${tag} + Cannot Pull Image ${ip} ${user} ${pwd} project${d} ${image}:${tag} err_msg=The image is not signed in Cosign. + + Cosign Generate Key Pair + Cosign Sign ${ip}/project${d}/${image}:${tag} + Retry Double Keywords When Error Retry Element Click ${artifact_list_refresh_btn} Should Be Signed By Cosign ${tag} + Pull image ${ip} ${user} ${pwd} project${d} ${image}:${tag} + + Retry Double Keywords When Error Delete Accessory ${tag} Should be Accessory deleted ${tag} \ No newline at end of file diff --git a/tests/robot-cases/Group1-Nightly/Notary.robot b/tests/robot-cases/Group1-Nightly/Notary.robot index 93b3f643d..54ad8ee1a 100644 --- a/tests/robot-cases/Group1-Nightly/Notary.robot +++ b/tests/robot-cases/Group1-Nightly/Notary.robot @@ -31,11 +31,11 @@ Test Case - Project Level Policy Notary Deployment security Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest Go Into Project project${d} Goto Project Config - Click Notary Deployment security + Click Notary Deployment Security Save Project Config # Verify # Unsigned image can not be pulled - Content Trust Should Be Selected + Content Notary Deployment security Be Selected Cannot Pull Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest err_msg=The image is not signed in Notary # Signed image can be pulled Body Of Admin Push Signed Image project${d} redis latest ${HARBOR_ADMIN} ${HARBOR_PASSWORD} diff --git a/tests/test-engine-image/Dockerfile.common b/tests/test-engine-image/Dockerfile.common index 58eefb639..dbf713853 100644 --- a/tests/test-engine-image/Dockerfile.common +++ b/tests/test-engine-image/Dockerfile.common @@ -68,6 +68,9 @@ RUN pwd && mkdir /tool/binary && \ # Install imgpkg IMGPKG_VERSION=0.22.0 && wget https://github.com/vmware-tanzu/carvel-imgpkg/releases/download/v$IMGPKG_VERSION/imgpkg-linux-amd64 && \ mv imgpkg-linux-amd64 /tool/binary/imgpkg && chmod +x /tool/binary/imgpkg && \ + # Install cosign + COSIGN_VERSION=1.4.1 && wget https://github.com/sigstore/cosign/releases/download/v$COSIGN_VERSION/cosign-linux-amd64 && \ + mv cosign-linux-amd64 /tool/binary/cosign && chmod +x /tool/binary/cosign && \ pwd #ubuntu diff --git a/tests/test-engine-image/Dockerfile.ui_test b/tests/test-engine-image/Dockerfile.ui_test index 61598f843..1a5d79363 100644 --- a/tests/test-engine-image/Dockerfile.ui_test +++ b/tests/test-engine-image/Dockerfile.ui_test @@ -3,6 +3,7 @@ FROM ubuntu:18.04 ENV LANG C.UTF-8 ENV HELM_EXPERIMENTAL_OCI=1 +ENV COSIGN_PASSWORD=Harbor12345 RUN apt-get update && apt-get install -y --no-install-recommends wget curl gnupg2 RUN apt-get install libseccomp2 RUN wget --no-check-certificate -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - @@ -35,8 +36,8 @@ RUN pip3 install --upgrade pip pyasn1 google-apitools==0.5.31 gsutil \ requests dbbot robotframework-seleniumlibrary==4.3.0 robotframework-pabot \ robotframework-JSONLibrary hurry.filesize --upgrade && \ apt-get clean all -# Because the old version of chromedriver can’t download files, upgrade it to version 93.0.4577.15, which can download files normally. -RUN wget -N http://chromedriver.storage.googleapis.com/95.0.4638.54/chromedriver_linux64.zip && \ +# Upgrade chromedriver version to 97.0.4692.71 +RUN wget -N http://chromedriver.storage.googleapis.com/97.0.4692.71/chromedriver_linux64.zip && \ unzip chromedriver_linux64.zip && \ chmod +x chromedriver && \ mv -f chromedriver /usr/local/share/chromedriver && \