diff --git a/.ci/bootstrap_catdog.sh b/.ci/bootstrap_catdog.sh index 2fcbf48f..cccb1288 100755 --- a/.ci/bootstrap_catdog.sh +++ b/.ci/bootstrap_catdog.sh @@ -16,6 +16,7 @@ pip install -r test_requirements.txt mkdir -p ../pulp_catdog/.ci/assets/bindings echo 'pypi_username: the_pypi_user' >> ../pulp_catdog/template_config.yml sed -i "s/test_s3: false/test_s3: true/g" ../pulp_catdog/template_config.yml +sed -i "s/test_gcp: false/test_gcp: true/g" ../pulp_catdog/template_config.yml sed -i "s/test_azure: false/test_azure: true/g" ../pulp_catdog/template_config.yml sed -i "s/disabled_redis_runners: \[\]/disabled_redis_runners: [s3]/g" ../pulp_catdog/template_config.yml ./plugin-template --all pulp_catdog diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml index 2e70d091..13d44bda 100644 --- a/.github/workflows/bootstrap.yaml +++ b/.github/workflows/bootstrap.yaml @@ -93,6 +93,7 @@ jobs: - TEST: pulp - TEST: docs - TEST: azure + - TEST: gcp - TEST: s3 - TEST: lowerbounds outputs: diff --git a/CHANGES/713.feature b/CHANGES/713.feature new file mode 100644 index 00000000..e1a2b51b --- /dev/null +++ b/CHANGES/713.feature @@ -0,0 +1 @@ +Add GCP scenario diff --git a/README.md b/README.md index 84bde013..701e78c4 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,9 @@ The following settings are stored in `template_config.yml`. test_azure Include azure job for running tests using [azurite](https://github.com/Azure/Azurite) to emulate Azure. + test_gcp Include gcp job for running tests using [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) + to emulate GCP. + test_s3 Include s3 job for running tests using [minio](https://github.com/minio/minio) to emulate S3. diff --git a/plugin-template b/plugin-template index 25903495..20c16d1c 100755 --- a/plugin-template +++ b/plugin-template @@ -54,6 +54,7 @@ DEFAULT_SETTINGS = { "publish_docs_to_pulpprojectdotorg": False, "pulp_settings": None, "pulp_settings_azure": None, + "pulp_settings_gcp": None, "pulp_settings_s3": None, "pulp_settings_stream": None, "pulpcore_branch": "main", @@ -79,6 +80,7 @@ DEFAULT_SETTINGS = { "test_reroute": True, "disabled_redis_runners": [], "test_azure": False, + "test_gcp": False, "test_s3": False, "test_stream": False, "update_github": True, diff --git a/templates/github/.ci/ansible/Containerfile.j2.copy b/templates/github/.ci/ansible/Containerfile.j2.copy index 47c3d02e..72491d9b 100644 --- a/templates/github/.ci/ansible/Containerfile.j2.copy +++ b/templates/github/.ci/ansible/Containerfile.j2.copy @@ -20,6 +20,9 @@ RUN pip3 install \ {%- if azure_test | default(false) -%} {{ " " }}django-storages[azure]>=1.12.2 {%- endif -%} +{%- if gcp_test | default(false) -%} +{{ " " }}django-storages[google]>=1.13.2 +{%- endif -%} {%- for item in plugins -%} {%- if item.name == "pulp-certguard" -%} {{ " " }}python-dateutil rhsm diff --git a/templates/github/.ci/ansible/settings.py.j2.copy b/templates/github/.ci/ansible/settings.py.j2.copy index 2dad6711..9470aafa 100644 --- a/templates/github/.ci/ansible/settings.py.j2.copy +++ b/templates/github/.ci/ansible/settings.py.j2.copy @@ -63,3 +63,10 @@ AZURE_OVERWRITE_FILES = True AZURE_URL_EXPIRATION_SECS = 120 AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;' {% endif %} + +{% if gcp_test | default(false) %} +DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" +MEDIA_ROOT = "" +GS_BUCKET_NAME = "gcppulp" +GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443" +{% endif %} diff --git a/templates/github/.github/workflows/ci.yml.j2 b/templates/github/.github/workflows/ci.yml.j2 index 20f50c0a..71718692 100644 --- a/templates/github/.github/workflows/ci.yml.j2 +++ b/templates/github/.github/workflows/ci.yml.j2 @@ -109,6 +109,9 @@ jobs: {%- if test_azure %} - TEST: azure {%- endif %} + {%- if test_gcp %} + - TEST: gcp + {%- endif %} {%- if test_s3 %} - TEST: s3 {%- endif %} diff --git a/templates/github/.github/workflows/nightly.yml.j2 b/templates/github/.github/workflows/nightly.yml.j2 index f69013a6..5a5ff7b8 100644 --- a/templates/github/.github/workflows/nightly.yml.j2 +++ b/templates/github/.github/workflows/nightly.yml.j2 @@ -36,6 +36,9 @@ jobs: {%- if test_azure %} - TEST: azure {%- endif %} + {%- if test_gcp %} + - TEST: gcp + {%- endif %} {%- if test_s3 %} - TEST: s3 {%- endif %} diff --git a/templates/github/.github/workflows/release.yml.j2 b/templates/github/.github/workflows/release.yml.j2 index 0d02a705..534c8d8a 100644 --- a/templates/github/.github/workflows/release.yml.j2 +++ b/templates/github/.github/workflows/release.yml.j2 @@ -77,6 +77,9 @@ jobs: {%- if test_azure %} - TEST: azure {%- endif %} + {%- if test_gcp %} + - TEST: gcp + {%- endif %} {%- if test_s3 %} - TEST: s3 {%- endif %} diff --git a/templates/github/.github/workflows/scripts/before_script.sh.j2 b/templates/github/.github/workflows/scripts/before_script.sh.j2 index 10b65f4a..ba7e6b14 100755 --- a/templates/github/.github/workflows/scripts/before_script.sh.j2 +++ b/templates/github/.github/workflows/scripts/before_script.sh.j2 @@ -28,10 +28,10 @@ tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" cmd_prefix bash -c "usermod -a -G wheel pulp" -SCENARIOS=("pulp" "performance" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds") +SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds") if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then # Many functional tests require these - cmd_prefix dnf install -yq lsof which dnf-plugins-core + cmd_prefix dnf install -yq lsof which fi if [[ "${REDIS_DISABLED:-false}" == true ]]; then diff --git a/templates/github/.github/workflows/scripts/install.sh.j2 b/templates/github/.github/workflows/scripts/install.sh.j2 index ead4a2f2..6cfaea03 100755 --- a/templates/github/.github/workflows/scripts/install.sh.j2 +++ b/templates/github/.github/workflows/scripts/install.sh.j2 @@ -135,7 +135,7 @@ VARSYAML {%- if docker_fixtures %} -SCENARIOS=("pulp" "performance" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds") +SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds") if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then sed -i -e '/^services:/a \ - name: pulp-fixtures\ @@ -203,6 +203,27 @@ pulp_scenario_settings: {{ pulp_settings_azure | tojson }}\ fi {%- endif %} +{%- if test_gcp %} + +if [ "$TEST" = "gcp" ]; then + mkdir -p cigcp + cd cigcp + openssl req -newkey rsa:2048 -x509 -nodes -keyout gcpkey.pem -new -out gcpcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:ci-gcp" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN" + sudo cp gcpcert.pem /usr/local/share/ca-certificates/gcpcert.crt + sudo dpkg-reconfigure ca-certificates + cd .. + sed -i -e '/^services:/a \ + - name: ci-gcp\ + image: fsouza/fake-gcs-server\ + volumes:\ + - storage_data:/etc/pulp\ + command: "{% if pulp_scheme == "https" %} -cert-location /etc/pulp/gcpcert.pem {% else %} -scheme http{% endif %}"' vars/main.yaml + sed -i -e '$a gcp_test: true\ +pulp_scenario_settings: {{ pulp_settings_gcp | tojson }}\ +' vars/main.yaml +fi +{%- endif %} + echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" if [ "${PULP_API_ROOT:-}" ]; then