diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 9470aafa0..c59f0284d 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -68,5 +68,6 @@ AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountNam DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" MEDIA_ROOT = "" GS_BUCKET_NAME = "gcppulp" -GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443" +GS_CUSTOM_ENDPOINT = "https://ci-gcp:4443" +GOOGLE_APPLICATION_CREDENTIALS = "/etc/pulp/credentials.json" {% endif %} diff --git a/.github/template_gitref b/.github/template_gitref index ff38bd047..aa38c47e7 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-192-gba67a84 +2021.08.26-192-g61ee942 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b96719c6a..219a89b34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,6 +96,7 @@ jobs: - TEST: pulp - TEST: docs - TEST: azure + - TEST: gcp - TEST: s3 - TEST: stream - TEST: lowerbounds diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bfc23c052..61b9b8099 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -25,6 +25,7 @@ jobs: - TEST: pulp - TEST: docs - TEST: azure + - TEST: gcp - TEST: s3 - TEST: stream - TEST: generate-bindings diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f30e8618..5c36f328a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,6 +83,7 @@ jobs: - TEST: pulp - TEST: docs - TEST: azure + - TEST: gcp - TEST: s3 - TEST: stream - TEST: generate-bindings diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 9b3f601e3..660c3b916 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -74,6 +74,7 @@ services: - ../../../pulp-openapi-generator:/root/pulp-openapi-generator env: PULP_WORKERS: "4" + GOOGLE_APPLICATION_CREDENTIALS: "/etc/pulp/credentials.json" VARSYAML cat >> vars/main.yaml << VARSYAML @@ -130,6 +131,35 @@ if [ "$TEST" = "azure" ]; then pulp_scenario_settings: null\ ' vars/main.yaml fi +cat >> credentials.json << GCP_JSON +{ + "client_id": "pulp-ci-test.apps.googleusercontent.com", + "client_secret": "pulp-Secret", + "refresh_token": "Pulp-CI-Refresh-Token", + "type": "authorized_user" +} +GCP_JSON +if [ "$TEST" = "gcp" ]; then + mkdir -p cigcp gcpdata/gcppulp + mv credentials.json cigcp/credentials.json + cd cigcp + openssl req -newkey rsa:2048 -x509 -nodes -keyout gcpkey.pem -new -out gcpcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:ci-gcp" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN" + sudo cp gcpcert.pem /usr/local/share/ca-certificates/gcpcert.crt + sudo dpkg-reconfigure ca-certificates + cd .. + sed -i -e '/^services:/a \ + - name: ci-gcp\ + image: fsouza/fake-gcs-server\ + volumes:\ + - ./cigcp:/etc/pulp\ + - ./gcpdata:/data\ + env:\ + GOOGLE_APPLICATION_CREDENTIALS: "/etc/pulp/credentials.json"\ + command: "-public-host https://ci-gcp:4443 -cert-location /etc/pulp/gcpcert.pem "' vars/main.yaml + sed -i -e '$a gcp_test: true\ +pulp_scenario_settings: null\ +' vars/main.yaml +fi echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" @@ -167,6 +197,10 @@ cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null sudo update-ca-certificates echo ::endgroup:: +if [[ "$TEST" = "gcp" ]]; then + cp cigcp/credentials.json settings/credentials.json +fi + if [[ "$TEST" = "azure" ]]; then AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())') cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI diff --git a/CHANGES/1140.feature b/CHANGES/1140.feature new file mode 100644 index 000000000..5f8aabb11 --- /dev/null +++ b/CHANGES/1140.feature @@ -0,0 +1 @@ +Added GCP storage support. diff --git a/pulp_container/app/redirects.py b/pulp_container/app/redirects.py index e50d6a469..125fac941 100644 --- a/pulp_container/app/redirects.py +++ b/pulp_container/app/redirects.py @@ -132,3 +132,21 @@ def redirect_to_object_storage(self, artifact, return_media_type): } content_url = artifact.file.storage.url(artifact.file.name, parameters=parameters) return redirect(content_url) + + +class GCloudStorageRedirects(S3StorageRedirects): + """ + A class that implements methods for the direct retrieval of manifest objects. + """ + + def redirect_to_object_storage(self, artifact, return_media_type): + """ + Redirect to the passed artifact's file stored in the GCP storage. + """ + filename = f"sha256:{artifact.sha256}" + parameters = { + "content_type": return_media_type, + "response_disposition": f"attachment;filename={filename}", + } + content_url = artifact.file.storage.url(artifact.file.name, parameters=parameters) + return redirect(content_url) diff --git a/pulp_container/app/registry_api.py b/pulp_container/app/registry_api.py index 8318a66eb..0c5100626 100644 --- a/pulp_container/app/registry_api.py +++ b/pulp_container/app/registry_api.py @@ -65,6 +65,7 @@ FileStorageRedirects, S3StorageRedirects, AzureStorageRedirects, + GCloudStorageRedirects, ) from pulp_container.app.token_verification import ( RegistryAuthentication, @@ -795,6 +796,8 @@ def __init__(self, *args, **kwargs): self.redirects_class = S3StorageRedirects elif settings.DEFAULT_FILE_STORAGE == "storages.backends.azure_storage.AzureStorage": self.redirects_class = AzureStorageRedirects + elif settings.DEFAULT_FILE_STORAGE == "storages.backends.gcloud.GoogleCloudStorage": + self.redirects_class = GCloudStorageRedirects else: raise NotImplementedError() diff --git a/template_config.yml b/template_config.yml index eddc3613f..ff845fee7 100644 --- a/template_config.yml +++ b/template_config.yml @@ -89,8 +89,8 @@ test_azure: true test_bindings: false test_cli: true test_deprecations: true -test_gcp: false test_lowerbounds: true +test_gcp: true test_performance: false test_released_plugin_with_next_pulpcore_release: false test_reroute: true