Added GCP storage support

.ci/ansible/settings.py.j2

@@ -68,5 +68,6 @@ AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountNam
 DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
 MEDIA_ROOT = ""
 GS_BUCKET_NAME = "gcppulp"
-GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
+GS_CUSTOM_ENDPOINT = "https://ci-gcp:4443"
+GOOGLE_APPLICATION_CREDENTIALS = "/etc/pulp/credentials.json"
 {% endif %}

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-192-gba67a84
+2021.08.26-192-g61ee942

.github/workflows/ci.yml

@@ -96,6 +96,7 @@ jobs:
           - TEST: pulp
           - TEST: docs
           - TEST: azure
+          - TEST: gcp
           - TEST: s3
           - TEST: stream
           - TEST: lowerbounds

.github/workflows/nightly.yml

@@ -25,6 +25,7 @@ jobs:
           - TEST: pulp
           - TEST: docs
           - TEST: azure
+          - TEST: gcp
           - TEST: s3
           - TEST: stream
           - TEST: generate-bindings

.github/workflows/release.yml

@@ -83,6 +83,7 @@ jobs:
           - TEST: pulp
           - TEST: docs
           - TEST: azure
+          - TEST: gcp
           - TEST: s3
           - TEST: stream
           - TEST: generate-bindings

.github/workflows/scripts/install.sh

@@ -74,6 +74,7 @@ services:
       - ../../../pulp-openapi-generator:/root/pulp-openapi-generator
     env:
       PULP_WORKERS: "4"
+      GOOGLE_APPLICATION_CREDENTIALS: "/etc/pulp/credentials.json"
 VARSYAML
 
 cat >> vars/main.yaml << VARSYAML
@@ -130,6 +131,35 @@ if [ "$TEST" = "azure" ]; then
 pulp_scenario_settings: null\
 ' vars/main.yaml
 fi
+cat >> credentials.json << GCP_JSON
+{
+  "client_id": "pulp-ci-test.apps.googleusercontent.com",
+  "client_secret": "pulp-Secret",
+  "refresh_token": "Pulp-CI-Refresh-Token",
+  "type": "authorized_user"
+}
+GCP_JSON
+if [ "$TEST" = "gcp" ]; then
+  mkdir -p cigcp gcpdata/gcppulp
+  mv credentials.json cigcp/credentials.json
+  cd cigcp
+  openssl req -newkey rsa:2048 -x509 -nodes -keyout gcpkey.pem -new -out gcpcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:ci-gcp" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN"
+  sudo cp gcpcert.pem /usr/local/share/ca-certificates/gcpcert.crt
+  sudo dpkg-reconfigure ca-certificates
+  cd ..
+  sed -i -e '/^services:/a \
+  - name: ci-gcp\
+    image: fsouza/fake-gcs-server\
+    volumes:\
+      - ./cigcp:/etc/pulp\
+      - ./gcpdata:/data\
+    env:\
+      GOOGLE_APPLICATION_CREDENTIALS: "/etc/pulp/credentials.json"\
+    command: "-public-host https://ci-gcp:4443  -cert-location /etc/pulp/gcpcert.pem "' vars/main.yaml
+  sed -i -e '$a gcp_test: true\
+pulp_scenario_settings: null\
+' vars/main.yaml
+fi
 
 echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
 
@@ -167,6 +197,10 @@ cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null
 sudo update-ca-certificates
 echo ::endgroup::
 
+if [[ "$TEST" = "gcp" ]]; then
+  cp cigcp/credentials.json settings/credentials.json
+fi
+
 if [[ "$TEST" = "azure" ]]; then
   AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())')
   cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI

CHANGES/1140.feature

@@ -0,0 +1 @@
+Added GCP storage support.

pulp_container/app/redirects.py

@@ -132,3 +132,21 @@ def redirect_to_object_storage(self, artifact, return_media_type):
         }
         content_url = artifact.file.storage.url(artifact.file.name, parameters=parameters)
         return redirect(content_url)
+
+
+class GCloudStorageRedirects(S3StorageRedirects):
+    """
+    A class that implements methods for the direct retrieval of manifest objects.
+    """
+
+    def redirect_to_object_storage(self, artifact, return_media_type):
+        """
+        Redirect to the passed artifact's file stored in the GCP storage.
+        """
+        filename = f"sha256:{artifact.sha256}"
+        parameters = {
+            "content_type": return_media_type,
+            "response_disposition": f"attachment;filename={filename}",
+        }
+        content_url = artifact.file.storage.url(artifact.file.name, parameters=parameters)
+        return redirect(content_url)

pulp_container/app/registry_api.py

@@ -65,6 +65,7 @@
     FileStorageRedirects,
     S3StorageRedirects,
     AzureStorageRedirects,
+    GCloudStorageRedirects,
 )
 from pulp_container.app.token_verification import (
     RegistryAuthentication,
@@ -795,6 +796,8 @@ def __init__(self, *args, **kwargs):
             self.redirects_class = S3StorageRedirects
         elif settings.DEFAULT_FILE_STORAGE == "storages.backends.azure_storage.AzureStorage":
             self.redirects_class = AzureStorageRedirects
+        elif settings.DEFAULT_FILE_STORAGE == "storages.backends.gcloud.GoogleCloudStorage":
+            self.redirects_class = GCloudStorageRedirects
         else:
             raise NotImplementedError()
 

template_config.yml

@@ -89,8 +89,8 @@ test_azure: true
 test_bindings: false
 test_cli: true
 test_deprecations: true
-test_gcp: false
 test_lowerbounds: true
+test_gcp: true
 test_performance: false
 test_released_plugin_with_next_pulpcore_release: false
 test_reroute: true