Allow read of production Studio bucket

contentcuration/contentcuration/production_settings.py

@@ -12,7 +12,7 @@
 
 MEDIA_ROOT = base_settings.STORAGE_ROOT
 
-DEFAULT_FILE_STORAGE = 'contentcuration.utils.gcs_storage.GoogleCloudStorage'
+DEFAULT_FILE_STORAGE = 'contentcuration.utils.gcs_storage.CompositeGCS'
 SESSION_ENGINE = "django.contrib.sessions.backends.db"
 
 # email settings

contentcuration/contentcuration/sandbox_settings.py

@@ -3,7 +3,7 @@
 
 DEBUG = True
 
-DEFAULT_FILE_STORAGE = "contentcuration.utils.gcs_storage.GoogleCloudStorage"
+DEFAULT_FILE_STORAGE = "contentcuration.utils.gcs_storage.CompositeGCS"
 
 LANGUAGES += (("ar", gettext("Arabic")),)  # noqa
 

contentcuration/contentcuration/settings.py

@@ -292,8 +292,9 @@ def gettext(s):
     # ('en-PT', gettext('English - Pirate')),
 )
 
+PRODUCTION_SITE_ID = 1
 SITE_BY_ID = {
-    'master': 1,
+    'master': PRODUCTION_SITE_ID,
     'unstable': 3,
     'hotfixes': 4,
 }

contentcuration/contentcuration/tests/test_gcs_storage.py

@@ -1,18 +1,15 @@
-#!/usr/bin/env python
-from future import standard_library
-standard_library.install_aliases()
 from io import BytesIO
 
-import pytest
+import mock
 from django.core.files import File
 from django.test import TestCase
+from google.cloud.storage import Bucket
 from google.cloud.storage import Client
 from google.cloud.storage.blob import Blob
 from mixer.main import mixer
-from mock import create_autospec
-from mock import patch
 
-from contentcuration.utils.gcs_storage import GoogleCloudStorage as gcs
+from contentcuration.utils.gcs_storage import CompositeGCS
+from contentcuration.utils.gcs_storage import GoogleCloudStorage
 
 
 class GoogleCloudStorageSaveTestCase(TestCase):
@@ -21,10 +18,10 @@ class GoogleCloudStorageSaveTestCase(TestCase):
     """
 
     def setUp(self):
-        self.blob_class = create_autospec(Blob)
+        self.blob_class = mock.create_autospec(Blob)
         self.blob_obj = self.blob_class("blob", "blob")
-        self.mock_client = create_autospec(Client)
-        self.storage = gcs(client=self.mock_client())
+        self.mock_client = mock.create_autospec(Client)
+        self.storage = GoogleCloudStorage(client=self.mock_client(), bucket_name="bucket")
         self.content = BytesIO(b"content")
 
     def test_calls_upload_from_file(self):
@@ -73,8 +70,8 @@ def test_uploads_cache_control_private_if_content_database(self):
         self.storage.save(filename, self.content, blob_object=self.blob_obj)
         assert "private" in self.blob_obj.cache_control
 
-    @patch("contentcuration.utils.gcs_storage.BytesIO")
-    @patch("contentcuration.utils.gcs_storage.GoogleCloudStorage._is_file_empty", return_value=False)
+    @mock.patch("contentcuration.utils.gcs_storage.BytesIO")
+    @mock.patch("contentcuration.utils.gcs_storage.GoogleCloudStorage._is_file_empty", return_value=False)
     def test_gzip_if_content_database(self, bytesio_mock, file_empty_mock):
         """
         Check that if we're uploading a gzipped content database and
@@ -99,17 +96,17 @@ class RandomFileSchema:
         filename = str
 
     def setUp(self):
-        self.blob_class = create_autospec(Blob)
+        self.blob_class = mock.create_autospec(Blob)
         self.blob_obj = self.blob_class("blob", "blob")
-        self.mock_client = create_autospec(Client)
-        self.storage = gcs(client=self.mock_client())
+        self.mock_client = mock.create_autospec(Client)
+        self.storage = GoogleCloudStorage(client=self.mock_client(), bucket_name="bucket")
         self.local_file = mixer.blend(self.RandomFileSchema)
 
     def test_raises_error_if_mode_is_not_rb(self):
         """
         open() should raise an assertion error if passed in a mode flag that's not "rb".
         """
-        with pytest.raises(AssertionError):
+        with self.assertRaises(AssertionError):
             self.storage.open("randfile", mode="wb")
 
     def test_calls_blob_download_to_file(self):
@@ -130,3 +127,88 @@ def test_returns_django_file(self):
         assert isinstance(f, File)
         # This checks that an actual temp file was written on disk for the file.git
         assert f.name
+
+
+class CompositeGCSTestCase(TestCase):
+    """
+    Tests for the GoogleCloudStorage class.
+    """
+
+    def setUp(self):
+        mock_client_cls = mock.MagicMock(spec_set=Client)
+        bucket_cls = mock.MagicMock(spec_set=Bucket)
+        self.blob_cls = mock.MagicMock(spec_set=Blob)
+
+        self.mock_default_client = mock_client_cls(project="project")
+        self.mock_anon_client = mock_client_cls(project=None)
+
+        self.mock_default_bucket = bucket_cls(self.mock_default_client, "bucket")
+        self.mock_default_client.get_bucket.return_value = self.mock_default_bucket
+        self.mock_anon_bucket = bucket_cls(self.mock_anon_client, "bucket")
+        self.mock_anon_client.get_bucket.return_value = self.mock_anon_bucket
+
+        with mock.patch("contentcuration.utils.gcs_storage._create_default_client", return_value=self.mock_default_client), \
+             mock.patch("contentcuration.utils.gcs_storage.Client.create_anonymous_client", return_value=self.mock_anon_client):
+            self.storage = CompositeGCS()
+
+    def test_get_writeable_backend(self):
+        backend = self.storage._get_writeable_backend()
+        self.assertEqual(backend.client, self.mock_default_client)
+
+    def test_get_writeable_backend__raises_error_if_none(self):
+        self.mock_default_client.project = None
+        with self.assertRaises(AssertionError):
+            self.storage._get_writeable_backend()
+
+    def test_get_readonly_backend(self):
+        self.mock_anon_bucket.get_blob.return_value = self.blob_cls("blob", "blob")
+        backend = self.storage._get_readable_backend("blob")
+        self.assertEqual(backend.client, self.mock_anon_client)
+
+    def test_get_readonly_backend__raises_error_if_not_found(self):
+        self.mock_default_bucket.get_blob.return_value = None
+        self.mock_anon_bucket.get_blob.return_value = None
+        with self.assertRaises(FileNotFoundError):
+            self.storage._get_readable_backend("blob")
+
+    def test_open(self):
+        self.mock_default_bucket.get_blob.return_value = self.blob_cls("blob", "blob")
+        f = self.storage.open("blob")
+        self.assertIsInstance(f, File)
+        self.mock_default_bucket.get_blob.assert_called_with("blob")
+
+    @mock.patch("contentcuration.utils.gcs_storage.Blob")
+    def test_save(self, mock_blob):
+        self.storage.save("blob", BytesIO(b"content"))
+        blob = mock_blob.return_value
+        blob.upload_from_file.assert_called()
+
+    def test_delete(self):
+        mock_blob = self.blob_cls("blob", "blob")
+        self.mock_default_bucket.get_blob.return_value = mock_blob
+        self.storage.delete("blob")
+        mock_blob.delete.assert_called()
+
+    def test_exists(self):
+        self.mock_default_bucket.get_blob.return_value = self.blob_cls("blob", "blob")
+        self.assertTrue(self.storage.exists("blob"))
+
+    def test_exists__returns_false_if_not_found(self):
+        self.mock_default_bucket.get_blob.return_value = None
+        self.assertFalse(self.storage.exists("blob"))
+
+    def test_size(self):
+        mock_blob = self.blob_cls("blob", "blob")
+        self.mock_default_bucket.get_blob.return_value = mock_blob
+        mock_blob.size = 4
+        self.assertEqual(self.storage.size("blob"), 4)
+
+    def test_url(self):
+        mock_blob = self.blob_cls("blob", "blob")
+        self.mock_default_bucket.get_blob.return_value = mock_blob
+        mock_blob.public_url = "https://storage.googleapis.com/bucket/blob"
+        self.assertEqual(self.storage.url("blob"), "https://storage.googleapis.com/bucket/blob")
+
+    def test_get_created_time(self):
+        self.mock_default_bucket.get_blob.return_value = self.blob_cls("blob", "blob")
+        self.assertEqual(self.storage.get_created_time("blob"), self.blob_cls.return_value.time_created)

contentcuration/contentcuration/utils/gcs_storage.py

@@ -18,16 +18,24 @@
 MAX_RETRY_TIME = 60  # seconds
 
 
-class GoogleCloudStorage(Storage):
-    def __init__(self, client=None):
+def _create_default_client(service_account_credentials_path=settings.GCS_STORAGE_SERVICE_ACCOUNT_KEY_PATH):
+    if service_account_credentials_path:
+        return Client.from_service_account_json(service_account_credentials_path)
+    return Client()
+
 
-        self.client = client if client else self._create_default_client()
-        self.bucket = self.client.get_bucket(settings.AWS_S3_BUCKET_NAME)
+class GoogleCloudStorage(Storage):
+    def __init__(self, client, bucket_name):
+        self.client = client
+        self.bucket = self.client.get_bucket(bucket_name)
 
-    def _create_default_client(self, service_account_credentials_path=settings.GCS_STORAGE_SERVICE_ACCOUNT_KEY_PATH):
-        if service_account_credentials_path:
-            return Client.from_service_account_json(service_account_credentials_path)
-        return Client()
+    @property
+    def writeable(self):
+        """
+        See `Client.create_anonymous_client()`
+        :return: True if the client has a project set, False otherwise.
+        """
+        return self.client.project is not None
 
     def open(self, name, mode="rb", blob_object=None):
         """
@@ -79,7 +87,7 @@ def exists(self, name):
         :return: True if the resource with the name exists, or False otherwise.
         """
         blob = self.bucket.get_blob(name)
-        return blob
+        return blob is not None
 
     def size(self, name):
         blob = self.bucket.get_blob(name)
@@ -199,3 +207,65 @@ def _is_file_empty(fobj):
             byt = fobj.read(1)
             fobj.seek(current_location)
         return len(byt) == 0
+
+
+class CompositeGCS(Storage):
+    def __init__(self):
+        self.backends = []
+        # Only add the studio-content bucket (the production bucket) if we're not in production
+        if settings.SITE_ID != settings.PRODUCTION_SITE_ID:
+            self.backends.append(GoogleCloudStorage(Client.create_anonymous_client(), "studio-content"))
+        self.backends.append(GoogleCloudStorage(_create_default_client(), settings.AWS_S3_BUCKET_NAME))
+
+    def _get_writeable_backend(self):
+        """
+        :rtype: GoogleCloudStorage
+        """
+        for backend in self.backends:
+            if backend.writeable:
+                return backend
+        raise AssertionError("No writeable backend found")
+
+    def _get_readable_backend(self, name):
+        """
+        :rtype: GoogleCloudStorage
+        """
+        for backend in self.backends:
+            if backend.exists(name):
+                return backend
+        raise FileNotFoundError("{} not found".format(name))
+
+    def open(self, name, mode='rb'):
+        return self._get_readable_backend(name).open(name, mode)
+
+    def save(self, name, content, max_length=None):
+        return self._get_writeable_backend().save(name, content, max_length=max_length)
+
+    def delete(self, name):
+        self._get_writeable_backend().delete(name)
+
+    def exists(self, name):
+        try:
+            self._get_readable_backend(name)
+            return True
+        except FileNotFoundError:
+            return False
+
+    def listdir(self, path):
+        # This method was not implemented on GoogleCloudStorage to begin with
+        raise NotImplementedError("listdir is not implemented for CompositeGCS")
+
+    def size(self, name):
+        return self._get_readable_backend(name).size(name)
+
+    def url(self, name):
+        return self._get_readable_backend(name).url(name)
+
+    def get_accessed_time(self, name):
+        return self._get_readable_backend(name).get_accessed_time(name)
+
+    def get_created_time(self, name):
+        return self._get_readable_backend(name).get_created_time(name)
+
+    def get_modified_time(self, name):
+        return self._get_readable_backend(name).get_modified_time(name)

contentcuration/contentcuration/utils/storage_common.py

@@ -6,6 +6,7 @@
 from django.core.files.storage import default_storage
 from django_s3_storage.storage import S3Storage
 
+from .gcs_storage import CompositeGCS
 from .gcs_storage import GoogleCloudStorage
 
 
@@ -61,7 +62,7 @@ def get_presigned_upload_url(
     # both storage types are having difficulties enforcing it.
 
     mimetype = determine_content_type(filepath)
-    if isinstance(storage, GoogleCloudStorage):
+    if isinstance(storage, (GoogleCloudStorage, CompositeGCS)):
         client = client or storage.client
         bucket = settings.AWS_S3_BUCKET_NAME
         upload_url = _get_gcs_presigned_put_url(client, bucket, filepath, md5sum_b64, lifetime_sec, mimetype=mimetype)