Merge pull request #80 from octue/access-ingress-path

Access ingress path

Author: thclark

django_gcp/storage/__init__.py

@@ -1,7 +1,8 @@
 # TODO: Refactor tests to import members directly then remove this module export
 from . import gcloud
-from .blob_utils import BlobFieldMixin, get_blob, get_blob_name, get_path, get_signed_url, upload_blob
+from .blob_utils import BlobFieldMixin, get_blob, get_blob_name, get_path, get_signed_url
 from .gcloud import GoogleCloudFile, GoogleCloudMediaStorage, GoogleCloudStaticStorage, GoogleCloudStorage
+from .operations import upload_blob
 
 __all__ = [
     "GoogleCloudStorage",

django_gcp/storage/blob_utils.py

@@ -1,74 +1,14 @@
 from datetime import timedelta
 from os.path import split
 
-from google.cloud.storage.blob import Blob
-
-
-def upload_blob(
-    instance,
-    field_name,
-    local_path,
-    destination_path=None,
-    attributes=None,
-    allow_overwrite=False,
-    existing_path=None,
-):
-    """Upload a file to the cloud store, using the instance and field name to determine the store details
-
-    You might use this utility to upload fixtue files for integration tests, as part of
-    a migration, or as part of a function creating local files. The field's own logic for generating paths
-    is used by default, although this can be overridden.
-
-    Returns the field value so you can use this to construct instances directly:
-
-    # Directly set the instance field without processing the blob ingress
-    with override_settings(GCP_STORAGE_OVERRIDE_BLOBFIELD_VALUE=True):
-        instance.field_name = upload_blob(...)
-
-    :param django.db.Model instance: An instance of a django Model which has a BlobField
-    :param str field_name: The name of the BlobField attribute on the instance
-    :param str local_path: The path to the file to upload
-    :param str destination_path: The path to upload the file to. If None, the remote path will be generated
-    from the BlobField. If setting this value, take care to override the value of the field on the instance
-    so it's path matches; this is not updated for you.
-    :param dict attributes: A dictionary of attributes to set on the blob eg content type
-    :param bool allow_overwrite: If true, allows existing blobs at the path to be overwritten. If destination_path is not given, this is provided to the get_destination_path callback (and may be overridden by that callback per its specification)
-    :param str existing_path: If destination_path is None, this is provided to the get_destination_path callback to simulate behaviour where there is an existing path
-    """
-    # Get the field (which
-    field = instance._meta.get_field(field_name)
-    if destination_path is None:
-        destination_path, allow_overwrite = field.get_destination_path(
-            instance,
-            original_name=local_path,
-            attributes=attributes,
-            allow_overwrite=allow_overwrite,
-            existing_path=existing_path,
-            bucket=field.storage.bucket,
-        )
-
-    # If not allowing overwrite, set generation matching constraints to prevent it
-    if_generation_match = None if allow_overwrite else 0
-
-    # Attributes must be a dict by default
-    attributes = attributes or {}
-
-    # Upload the file
-    Blob(destination_path, bucket=field.storage.bucket).upload_from_filename(
-        local_path, if_generation_match=if_generation_match, **attributes
-    )
-
-    # Return the field value
-    return {"path": destination_path}
-
 
 def get_path(instance, field_name):
     """Get the path of the blob in the object store"""
     field_value = getattr(instance, field_name)
     return field_value.get("path", None) if field_value is not None else None
 
 
-def get_blob(instance, field_name):
+def get_blob(instance, field_name, reload=True):
     """Get a blob from a model instance containing a BlobField
 
     This allows you to download the blob to a local file. For example:
@@ -85,11 +25,16 @@ def get_blob(instance, field_name):
 
     :param django.db.Model instance: An instance of a django Model which has a BlobField
     :param str field_name: The name of the BlobField attribute on the instance
+    :param bool reload: Default True. If it is not essential to have up-to-date information from the store, speed up the call to get_blob using call with reload_blob=False
     """
     path = get_path(instance, field_name)
     if path is not None:
         field = instance._meta.get_field(field_name)
-        return field.storage.bucket.blob(path)
+        blob = field.storage.bucket.blob(path)
+        if reload:
+            blob.reload()
+
+        return blob
 
 
 def get_blob_name(instance, field_name):

django_gcp/storage/fields.py

@@ -105,6 +105,7 @@ def __init__(
         max_size_bytes=DEFAULT_MAX_SIZE_BYTES,
         overwrite_mode=DEFAULT_OVERWRITE_MODE,
         on_change=None,
+        update_attributes=None,
         **kwargs,
     ):
         self._validated = False
@@ -117,7 +118,9 @@ def __init__(
         self._choices_set_explicitly = "choices" in kwargs
         self.overwrite_mode = overwrite_mode
         self.get_destination_path = get_destination_path
+        self.update_attributes = update_attributes
         self.ingress_to = ingress_to
+        self.ingress_path = None
         self.store_key = store_key
         self.accept_mimetype = accept_mimetype
         self.max_size_bytes = max_size_bytes
@@ -152,6 +155,7 @@ def check(self, **kwargs):
             *super().check(**kwargs),
             *self._check_explicit(),
             *self._check_get_destination_path(),
+            *self._check_update_attributes(),
             *self._check_ingress_to(),
             *self._check_overwrite_mode(),
             *self._check_on_change(),
@@ -164,6 +168,7 @@ def deconstruct(self):
         kwargs["accept_mimetype"] = self.accept_mimetype
         kwargs["overwrite_mode"] = self.overwrite_mode
         kwargs["get_destination_path"] = self.get_destination_path
+        kwargs["update_attributes"] = self.update_attributes
         kwargs["on_change"] = self.on_change
         return name, path, args, kwargs
 
@@ -248,15 +253,29 @@ def on_commit_blank():
 
                 elif adding_valid or updating_blank_to_valid or updating_valid_to_valid:
                     new_value = {}
+
+                    allow_overwrite = self._get_allow_overwrite(add)
+
+                    attributes = self._update_attributes(
+                        getattr(value, "attributes", {}),
+                        instance=model_instance,
+                        original_name=value["name"],
+                        existing_path=existing_path,
+                        temporary_path=value["_tmp_path"],
+                        adding=add,
+                        bucket=self.storage.bucket,
+                    )
+
                     new_value["path"], allow_overwrite = self._get_destination_path(
                         instance=model_instance,
                         original_name=value["name"],
-                        attributes=getattr(value, "attributes", None),
+                        attributes=attributes,
                         allow_overwrite=self._get_allow_overwrite(add),
                         existing_path=existing_path,
                         temporary_path=value["_tmp_path"],
                         bucket=self.storage.bucket,
                     )
+
                     logger.info(
                         "Adding/updating cloud object via temporary ingress at %s to %s",
                         value["_tmp_path"],
@@ -275,7 +294,7 @@ def on_commit_valid():
                             new_value["path"],
                             move=True,
                             overwrite=allow_overwrite,
-                            attributes=value.get("attributes", None),
+                            attributes=attributes,
                         )
                         if self.on_change is not None:
                             self.on_change(new_value, instance=model_instance)
@@ -482,6 +501,17 @@ def _check_get_destination_path(self):
             ]
         return []
 
+    def _check_update_attributes(self):
+        if self.update_attributes is not None and not callable(self.update_attributes):
+            return [
+                checks.Error(
+                    f"'update_attributes' argument in {self.__class__.__name__} must be None, or a callable function that updates attributes to be set on ingressed blobs.",
+                    obj=self,
+                    id="fields.E201",
+                )
+            ]
+        return []
+
     def _check_on_change(self):
         if self.on_change is not None and not callable(self.on_change):
             return [
@@ -604,3 +634,17 @@ def _get_destination_path(self, *args, **kwargs):
             self.get_destination_path,
         )
         return get_destination_path(*args, **kwargs)
+
+    def _update_attributes(self, attributes, **kwargs):
+        """Call the update_attributes callback unless an override is defined in
+        settings. This funcitonality is intended for test purposes only, because
+        patching the callback in a test framework is a struggle
+        """
+        update_attributes = getattr(
+            settings,
+            "GCP_STORAGE_OVERRIDE_UPDATE_ATTRIBUTES_CALLBACK",
+            self.update_attributes,
+        )
+        if update_attributes is not None:
+            return update_attributes(attributes, **kwargs)
+        return attributes

django_gcp/storage/operations.py

@@ -1,9 +1,11 @@
 import datetime
 import logging
+
 from django.utils import timezone
-from django_gcp.exceptions import AttemptedOverwriteError, MissingBlobError
 from google.cloud.exceptions import NotFound, PreconditionFailed
+from google.cloud.storage.blob import Blob
 
+from django_gcp.exceptions import AttemptedOverwriteError, MissingBlobError
 
 logger = logging.getLogger(__name__)
 
@@ -14,6 +16,64 @@ def blob_exists(bucket, blob_name):
     return blob.exists()
 
 
+def upload_blob(
+    instance,
+    field_name,
+    local_path,
+    destination_path=None,
+    attributes=None,
+    allow_overwrite=False,
+    existing_path=None,
+):
+    """Upload a file to the cloud store, using the instance and field name to determine the store details
+
+    You might use this utility to upload fixtue files for integration tests, as part of
+    a migration, or as part of a function creating local files. The field's own logic for generating paths
+    is used by default, although this can be overridden.
+
+    Returns the field value so you can use this to construct instances directly:
+
+    # Directly set the instance field without processing the blob ingress
+    with override_settings(GCP_STORAGE_OVERRIDE_BLOBFIELD_VALUE=True):
+        instance.field_name = upload_blob(...)
+
+    :param django.db.Model instance: An instance of a django Model which has a BlobField
+    :param str field_name: The name of the BlobField attribute on the instance
+    :param str local_path: The path to the file to upload
+    :param str destination_path: The path to upload the file to. If None, the remote path will be generated
+    from the BlobField. If setting this value, take care to override the value of the field on the instance
+    so it's path matches; this is not updated for you.
+    :param dict attributes: A dictionary of attributes to set on the blob eg content type
+    :param bool allow_overwrite: If true, allows existing blobs at the path to be overwritten. If destination_path is not given, this is provided to the get_destination_path callback (and may be overridden by that callback per its specification)
+    :param str existing_path: If destination_path is None, this is provided to the get_destination_path callback to simulate behaviour where there is an existing path
+    """
+    # Get the field (which
+    field = instance._meta.get_field(field_name)
+    if destination_path is None:
+        destination_path, allow_overwrite = field.get_destination_path(
+            instance,
+            original_name=local_path,
+            attributes=attributes,
+            allow_overwrite=allow_overwrite,
+            existing_path=existing_path,
+            bucket=field.storage.bucket,
+        )
+
+    # If not allowing overwrite, set generation matching constraints to prevent it
+    if_generation_match = None if allow_overwrite else 0
+
+    # Attributes must be a dict by default
+    attributes = attributes or {}
+
+    # Upload the file
+    Blob(destination_path, bucket=field.storage.bucket).upload_from_filename(
+        local_path, if_generation_match=if_generation_match, **attributes
+    )
+
+    # Return the field value
+    return {"path": destination_path}
+
+
 def copy_blob(
     source_bucket,
     source_blob_name,

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "django-gcp"
-version = "0.16.1"
+version = "0.17.0"
 description = "Utilities to run Django on Google Cloud Platform"
 authors = ["Tom Clark"]
 license = "MIT"

tests/server/example/migrations/0008_exampleupdateattributesblobfieldmodel.py

@@ -0,0 +1,45 @@
+# Generated by Django 4.2.19 on 2025-03-20 13:00
+
+from django.db import migrations, models
+import django_gcp.storage.fields
+import tests.server.example.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("example", "0007_exampleneveroverwriteblobfieldmodel_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ExampleUpdateAttributesBlobFieldModel",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("category", models.CharField(blank=True, max_length=20, null=True)),
+                (
+                    "blob",
+                    django_gcp.storage.fields.BlobField(
+                        accept_mimetype="*/*",
+                        blank=True,
+                        default=None,
+                        get_destination_path=tests.server.example.models.get_destination_path,
+                        help_text="On save, metadata should be updated on the blob.",
+                        ingress_to="_tmp/",
+                        null=True,
+                        on_change=None,
+                        overwrite_mode="never",
+                        store_key="media",
+                        update_attributes=tests.server.example.models.update_attributes,
+                    ),
+                ),
+            ],
+        ),
+    ]

tests/server/example/models.py

@@ -363,3 +363,45 @@ class Meta:
         """Metaclass defining this model to reside in the example app"""
 
         app_label = "example"
+
+
+def update_attributes(attributes, instance, original_name, existing_path, temporary_path, adding, bucket):
+    """Update the attributes set on the blob upon move to its destination path
+
+    These attributes can be any valid attributes that are set on GCP blobs, such as content-type and metadata.
+
+    This callback is executed prior to the save() method so the file will be residing in the ingress directory.
+    It's advised not to download the file during this stage due to the duration of that operation (we recommend that
+    if full file download is required, these attributes are set later in a separate or async task).
+
+    In some cases we've seen the first few bytes of files being streamed in order to read and extract values from headers
+    which is a neat approach.
+
+    See https://cloud.google.com/python/docs/reference/storage/latest/google.cloud.storage.blob.Blob
+    """
+    attributes["content_type"] = "image/png"
+    attributes["metadata"] = {"category": instance.category}
+    return attributes
+
+
+class ExampleUpdateAttributesBlobFieldModel(Model):
+    """
+    As ExampleBlobFieldModel but showing use of the update_attributes callback
+    (This is mostly used for widget development and testing)
+    """
+
+    category = CharField(max_length=20, blank=True, null=True)
+
+    blob = BlobField(
+        get_destination_path=get_destination_path,
+        update_attributes=update_attributes,
+        store_key="media",
+        blank=True,
+        null=True,
+        help_text="On save, metadata should be updated on the blob.",
+    )
+
+    class Meta:
+        """Metaclass defining this model to reside in the example app"""
+
+        app_label = "example"