Added celery support

docs/pages/customising/settings.rst

@@ -104,3 +104,19 @@ If the setting is ``True``, the admin interface is slightly changed:
    fix this would be welcome!
 
 .. _Django's site framework: http://django.readthedocs.org/en/latest/ref/contrib/sites.html
+
+
+PHOTOLOGUE_RUN_ASYNC
+--------------------
+
+    Default: ``False``
+
+Whether or not Photologue will start processing uploaded files in an async manner.
+
+PHOTOLOGUE_ASYNC_SERVICE
+------------------------
+
+    Default: ``'celery'``
+
+Which async task service Photologue should use. At this moment, only celery is supported.
+Setting up a celery worker is out of the scope of this documentation.

photologue/forms.py

@@ -1,53 +1,37 @@
 import zipfile
 from zipfile import BadZipFile
 import logging
-import os
-from io import BytesIO
-
-from PIL import Image
-
-
 from django import forms
 from django.utils.translation import ugettext_lazy as _
-from django.contrib import messages
-from django.contrib.sites.models import Site
 from django.conf import settings
-from django.utils.encoding import force_text
-from django.template.defaultfilters import slugify
-from django.core.files.base import ContentFile
-
-from .models import Gallery, Photo
+from .models import Gallery, ZipUploadModel
+from photologue.utils.zipfile import handle_zip
 
 logger = logging.getLogger('photologue.forms')
 
+RUN_ASYNC = getattr(settings, 'PHOTOLOGUE_PROCESSING_MODE', 'sync') == 'async'
+
+if RUN_ASYNC:
+    ASYNC_IMPLEMENTATION = getattr(settings, 'PHOTOLOGUE_ASYNC_SERVICE', 'celery')
+    if ASYNC_IMPLEMENTATION == 'celery':
+        from .tasks import parse_zip as parse_zip_task
 
-class UploadZipForm(forms.Form):
-    zip_file = forms.FileField()
 
-    title = forms.CharField(label=_('Title'),
-                            max_length=250,
-                            required=False,
-                            help_text=_('All uploaded photos will be given a title made up of this title + a '
-                                        'sequential number.<br>This field is required if creating a new '
-                                        'gallery, but is optional when adding to an existing gallery - if '
-                                        'not supplied, the photo titles will be creating from the existing '
-                                        'gallery name.'))
-    gallery = forms.ModelChoiceField(Gallery.objects.all(),
-                                     label=_('Gallery'),
-                                     required=False,
-                                     help_text=_('Select a gallery to add these images to. Leave this empty to '
-                                                 'create a new gallery from the supplied title.'))
-    caption = forms.CharField(label=_('Caption'),
-                              required=False,
-                              help_text=_('Caption will be added to all photos.'))
-    description = forms.CharField(label=_('Description'),
-                                  required=False,
-                                  help_text=_('A description of this Gallery. Only required for new galleries.'))
-    is_public = forms.BooleanField(label=_('Is public'),
-                                   initial=True,
-                                   required=False,
-                                   help_text=_('Uncheck this to make the uploaded '
-                                               'gallery and included photographs private.'))
+        def zip_handler(instance, request):
+            instance.save()
+            parse_zip_task.delay(instance.id)
+    else:
+        raise NotImplementedError('Django-photologue only supports celery at this moment')
+
+else:
+    def zip_handler(instance, request):
+        handle_zip(instance, request=request)
+
+
+class UploadZipForm(forms.ModelForm):
+    class Meta:
+        model = ZipUploadModel
+        exclude = ()
 
     def clean_zip_file(self):
         """Open the zip file a first time, to check that it is a valid zip archive.
@@ -85,90 +69,7 @@ def clean(self):
     def save(self, request=None, zip_file=None):
         if not zip_file:
             zip_file = self.cleaned_data['zip_file']
-        zip = zipfile.ZipFile(zip_file)
-        count = 1
-        current_site = Site.objects.get(id=settings.SITE_ID)
-        if self.cleaned_data['gallery']:
-            logger.debug('Using pre-existing gallery.')
-            gallery = self.cleaned_data['gallery']
-        else:
-            logger.debug(
-                force_text('Creating new gallery "{0}".').format(self.cleaned_data['title']))
-            gallery = Gallery.objects.create(title=self.cleaned_data['title'],
-                                             slug=slugify(self.cleaned_data['title']),
-                                             description=self.cleaned_data['description'],
-                                             is_public=self.cleaned_data['is_public'])
-            gallery.sites.add(current_site)
-        for filename in sorted(zip.namelist()):
-
-            logger.debug('Reading file "{}".'.format(filename))
-
-            if filename.startswith('__') or filename.startswith('.'):
-                logger.debug('Ignoring file "{}".'.format(filename))
-                continue
-
-            if os.path.dirname(filename):
-                logger.warning('Ignoring file "{}" as it is in a subfolder; all images should be in the top '
-                               'folder of the zip.'.format(filename))
-                if request:
-                    messages.warning(request,
-                                     _('Ignoring file "{filename}" as it is in a subfolder; all images should '
-                                       'be in the top folder of the zip.').format(filename=filename),
-                                     fail_silently=True)
-                continue
-
-            data = zip.read(filename)
-
-            if not len(data):
-                logger.debug('File "{}" is empty.'.format(filename))
-                continue
-
-            photo_title_root = self.cleaned_data['title'] if self.cleaned_data['title'] else gallery.title
-
-            # A photo might already exist with the same slug. So it's somewhat inefficient,
-            # but we loop until we find a slug that's available.
-            while True:
-                photo_title = ' '.join([photo_title_root, str(count)])
-                slug = slugify(photo_title)
-                if Photo.objects.filter(slug=slug).exists():
-                    count += 1
-                    continue
-                break
-
-            photo = Photo(title=photo_title,
-                          slug=slug,
-                          caption=self.cleaned_data['caption'],
-                          is_public=self.cleaned_data['is_public'])
-
-            # Basic check that we have a valid image.
-            try:
-                file = BytesIO(data)
-                opened = Image.open(file)
-                opened.verify()
-            except Exception:
-                # Pillow doesn't recognize it as an image.
-                # If a "bad" file is found we just skip it.
-                # But we do flag this both in the logs and to the user.
-                logger.error('Could not process file "{}" in the .zip archive.'.format(
-                    filename))
-                if request:
-                    messages.warning(request,
-                                     _('Could not process file "{0}" in the .zip archive.').format(
-                                         filename),
-                                     fail_silently=True)
-                continue
-
-            contentfile = ContentFile(data)
-            photo.image.save(filename, contentfile)
-            photo.save()
-            photo.sites.add(current_site)
-            gallery.photos.add(photo)
-            count += 1
-
-        zip.close()
 
-        if request:
-            messages.success(request,
-                             _('The photos have been added to gallery "{0}".').format(
-                                 gallery.title),
-                             fail_silently=True)
+        instance = super().save(commit=False)
+        instance.zip_file = zip_file
+        zip_handler(instance, request)

photologue/migrations/0012_zipuploadmodel.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.0.3 on 2020-08-02 11:29
+
+import django.core.files.storage
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('photologue', '0011_auto_20190223_2138'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ZipUploadModel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('zip_file', models.FileField(storage=django.core.files.storage.FileSystemStorage(), upload_to='gallery-zip-files/')),
+                ('title', models.CharField(blank=True, help_text='All uploaded photos will be given a title made up of this title + a sequential number.<br>This field is required if creating a new gallery, but is optional when adding to an existing gallery - if not supplied, the photo titles will be creating from the existing gallery name.', max_length=255, null=True)),
+                ('caption', models.TextField(blank=True, help_text='Caption will be added to all photos.', null=True)),
+                ('description', models.TextField(blank=True, help_text='A description of this Gallery. Only required for new galleries.', null=True)),
+                ('is_public', models.BooleanField(default=True, help_text='Uncheck this to make the uploaded gallery and included photographs private.')),
+                ('gallery', models.ForeignKey(blank=True, help_text='Select a gallery to add these images to. Leave this empty to create a new gallery from the supplied title.', null=True, on_delete=django.db.models.deletion.CASCADE, to='photologue.Gallery')),
+            ],
+        ),
+    ]

photologue/models.py

@@ -13,7 +13,7 @@
 from django.contrib.sites.models import Site
 from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
-from django.core.files.storage import default_storage
+from django.core.files.storage import default_storage, FileSystemStorage
 from django.core.validators import RegexValidator
 from django.db import models
 from django.db.models.signals import post_save
@@ -51,6 +51,12 @@
 # Photologue image path relative to media root
 PHOTOLOGUE_DIR = getattr(settings, 'PHOTOLOGUE_DIR', 'photologue')
 
+# Use tasks to speed up page loading after uploading photos?
+RUN_ASYNC = getattr(settings, 'PHOTOLOGUE_PROCESSING_MODE', 'sync') == 'async'
+
+# When using celery, where do we want to temporarily store our zip-file after upload?
+TEMP_ZIP_STORAGE = getattr(settings, 'PHOTOLOGUE_TEMP_ZIP_STORAGE', default_storage)
+
 # Look for user function to define file paths
 PHOTOLOGUE_PATH = getattr(settings, 'PHOTOLOGUE_PATH', None)
 if PHOTOLOGUE_PATH is not None:
@@ -464,7 +470,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._old_image = self.image
 
-    def save(self, *args, **kwargs):
+    def save(self, *args, running_in_task=False, **kwargs):
         image_has_changed = False
         if self._get_pk_val() and (self._old_image != self.image):
             image_has_changed = True
@@ -490,7 +496,13 @@ def save(self, *args, **kwargs):
             except:
                 logger.error('Failed to read EXIF DateTimeOriginal', exc_info=True)
         super().save(*args, **kwargs)
-        self.pre_cache()
+        if (not RUN_ASYNC) or running_in_task:
+            # If we upload lots of photos at once (in a zip), there will already be a task made for opening the zip
+            # this means we are now probably already 'running in a task' and there is no reason to add additional tasks
+            self.pre_cache()
+        else:
+            from photologue import tasks  # Imported here to avoid circular dependency
+            tasks.pre_cache.delay(self.id)
 
     def delete(self):
         assert self._get_pk_val() is not None, \
@@ -899,5 +911,24 @@ def add_default_site(instance, created, **kwargs):
     instance.sites.add(Site.objects.get_current())
 
 
+class ZipUploadModel(models.Model):
+    zip_file = models.FileField(upload_to='gallery-zip-files/', storage=TEMP_ZIP_STORAGE)
+    title = models.CharField(max_length=255, null=True, blank=True,
+                             help_text=_('All uploaded photos will be given a title made up of this title + a '
+                                         'sequential number. This field is required if creating a new '
+                                         'gallery, but is optional when adding to an existing gallery - if '
+                                         'not supplied, the photo titles will be creating from the existing '
+                                         'gallery name.'))
+    gallery = models.ForeignKey(Gallery, null=True, blank=True, on_delete=models.CASCADE,
+                                help_text=_('Select a gallery to add these images to. Leave this empty to '
+                                            'create a new gallery from the supplied title.'))
+    caption = models.TextField(null=True, blank=True,
+                               help_text=_('Caption will be added to all photos.'))
+    description = models.TextField(null=True, blank=True,
+                                   help_text=_('A description of this Gallery. Only required for new galleries.'))
+    is_public = models.BooleanField(default=True, help_text=_(
+        'Uncheck this to make the uploaded gallery and included photographs private.'))
+
+
 post_save.connect(add_default_site, sender=Gallery)
 post_save.connect(add_default_site, sender=Photo)

photologue/tasks.py

@@ -0,0 +1,17 @@
+from celery import shared_task
+from .models import ZipUploadModel, Photo
+from .utils.zipfile import handle_zip
+
+
+@shared_task(name='galleries.tasks.pre_cache', max_retries=5)
+def pre_cache(photo_id):
+    image = Photo.objects.get(id=photo_id)
+    image.pre_cache()
+
+
+@shared_task(name='galleries.tasks.parse_zip', max_retries=5)
+def parse_zip(zip_file_id):
+    instance = ZipUploadModel.objects.get(id=zip_file_id)
+    handle_zip(instance)
+    instance.zip_file.delete(False)
+    instance.delete()