Add submissions zip download feature

exercise/api/csv/views.py

@@ -158,6 +158,8 @@ def submitted_field(submission, name):
         response = Response(data)
         if isinstance(getattr(request, 'accepted_renderer'), CSVRenderer):
             response['Content-Disposition'] = 'attachment; filename="submissions.csv"'
+        else:
+            response['Content-Disposition'] = 'attachment; filename="submissions.json"'
         return response
 
     def get_renderer_context(self):

exercise/api/views.py

@@ -1,11 +1,13 @@
+from io import BytesIO
+import zipfile
+
 from aplus_auth.payload import Permission
 from django.core.exceptions import PermissionDenied
-from django.http.response import HttpResponse
+from django.http.response import HttpResponse, FileResponse
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from wsgiref.util import FileWrapper
-from rest_framework import mixins, permissions, viewsets
-from rest_framework import status
+from rest_framework import mixins, permissions, status, viewsets
 from rest_framework.response import Response
 from rest_framework.reverse import reverse
 from rest_framework.decorators import action
@@ -28,6 +30,7 @@
 from course.api.mixins import CourseResourceMixin
 from exercise.async_views import _post_async_submission
 
+from ..cache.points import ExercisePoints
 from ..models import (
     Submission,
     SubmittedFile,
@@ -322,6 +325,107 @@ def submit(self, request, *args, **kwargs):
 
         return Response(data, status=status_code, headers=headers)
 
+    @action(
+        detail=False,
+        url_path='zip',
+        url_name='zip',
+        methods=['get'],
+    )
+    def zip(self, request, exercise_id, *args, **kwargs): # noqa: MC0001
+        if not self.instance.is_teacher(request.user):
+            return Response(
+                'Only a teacher can download submissions via this API',
+                status=status.HTTP_403_FORBIDDEN,
+        )
+        exercise = None
+        try:
+            exercise = BaseExercise.objects.get(id=exercise_id)
+        except BaseExercise.DoesNotExist:
+            return Response('Exercise not found', status=status.HTTP_404_NOT_FOUND)
+
+        best = request.query_params.get('best') == 'yes'
+        submissions = Submission.objects.filter(exercise__id=exercise_id).order_by('submission_time')
+
+        def get_group_id(submission):
+            group_id = None
+            if 'group' in submission.meta_data:
+                group_id = submission.meta_data['group']
+            if group_id is None:
+                for lst in submission.submission_data:
+                    if '_aplus_group' in lst:
+                        group_id = lst[1]
+                        break
+            return group_id
+
+        def handle_submission(submission, submitters, info_csv):
+            group_id = None
+            if len(submitters) > 1:
+                group_id = get_group_id(submission)
+                if group_id is not None:
+                    try:
+                        group_id = int(group_id)
+                    except ValueError:
+                        return info_csv
+            submission_time = submission.submission_time.strftime('%Y-%m-%d %H:%M:%S %z')
+            submitted_files = SubmittedFile.objects.filter(submission=submission)
+            student_ids = sorted([submitter.student_id for submitter in submitters])
+            submitters_string = '+'.join(student_ids)
+            submission_num = list(
+                dict.fromkeys( # Remove duplicates
+                    Submission.objects.filter(exercise__id=exercise_id, submitters__in=submitters)
+                    .order_by('submission_time')
+                )
+            ).index(submission) + 1
+            for i, submitted_file in enumerate(submitted_files, start=1):
+                filename = f"{submitters_string}_file{i}_submission{submission_num}"
+                try:
+                    with submitted_file.file_object.file.open('rb') as file:
+                        zip_file.writestr(f'{filename}', file.read())
+                    if group_id is not None:
+                        info_csv += f"{filename},group{group_id},{submission_time}\n"
+                    else:
+                        info_csv += f"{filename},{submitters_string},{submission_time}\n"
+                except OSError:
+                    pass
+            return info_csv
+
+        # Create a zip file in memory
+        zip_buffer = BytesIO()
+        with zipfile.ZipFile(zip_buffer, 'w') as zip_file:
+            info_csv = "filename,label,created_at\n"
+            if best:
+                unique_submitters = []
+                for submission in submissions:
+                    submitters = submission.submitters.all()
+                    if any(self.instance.is_course_staff(submitter.user) for submitter in submitters):
+                        # Skip staff submissions
+                        continue
+                    for submitter in submitters:
+                        if submitter not in unique_submitters:
+                            unique_submitters.append(submitter)
+                unique_submissions = []
+                for submitter in unique_submitters:
+                    submission_entry = ExercisePoints.get(exercise, submitter).best_submission
+                    if submission_entry is None:
+                        continue
+                    submission = submissions.filter(id=submission_entry.id).first()
+                    # Prevent duplicate best submissions due to group submissions
+                    if submission not in unique_submissions:
+                        unique_submissions.append(submission)
+                        info_csv = handle_submission(submission, submission.submitters.all(), info_csv)
+            else:
+                for submission in submissions:
+                    submitters = submission.submitters.all()
+                    if any(self.instance.is_course_staff(submitter.user) for submitter in submitters):
+                        # Skip staff submissions
+                        continue
+                    info_csv = handle_submission(submission, submitters, info_csv)
+            zip_file.writestr('info.csv', info_csv)
+        zip_buffer.seek(0)
+
+        response = FileResponse(zip_buffer, as_attachment=True, filename='submissions.zip')
+        return response
+
     def get_access_mode(self):
         # The API is not supposed to use the access mode permission in views,
         # but this is currently required so that enrollment exercises work in

exercise/staff_views.py

@@ -92,6 +92,7 @@ def get_common_objects(self) -> None:
 
         for submission in qs:
             format_submission(submission, self.pseudonymize)
+        self.has_files = any(len(submission.files.all()) > 0 for submission in qs)
         self.limited = self.request.GET.get('limited', False)
         self.not_all_url = self.exercise.get_submission_list_url() + "?limited=true"
         self.all_url = self.exercise.get_submission_list_url()
@@ -102,7 +103,16 @@ def get_common_objects(self) -> None:
         self.percentage_graded = (
             f"{graded_submitters} / {total_submitters} ({percentage}%)"
         )
-        self.note("limited", "not_all_url", "all_url", "submissions", "default_limit", "count", "percentage_graded")
+        self.note(
+            "has_files",
+            "limited",
+            "not_all_url",
+            "all_url",
+            "submissions",
+            "default_limit",
+            "count",
+            "percentage_graded",
+        )
 
 
 class SubmissionsSummaryView(ExerciseBaseView):
@@ -141,6 +151,8 @@ def get_common_objects(self) -> None:
             .order_by()
         )
 
+        self.has_files = any(len(submission.files.all()) > 0 for submission in self.exercise.submissions.all())
+
         # Get a dict of submitters, accessed by their id.
         profiles = (
             UserProfile.objects
@@ -160,7 +172,7 @@ def get_common_objects(self) -> None:
             if submitter_id is not None:
                 profile = profiles[submitter_id]
                 self.submitters.append({'profile': profile, **submitter_summary})
-        self.note('submitters')
+        self.note('submitters', 'has_files')
 
 
 class InspectSubmitterView(ExerciseBaseView, BaseRedirectView):

exercise/templates/exercise/staff/_submissions_table.html

@@ -27,7 +27,7 @@
 				data-toggle="dropdown"
 				id="download-data"
 			>
-				{% translate "DOWNLOAD" %} <span class="caret"></span>
+				{% translate "DOWNLOAD_POINTS" %} <span class="caret"></span>
 			</button>
 			<ul class="dropdown-menu dropdown-menu-right" aria-labeledby="download-data">
 				{% get_format_info_list "json csv excel.csv" as formats %}
@@ -40,6 +40,28 @@
 				{% endfor %}
 			</ul>
 		</span>
+		{% if has_files %}
+			<span class="dropdown">
+				<button
+					class="aplus-button--secondary aplus-button--xs dropdown-toggle"
+					type="button"
+					data-toggle="dropdown"
+					id="download-zip-data"
+				>
+					{% translate "DOWNLOAD_SUBMISSIONS" %} <span class="caret"></span>
+				</button>
+				<ul class="dropdown-menu dropdown-menu-right" aria-labeledby="download-zip-data">
+					{% get_zip_info_list "all best" as types %}
+					{% for type in types %}
+					<li>
+						<a href="{% url 'api:exercise-submissions-zip' version=2 exercise_id=exercise.id %}?best={{ type.best }}">
+							{{ type.verbose_name }}
+						</a>
+					</li>
+					{% endfor %}
+				</ul>
+			</span>
+		{% endif %}
 	</div>
 	<p>
 		{% exercise_text_stats exercise %} |
@@ -140,4 +162,4 @@
 </table>
 
 {% include "exercise/staff/_regrade_submissions.html" %}
-<script src="{% static 'exercise/filter_submissions_by_tag.js' %}"></script>
+<script src="{% static 'exercise/filter_submissions_by_tag.js' %}"></script>

exercise/templates/exercise/staff/_submitters_table.html

@@ -26,7 +26,7 @@
 				data-toggle="dropdown"
 				id="download-data"
 			>
-				{% translate "DOWNLOAD" %} <span class="caret"></span>
+				{% translate "DOWNLOAD_POINTS" %} <span class="caret"></span>
 			</button>
 			<ul class="dropdown-menu dropdown-menu-right" aria-labeledby="download-data">
 				{% get_format_info_list "json csv excel.csv" as formats %}
@@ -39,6 +39,28 @@
 				{% endfor %}
 			</ul>
 		</span>
+		{% if has_files %}
+			<span class="dropdown">
+				<button
+					class="aplus-button--secondary aplus-button--xs dropdown-toggle"
+					type="button"
+					data-toggle="dropdown"
+					id="download-zip-data"
+				>
+					{% translate "DOWNLOAD_SUBMISSIONS" %} <span class="caret"></span>
+				</button>
+				<ul class="dropdown-menu dropdown-menu-right" aria-labeledby="download-zip-data">
+					{% get_zip_info_list "all best" as types %}
+					{% for type in types %}
+					<li>
+						<a href="{% url 'api:exercise-submissions-zip' version=2 exercise_id=exercise.id %}?best={{ type.best }}">
+							{{ type.verbose_name }}
+						</a>
+					</li>
+					{% endfor %}
+				</ul>
+			</span>
+		{% endif %}
 	</div>
 	<p>
 		{% exercise_text_stats exercise %}

exercise/templatetags/exercise.py

@@ -348,6 +348,30 @@ def get_format_info(format): # pylint: disable=redefined-builtin
 def get_format_info_list(formats):
     return [get_format_info(format) for format in formats.split()]
 
+
+@register.simple_tag
+def get_zip_info(type): # pylint: disable=redefined-builtin
+    zip_infos = {
+        'all' : {
+            'best': 'no',
+            'verbose_name': _('ALL_SUBMISSIONS'),
+        },
+        'best': {
+            'best': 'yes',
+            'verbose_name': _('BEST_SUBMISSIONS'),
+        },
+    }
+    try:
+        return zip_infos[type]
+    except KeyError as e:
+        raise RuntimeError('Invalid zip type: \'{}\''.format(type)) from e
+
+
+@register.simple_tag
+def get_zip_info_list(types):
+    return [get_zip_info(type) for type in types.split()]
+
+
 @register.simple_tag
 def get_regrade_info(index):
     regrade_infos = {