[BD-05] [TNL-7308] Add ORA Zipped File Download for Submission Text + Attached Files

lms/djangoapps/instructor/tests/test_api.py

@@ -149,6 +149,7 @@
     'calculate_grades_csv',
     'change_due_date',
     'export_ora2_data',
+    'export_ora2_submission_files',
     'get_grading_config',
     'get_problem_responses',
     'get_proctored_exam_results',
@@ -428,6 +429,7 @@ def setUp(self):
             ('get_proctored_exam_results', {}),
             ('get_problem_responses', {}),
             ('export_ora2_data', {}),
+            ('export_ora2_submission_files', {}),
             ('rescore_problem',
              {'problem_to_reset': self.problem_urlname, 'unique_student_identifier': self.user.email}),
             ('override_problem_score',
@@ -2875,6 +2877,32 @@ def test_get_ora2_responses_already_running(self):
 
         self.assertContains(response, already_running_status, status_code=400)
 
+    def test_get_ora2_submission_files_success(self):
+        url = reverse('export_ora2_submission_files', kwargs={'course_id': text_type(self.course.id)})
+
+        with patch(
+            'lms.djangoapps.instructor_task.api.submit_export_ora2_submission_files'
+        ) as mock_submit_ora2_task:
+            mock_submit_ora2_task.return_value = True
+            response = self.client.post(url, {})
+
+        success_status = 'Attachments archive is being created.'
+
+        self.assertContains(response, success_status)
+
+    def test_get_ora2_submission_files_already_running(self):
+        url = reverse('export_ora2_submission_files', kwargs={'course_id': text_type(self.course.id)})
+        task_type = 'export_ora2_submission_files'
+        already_running_status = generate_already_running_error_message(task_type)
+
+        with patch(
+            'lms.djangoapps.instructor_task.api.submit_export_ora2_submission_files'
+        ) as mock_submit_ora2_task:
+            mock_submit_ora2_task.side_effect = AlreadyRunningError(already_running_status)
+            response = self.client.post(url, {})
+
+        self.assertContains(response, already_running_status, status_code=400)
+
     def test_get_student_progress_url(self):
         """ Test that progress_url is in the successful response. """
         url = reverse('get_student_progress_url', kwargs={'course_id': text_type(self.course.id)})

lms/djangoapps/instructor/views/api.py

@@ -2049,6 +2049,28 @@ def export_ora2_data(request, course_id):
     return JsonResponse({"status": success_status})
 
 
+@transaction.non_atomic_requests
+@require_POST
+@ensure_csrf_cookie
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+@require_course_permission(permissions.CAN_RESEARCH)
+@common_exceptions_400
+def export_ora2_submission_files(request, course_id):
+    """
+    Pushes a Celery task which will download and compress all submission
+    files (texts, attachments) into a zip archive.
+    """
+    course_key = CourseKey.from_string(course_id)
+
+    task_api.submit_export_ora2_submission_files(request, course_key)
+
+    return JsonResponse({
+        "status": _(
+            "Attachments archive is being created."
+        )
+    })
+
+
 @transaction.non_atomic_requests
 @require_POST
 @ensure_csrf_cookie

lms/djangoapps/instructor/views/api_urls.py

@@ -54,6 +54,9 @@
     url(r'^get_course_survey_results$', api.get_course_survey_results, name='get_course_survey_results'),
     url(r'^export_ora2_data', api.export_ora2_data, name='export_ora2_data'),
 
+    url(r'^export_ora2_submission_files', api.export_ora2_submission_files,
+        name='export_ora2_submission_files'),
+
     # spoc gradebook
     url(r'^gradebook$', gradebook_api.spoc_gradebook, name='spoc_gradebook'),
 

lms/djangoapps/instructor/views/instructor_dashboard.py

@@ -628,6 +628,9 @@ def _section_data_download(course, access):
             'get_course_survey_results', kwargs={'course_id': six.text_type(course_key)}
         ),
         'export_ora2_data_url': reverse('export_ora2_data', kwargs={'course_id': six.text_type(course_key)}),
+        'export_ora2_submission_files_url': reverse(
+            'export_ora2_submission_files', kwargs={'course_id': six.text_type(course_key)}
+        ),
     }
     if not access.get('data_researcher'):
         section_data['is_hidden'] = True

lms/djangoapps/instructor_task/api.py

@@ -35,6 +35,7 @@
     course_survey_report_csv,
     delete_problem_state,
     export_ora2_data,
+    export_ora2_submission_files,
     generate_certificates,
     override_problem_score,
     proctored_exam_results_csv,
@@ -450,6 +451,19 @@ def submit_export_ora2_data(request, course_key):
     return submit_task(request, task_type, task_class, course_key, task_input, task_key)
 
 
+def submit_export_ora2_submission_files(request, course_key):
+    """
+    Submits a task to download and compress all submissions
+    files (texts, attachments) for given course.
+    """
+    task_type = 'export_ora2_submission_files'
+    task_class = export_ora2_submission_files
+    task_input = {}
+    task_key = ''
+
+    return submit_task(request, task_type, task_class, course_key, task_input, task_key)
+
+
 def generate_certificates_for_students(request, course_key, student_set=None, specific_student_id=None):
     """
     Submits a task to generate certificates for given students enrolled in the course.

lms/djangoapps/instructor_task/models.py

@@ -280,9 +280,14 @@ def store(self, course_id, filename, buff):
         """
         path = self.path_to(course_id, filename)
         # See https://github.com/boto/boto/issues/2868
-        # Boto doesn't play nice with unicod in python3
+        # Boto doesn't play nice with unicode in python3
         if not six.PY2:
-            buff = ContentFile(buff.read().encode('utf-8'))
+            buff_contents = buff.read()
+
+            if not isinstance(buff_contents, bytes):
+                buff_contents = buff_contents.encode('utf-8')
+
+            buff = ContentFile(buff_contents)
 
         self.storage.save(path, buff)
 

lms/djangoapps/instructor_task/tasks.py

@@ -40,6 +40,7 @@
     cohort_students_and_upload,
     upload_course_survey_report,
     upload_ora2_data,
+    upload_ora2_submission_files,
     upload_proctored_exam_results_report
 )
 from lms.djangoapps.instructor_task.tasks_helper.module_state import (
@@ -292,3 +293,14 @@ def export_ora2_data(entry_id, xmodule_instance_args):
     action_name = ugettext_noop('generated')
     task_fn = partial(upload_ora2_data, xmodule_instance_args)
     return run_main_task(entry_id, task_fn, action_name)
+
+
+@task(base=BaseInstructorTask)
+def export_ora2_submission_files(entry_id, xmodule_instance_args):
+    """
+    Download all submission files, generate csv downloads list,
+    put all this into zip archive and push it to S3.
+    """
+    action_name = ugettext_noop('compressed')
+    task_fn = partial(upload_ora2_submission_files, xmodule_instance_args)
+    return run_main_task(entry_id, task_fn, action_name)

lms/djangoapps/instructor_task/tasks_helper/misc.py

@@ -7,16 +7,21 @@
 
 import logging
 from collections import OrderedDict
+from contextlib import contextmanager
 from datetime import datetime
+from io import StringIO
+from tempfile import TemporaryFile
 from time import time
+from zipfile import ZipFile
 import csv
+import os
 import unicodecsv
 import six
 
 from django.contrib.auth.models import User
 from django.core.exceptions import ValidationError
 from django.core.files.storage import DefaultStorage
-from openassessment.data import OraAggregateData
+from openassessment.data import OraAggregateData, OraDownloadData
 from pytz import UTC
 
 from lms.djangoapps.instructor_analytics.basic import get_proctored_exam_results
@@ -27,7 +32,12 @@
 from util.file import UniversalNewlineIterator
 
 from .runner import TaskProgress
-from .utils import UPDATE_STATUS_FAILED, UPDATE_STATUS_SUCCEEDED, upload_csv_to_report_store
+from .utils import (
+    UPDATE_STATUS_FAILED,
+    UPDATE_STATUS_SUCCEEDED,
+    upload_csv_to_report_store,
+    upload_zip_to_report_store,
+)
 
 # define different loggers for use within tasks and on client side
 TASK_LOG = logging.getLogger('edx.celery.task')
@@ -340,3 +350,108 @@ def upload_ora2_data(
     TASK_LOG.info(u'%s, Task type: %s, Upload complete.', task_info_string, action_name)
 
     return UPDATE_STATUS_SUCCEEDED
+
+
+def _task_step(task_progress, task_info_string, action_name):
+    """
+    Returns a context manager, that logs error and updates TaskProgress
+    filures counter in case inner block throws an exception.
+    """
+
+    @contextmanager
+    def _step_context_manager(step_description, exception_text, step_error_description):
+        curr_step = {'step': step_description}
+        TASK_LOG.info(
+            '%s, Task type: %s, Current step: %s',
+            task_info_string,
+            action_name,
+            curr_step,
+        )
+
+        task_progress.update_task_state(extra_meta=curr_step)
+
+        try:
+            yield
+
+        # Update progress to failed regardless of error type
+        except Exception:  # pylint: disable=broad-except
+            TASK_LOG.exception(exception_text)
+            task_progress.failed = 1
+
+            task_progress.update_task_state(extra_meta={'step': step_error_description})
+
+    return _step_context_manager
+
+
+def upload_ora2_submission_files(
+    _xmodule_instance_args, _entry_id, course_id, _task_input, action_name
+):
+    """
+    Creates zip archive with submission files in three steps:
+
+    1. Collect all files information using ORA download helper.
+    2. Download all submission attachments, put them in temporary zip
+        file along with submission texts and csv downloads list.
+    3. Upload zip file into reports storage.
+    """
+
+    start_time = time()
+    start_date = datetime.now(UTC)
+
+    num_attempted = 1
+    num_total = 1
+
+    fmt = 'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}'
+    task_info_string = fmt.format(
+        task_id=_xmodule_instance_args.get('task_id') if _xmodule_instance_args is not None else None,
+        entry_id=_entry_id,
+        course_id=course_id,
+        task_input=_task_input
+    )
+    TASK_LOG.info(u'%s, Task type: %s, Starting task execution', task_info_string, action_name)
+
+    task_progress = TaskProgress(action_name, num_total, start_time)
+    task_progress.attempted = num_attempted
+
+    step_manager = _task_step(task_progress, task_info_string, action_name)
+
+    submission_files_data = None
+    with step_manager(
+        'Collecting attachments data',
+        'Failed to get ORA submissions attachments data.',
+        'Error while collecting data',
+    ):
+        submission_files_data = OraDownloadData.collect_ora2_submission_files(course_id)
+
+    if submission_files_data is None:
+        return UPDATE_STATUS_FAILED
+
+    with TemporaryFile('rb+') as zip_file:
+        compressed = None
+        with step_manager(
+            'Downloading and compressing attachments files',
+            'Failed to download and compress submissions attachments.',
+            'Error while downloading and compressing submissions attachments',
+        ):
+            compressed = OraDownloadData.create_zip_with_attachments(zip_file, course_id, submission_files_data)
+
+        if compressed is None:
+            return UPDATE_STATUS_FAILED
+
+        zip_filename = None
+        with step_manager(
+            'Uploading zip file to storage',
+            'Failed to upload zip file to storage.',
+            'Error while uploading zip file to storage',
+        ):
+            zip_filename = upload_zip_to_report_store(zip_file, 'submission_files', course_id, start_date),
+
+        if not zip_filename:
+            return UPDATE_STATUS_FAILED
+
+    task_progress.succeeded = 1
+    curr_step = {'step': 'Finalizing attachments extracting'}
+    task_progress.update_task_state(extra_meta=curr_step)
+    TASK_LOG.info(u'%s, Task type: %s, Upload complete.', task_info_string, action_name)
+
+    return UPDATE_STATUS_SUCCEEDED

lms/djangoapps/instructor_task/tasks_helper/utils.py

@@ -49,6 +49,23 @@ def upload_csv_to_report_store(rows, csv_name, course_id, timestamp, config_name
     return report_name, report_path
 
 
+def upload_zip_to_report_store(file, zip_name, course_id, timestamp, config_name='GRADES_DOWNLOAD'):
+    """
+    Upload given file buffer as a zip file using ReportStore.
+    """
+    report_store = ReportStore.from_config(config_name)
+
+    report_name = u"{course_prefix}_{zip_name}_{timestamp_str}.zip".format(
+        course_prefix=course_filename_prefix_generator(course_id),
+        zip_name=zip_name,
+        timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M")
+    )
+
+    report_store.store(course_id, report_name, file)
+    tracker_emit(zip_name)
+    return report_name
+
+
 def tracker_emit(report_name):
     """
     Emits a 'report.requested' event for the given report.

lms/djangoapps/instructor_task/tests/test_api.py

@@ -27,6 +27,7 @@
     submit_delete_entrance_exam_state_for_student,
     submit_delete_problem_state_for_all_students,
     submit_export_ora2_data,
+    submit_export_ora2_submission_files,
     submit_override_score,
     submit_rescore_entrance_exam_for_student,
     submit_rescore_problem_for_all_students,
@@ -36,7 +37,7 @@
 )
 from lms.djangoapps.instructor_task.api_helper import AlreadyRunningError, QueueConnectionError
 from lms.djangoapps.instructor_task.models import PROGRESS, InstructorTask
-from lms.djangoapps.instructor_task.tasks import export_ora2_data
+from lms.djangoapps.instructor_task.tasks import export_ora2_data, export_ora2_submission_files
 from lms.djangoapps.instructor_task.tests.test_base import (
     TEST_COURSE_KEY,
     InstructorTaskCourseTestCase,
@@ -282,6 +283,22 @@ def test_submit_ora2_request_task(self):
             mock_submit_task.assert_called_once_with(
                 request, 'export_ora2_data', export_ora2_data, self.course.id, {}, '')
 
+    def test_submit_export_ora2_submission_files(self):
+        request = self.create_task_request(self.instructor)
+
+        with patch('lms.djangoapps.instructor_task.api.submit_task') as mock_submit_task:
+            mock_submit_task.return_value = MagicMock()
+            submit_export_ora2_submission_files(request, self.course.id)
+
+            mock_submit_task.assert_called_once_with(
+                request,
+                'export_ora2_submission_files',
+                export_ora2_submission_files,
+                self.course.id,
+                {},
+                ''
+            )
+
     def test_submit_generate_certs_students(self):
         """
         Tests certificates generation task submission api