[BD-27] Create transcript nodes in video blocks

src/cc2olx/iframe_link_parser.py

@@ -99,14 +99,24 @@ def _create_video_olx(self, doc, url_row):
         attributes = {}
         edx_id = url_row.get("Edx Id", "")
         youtube_id = url_row.get("Youtube Id", "")
+        languages = url_row.get("Languages", "")
         if edx_id.strip() != "":
             attributes["edx_video_id"] = edx_id
         elif youtube_id.strip() != "":
             attributes["youtube"] = "1.00:" + youtube_id
             attributes["youtube_id_1_0"] = youtube_id
         else:
             raise IframeLinkParserError("Missing Edx Id or Youtube Id for video conversion.")
-        child = xml_element("video", children=None, attributes=attributes)
+
+        children = []
+
+        # For rows that contain languages generate transcript nodes
+        if languages != "":
+            for lang in languages.split("-"):
+                src = f"{edx_id}-{lang}.srt"
+                transcript = xml_element("transcript", children=None, attributes={"language": lang, "src": src})
+                children.append(transcript)
+        child = xml_element("video", children=children, attributes=attributes)
         return child
 
 

src/cc2olx/tools/README.rst

@@ -4,9 +4,9 @@ Tools
 .. _video_upload_tool:
 
 Video Upload Tool
------------------
+=================
 
-The video upload tool uploads video files to edX's video encoding pipeline.
+The video upload tool uploads video files and associated transcripts to edX's video encoding pipeline.
 It also generates an output CSV that can be used as input to the video xBlock conversion tool.
 
 Input
@@ -16,8 +16,11 @@ The tool also optionally takes as a keyword argument an ``--output-csv``.
 
 The ``course-id`` argument is the ID of the course as it appears in Studio. For example, ``course-v1:edX+111222+111222``.
 
-The ``directory`` argument is a directory containing video files that will be uploaded to edX's video encoding pipeline.
+The ``directory`` argument is a directory (such as the one created by the video download tool) containing video files and transcripts, that will be uploaded to edX's video encoding pipeline.
 The currently supported formats for the videos are ``mp4`` and ``mov``, which are the files supported by the edX video encoding pipeline.
+The tool will recurse into the provided directory and files at any depth, and matching the supported formats will be uploaded.
+The only supported format for transcripts is ``srt``, and transcript files must have the same location and base name as their associated video, apart from the suffix.
+For example for a video file named video_name.mp4 the tool will find transcripts in the same directory, named like video_name.XX.srt where XX is a language code.
 Note that files of other types will be ignored by the tool.
 
 The ``input-csv`` argument is a file that is meant to establish a relationship between a particular video, its externally hosted URL, and the edX video ID that is generated by the video encoding pipeline when the video is uploaded.
@@ -36,7 +39,10 @@ Output
 ------
 The tool will generate a CSV file. If the optional command line argument ``--output-csv`` is supplied, then the output CSV will be saved to that path. Otherwise, if the command line argument is not supplied, then the file will be saved to the same directory as the input file as specified by the ``input-csv`` command line argument, and the name of the CSV fill will be the name of the original CSV file supplied as a command line argument with "upload-results" appended to the end of the name, i.e. ``<original-filename>-upload-results.csv``.
 
-A new column, "Edx Id", which represents the edX Video ID for each video, will be appended to the end of the file.
+Two new columns will be appended to the end of the file:
+* "Edx Id", which represents the edX Video ID for each video
+* "Languages", a list of dash separated, alphabetically ordered language codes for which transcripts were uploaded for each video.
+
 Otherwise, the file is identical to the file supplied as a command line argument.
 
 Logs
@@ -72,3 +78,38 @@ Once these are done, you can run the tool as follows::
 Or, if you would like to specify the path to the output CSV file::
     python src/cc2olx/tools/video_upload.py course-v1:edX+111222+111222 /Users/example/workspaces/videos /Users/example/workspaces/video-data.csv --output-csv /Users/example/workspaces/video-data-output.csv
 
+.. _video_download_tool:
+
+Video Download Tool
+===================
+The download tool will accept a common cartridge format course, and search for matching videos to download, in preparation for use with the video upload tool.
+It also generates a CSV in a format that can be used by the video upload tool for inclusion in the edX video pipeline (by default "out.csv").
+
+Input
+-----
+The tool has one required parameter ``-i`` or ``--input``, which is the Common Cartridge Course (IMSCC) format archive, or single HTML file. This is searched for iframes with video links.
+The tool will download the raw video for any embedded video links, as well as associated transcripts, and store these in the `downloads` folder in the current working directory.
+
+The ``--output`` argument allows for specifying an alternative filename to store the CSV metadata generated for downloaded videos.
+
+The ``--downloads`` argument will specify an alternative directory for storing downloaded video data.
+
+The ``--simulate`` argument allows for a dry-run of parsing and extracting URLs for download, without actually downloading the video content.
+
+The ``--config`` argument allows for providing extra configuration to youtube-dl which is used to handle video downloading.
+
+Output
+------
+Unless otherwise specified with the options above, the tool will generate a file ``out.csv`` containing the URL to the video, file path of the downloaded video, and a YouTube ID if the video was originally hosted on YouTube.
+The tool will also create a `downloads` directory with the raw videos downloaded, and any transcripts or subtitles associated with the videos.
+
+Use
+---
+The video download tool is a basic command-line Python 3 program.
+This can be run simply against a Common Cartridge Format Course (IMSCC) file as so::
+
+    python src/cc2olx/tools/video_download.py -i ~/ushistory.imscc
+
+This tool is most useful by chaining it with the video upload tool, in which case the video upload tool can be run in the same directory after download is complete like so::
+
+    python src/cc2olx/tools/video_upload.py course-v1:edX+111222+111222 ./downloads ./out.csv

src/cc2olx/tools/video_upload.py

@@ -8,8 +8,11 @@
 
 OAUTH_TOKEN_URL = "https://courses.edx.org/oauth2/access_token"
 GENERATE_UPLOAD_LINK_BASE_URL = "https://studio.edx.org/generate_video_upload_link/"
+TRANSCRIPT_UPLOAD_LINK = "https://studio.edx.org/transcript_upload_api/"
 # OAUTH_TOKEN_URL = "https://courses.stage.edx.org/oauth2/access_token"
 # GENERATE_UPLOAD_LINK_BASE_URL = "https://studio.stage.edx.org/generate_video_upload_link/"
+# TRANSCRIPT_UPLOAD_LINK = "https://studio.stage.edx.org/transcript_upload_api/"
+
 VIDEO_EXTENSION_CONTENT_TYPES = {
     ".mp4": "video/mp4",
     ".mov": "video/quicktime",
@@ -55,7 +58,7 @@ def get_access_token():
     return data["access_token"]
 
 
-def parse_args():
+def parse_args(args=None):
     """Set up and return command line arguments for the video upload tool."""
     parser = argparse.ArgumentParser(description="Upload video files to edX via Studio's video encoding pipeline.")
     parser.add_argument(
@@ -74,9 +77,10 @@ def parse_args():
     )
     parser.add_argument(
         "--output-csv",
+        "-o",
         help="path to where the output CSV should be stored; this will overwrite existing files",
     )
-    return parser.parse_args()
+    return parser.parse_args(args)
 
 
 def make_generate_upload_link_request(url, data, filename, access_token):
@@ -123,6 +127,42 @@ def make_generate_upload_link_request(url, data, filename, access_token):
     return response
 
 
+def upload_transcript(filename, edx_video_id, language_code, access_token):
+    """
+    Make a POST request against the Studio upload transcript API and return the
+    response. If errors occur during the API call, log to the console.
+
+    Arguments:
+        * filename: the transcript filename
+        * edx_video_id: the video ID of the video this transcript is for
+        * language_code: the language of the transcript
+        * access_token: access token to be able to make authenticated calls to the Studio API
+
+    Returns:
+        * response: the response object from the POST API call
+    """
+    s = requests.Session()
+    s.auth = SuppliedJwtAuth(access_token)
+
+    data = {"edx_video_id": edx_video_id, "language_code": language_code, "new_language_code": language_code}
+    files = {"file": open(filename, "rb")}
+
+    try:
+        response = s.post(TRANSCRIPT_UPLOAD_LINK, data=data, files=files)
+        response.raise_for_status()
+    except requests.exceptions.HTTPError as error:
+        print(
+            "An HTTP error occurred calling the Studio transcript upload link API "
+            "for transcript: {}: {}".format(filename, repr(error))
+        )
+    if response.status_code == 201:
+        print(f"Successfully uploaded transcript {filename}.")
+    else:
+        print(f"Transcript {filename} was unable to be uploaded.")
+
+    return response
+
+
 def make_upload_video_request(url, data, headers, filename):
     """
     Make a PUT request against the AWS upload video API.
@@ -185,6 +225,7 @@ def write_upload_results_csv(input_csv_path, output_csv_path, file_data):
 
         new_fieldnames = reader.fieldnames.copy()
         new_fieldnames.append("Edx Id")
+        new_fieldnames.append("Languages")
 
         writer = csv.DictWriter(output_csv, new_fieldnames)
         writer.writeheader()
@@ -199,6 +240,7 @@ def write_upload_results_csv(input_csv_path, output_csv_path, file_data):
 
             new_row = row.copy()
             new_row["Edx Id"] = data["edx_video_id"]
+            new_row["Languages"] = data["lang"]
 
             writer.writerow(new_row)
 
@@ -260,6 +302,15 @@ def main():
                         make_upload_video_request(upload_url, data, headers, filename)
 
                 files_data[str(relative_path)] = {"edx_video_id": edx_video_id}
+                langs = []
+
+                # Look for files with the same name as our video but with a ${LANG}.srt suffix
+                for srt_path in sorted(full_path.parent.glob(full_path.stem + "*.srt")):
+                    lang = srt_path.suffixes[0][1:]
+                    langs.append(lang)
+                    upload_transcript(srt_path, edx_video_id, lang, access_token)
+
+                files_data[str(relative_path)]["lang"] = "-".join(langs)
 
     input_csv_path = Path(args.input_csv)
 

tests/conftest.py

@@ -127,6 +127,21 @@ def link_map_csv(fixtures_data_dir):
     return link_map_csv_file_path
 
 
+@pytest.fixture(scope="session")
+def link_map_languages_csv(fixtures_data_dir):
+    """
+        This fixture helps to provide csv file path with transcript languages included
+    Args:
+        fixtures_data_dir ([str]): Path to the directory where fixture data is present.
+
+    Returns:
+        [str]: Path to the csv
+    """
+
+    link_map_csv_file_path = str(fixtures_data_dir / "link_map_languages.csv")
+    return link_map_csv_file_path
+
+
 @pytest.fixture(scope="session")
 def link_map_edx_only_csv(fixtures_data_dir):
     """
@@ -219,3 +234,11 @@ def bad_passports_csv(fixtures_data_dir):
     """
     bad_passports_csv = str(fixtures_data_dir / "bad_passports.csv")
     return bad_passports_csv
+
+
+@pytest.fixture(scope="session")
+def transcript_file(fixtures_data_dir):
+    transcript_file_path = str(
+        fixtures_data_dir / "video_files/01___Intro_to_Knowledge_Based_AI/0 - Introductions.en.srt"
+    )
+    return transcript_file_path

tests/fixtures_data/link_map_languages.csv

@@ -0,0 +1,6 @@
+External Video Link,Edx Id,Youtube Id,Languages
+https://cdnapisec.kaltura.com/p/2019031/sp/201903100/playManifest/entryId/1_zeqnrfgw/format/url/protocol/https,42d2a5e2-bced-45d6-b8dc-2f5901c9fdd0,onRUvL2SBG8,en-fr
+https://cdnapisec.kaltura.com/p/2019031/sp/201903100/playManifest/entryId/1_9if7cth0/format/url/protocol/https,42d2a5e2-bced-45d6-b8dc-2f5901c9fdd1,NXlG00JYX-o,en
+https://cdnapisec.kaltura.com/p/2019031/sp/201903100/playManifest/entryId/1_c7j5va21/format/url/protocol/https,42d2a5e2-bced-45d6-b8dc-2f5901c9fdd2,_SIvUj7xUKc,fr
+https://cdnapisec.kaltura.com/p/2019031/sp/201903100/playManifest/entryId/1_xcjzc0q5/format/url/protocol/https,42d2a5e2-bced-45d6-b8dc-2f5901c9fdd3,3pT8dh4ftbc
+https://cdnapisec.kaltura.com/p/2019031/sp/201903100/playManifest/entryId/1_t1jur7p3/format/url/protocol/https,42d2a5e2-bced-45d6-b8dc-2f5901c9fdd4,uwuZKEKpq8k,de-en-fr

tests/test_kaltura_link_parser.py

@@ -152,3 +152,22 @@ def test_video_olx_bad_link_map(self, iframes, link_map_bad_csv):
 
         with pytest.raises(Exception):
             video_olx, _ = iframe_link_parser.get_video_olx(doc, iframes)
+
+    def test_video_olx_languages(self, iframes, link_map_languages_csv):
+        """
+        Test that video olx is generated and produced when transcript languages are provided
+        """
+        iframe_link_parser = KalturaIframeLinkParser(link_map_languages_csv)
+        doc = xml.dom.minidom.Document()
+        video_olx, _ = iframe_link_parser.get_video_olx(doc, iframes)
+
+        assert len(video_olx) == 1
+
+        actual_video_olx = video_olx[0]
+
+        # The first line in the fixtures file has two languages listed
+        assert len(actual_video_olx.childNodes) == 2
+
+        assert actual_video_olx.firstChild.nodeName == "transcript"
+        assert actual_video_olx.firstChild.hasAttribute("language")
+        assert actual_video_olx.firstChild.hasAttribute("src")

tests/test_tools.py

@@ -1,6 +1,14 @@
+from argparse import Namespace
 from unittest.mock import Mock, call
 
-from cc2olx.tools.video_upload import main, OAUTH_TOKEN_URL, GENERATE_UPLOAD_LINK_BASE_URL
+from cc2olx.tools.video_upload import (
+    main,
+    parse_args,
+    upload_transcript,
+    OAUTH_TOKEN_URL,
+    GENERATE_UPLOAD_LINK_BASE_URL,
+    TRANSCRIPT_UPLOAD_LINK,
+)
 
 MOCK_UPLOAD_LINK_ROOT = "example.com/upload"
 
@@ -35,8 +43,12 @@ def post_side_effect(*args, **kwargs):
         }
         mock.json.return_value = json
         return mock
+    elif args[0] == TRANSCRIPT_UPLOAD_LINK:
+        mock = Mock()
+        mock.status_code = 201
+        return mock
     else:
-        return mock()
+        return Mock()
 
 
 def put_side_effect(*args, **kwargs):
@@ -86,6 +98,7 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args):
                     "Additional Notes": "Additional Notes",
                     "Youtube ID": "Youtube ID",
                     "External Video Link": "External Video Link",
+                    "Languages": "Languages",
                 }
             ),
             call(
@@ -96,6 +109,7 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args):
                     "Relative File Path": "01___Intro_to_Knowledge_Based_AI/0 - Introductions.mp4",
                     "Additional Notes": "This is the first video.",
                     "Youtube ID": "onRUvL2SBG8",
+                    "Languages": "en",
                 }
             ),
             call(
@@ -106,6 +120,7 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args):
                     "Relative File Path": "01___Intro_to_Knowledge_Based_AI/1 - Preview.mp4",
                     "Additional Notes": "",
                     "Youtube ID": "NXlG00JYX-o",
+                    "Languages": "en-fr",
                 }
             ),
             call(
@@ -116,6 +131,7 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args):
                     "Relative File Path": "01___Intro_to_Knowledge_Based_AI/2 - Conundrums in AI.mov",
                     "Additional Notes": "",
                     "Youtube ID": "_SIvUj7xUKc",
+                    "Languages": "fr",
                 }
             ),
             call(
@@ -126,7 +142,46 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args):
                     "Relative File Path": "01___Intro_to_Knowledge_Based_AI/3 - Characteristics of AI Problems.mov",
                     "Additional Notes": "This is the last video.",
                     "Youtube ID": "3pT8dh4ftbc",
+                    "Languages": "",
                 }
             ),
         ]
         csv_writerow_mock.assert_has_calls(expected_csv_writerow_call_args, any_order=True)
+
+
+def test_parse_args():
+    """
+    Basic cli test.
+    """
+
+    parsed_args = parse_args(["courseid", "dirname", "input.csv", "--output-csv", "output.csv"])
+
+    assert parsed_args == Namespace(
+        course_id="courseid", directory="dirname", input_csv="input.csv", output_csv="output.csv"
+    )
+
+
+def upload_transcript_side_effect(*args, **kwargs):
+    print(kwargs["data"])
+    mock = Mock()
+
+    if kwargs["data"].get("edx_video_id") and kwargs["data"].get("language_code"):
+        mock.status_code = 201
+    else:
+        mock.status_code = 400
+
+    return mock
+
+
+def test_transcript_upload(mocker, transcript_file):
+    mocker.patch("cc2olx.tools.video_upload.requests.post", side_effect=upload_transcript_side_effect)
+    mocker.patch("cc2olx.tools.video_upload.requests.Session.post", side_effect=upload_transcript_side_effect)
+
+    response = upload_transcript(transcript_file, "edxid", "en", "test_access_token")
+    assert response.status_code == 201
+
+    response = upload_transcript(transcript_file, "edxid", None, "test_access_token")
+    assert response.status_code == 400
+
+    response = upload_transcript(transcript_file, None, "en", "test_access_token")
+    assert response.status_code == 400