From 2b7b03dbe130961a504f38551c6dfe8b3159b37e Mon Sep 17 00:00:00 2001 From: Jani Monoses Date: Mon, 17 May 2021 10:35:49 +0300 Subject: [PATCH 01/13] Upload video transcripts if found --- src/cc2olx/tools/video_upload.py | 40 +++++++++++++++++++ .../0 - Introductions.en.srt | 0 .../1 - Preview.en.srt | 0 .../1 - Preview.fr.srt | 0 .../2 - Conundrums in AI.fr.srt | 0 tests/test_tools.py | 6 ++- 6 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/0 - Introductions.en.srt create mode 100644 tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/1 - Preview.en.srt create mode 100644 tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/1 - Preview.fr.srt create mode 100644 tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/2 - Conundrums in AI.fr.srt diff --git a/src/cc2olx/tools/video_upload.py b/src/cc2olx/tools/video_upload.py index 2ce90297..587c5f72 100644 --- a/src/cc2olx/tools/video_upload.py +++ b/src/cc2olx/tools/video_upload.py @@ -8,8 +8,11 @@ OAUTH_TOKEN_URL = "https://courses.edx.org/oauth2/access_token" GENERATE_UPLOAD_LINK_BASE_URL = "https://studio.edx.org/generate_video_upload_link/" +TRANSCRIPT_UPLOAD_LINK = "https://studio.edx.org/transcript_upload/" # OAUTH_TOKEN_URL = "https://courses.stage.edx.org/oauth2/access_token" # GENERATE_UPLOAD_LINK_BASE_URL = "https://studio.stage.edx.org/generate_video_upload_link/" +# TRANSCRIPT_UPLOAD_LINK = "https://studio.stage.edx.org/transcript_upload/" + VIDEO_EXTENSION_CONTENT_TYPES = { ".mp4": "video/mp4", ".mov": "video/quicktime", @@ -123,6 +126,38 @@ def make_generate_upload_link_request(url, data, filename, access_token): return response +def upload_transcript(filename, edx_video_id, language_code): + """ + Make a POST request against the Studio upload transcript API and return the + response. If errors occur during the API call, log to the console. + + Arguments: + * filename: the transcript filename + * edx_video_id: the video ID of the video this transcript is for + * language_code: the language of the transcript + + Returns: + * response: the response object from the POST API call + """ + data = {"edx_video_id": edx_video_id, "language_code": language_code, "new_language_code": language_code} + files = {"file": open(filename, "rb")} + + try: + response = requests.post(TRANSCRIPT_UPLOAD_LINK, data=data, files=files) + response.raise_for_status() + except requests.exceptions.HTTPError as error: + print( + "An HTTP error occurred calling the Studio transcript upload link API " + "for transcript: {}: {}".format(filename, repr(error)) + ) + if response.status_code == 201: + print(f"Successfully uploaded transcript {filename}.") + else: + print(f"Transcript {filename} was unable to be uploaded.") + + return response + + def make_upload_video_request(url, data, headers, filename): """ Make a PUT request against the AWS upload video API. @@ -261,6 +296,11 @@ def main(): files_data[str(relative_path)] = {"edx_video_id": edx_video_id} + # Look for files with the same name as our video but with a ${LANG}.srt suffix + for srt_path in full_path.parent.glob(full_path.stem + "*.srt"): + lang = srt_path.suffixes[0][1:] + upload_transcript(srt_path, edx_video_id, lang) + input_csv_path = Path(args.input_csv) output_csv_path = args.output_csv diff --git a/tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/0 - Introductions.en.srt b/tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/0 - Introductions.en.srt new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/1 - Preview.en.srt b/tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/1 - Preview.en.srt new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/1 - Preview.fr.srt b/tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/1 - Preview.fr.srt new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/2 - Conundrums in AI.fr.srt b/tests/fixtures_data/video_files/01___Intro_to_Knowledge_Based_AI/2 - Conundrums in AI.fr.srt new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_tools.py b/tests/test_tools.py index 03a0cf51..b8b14ba9 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,6 +1,6 @@ from unittest.mock import Mock, call -from cc2olx.tools.video_upload import main, OAUTH_TOKEN_URL, GENERATE_UPLOAD_LINK_BASE_URL +from cc2olx.tools.video_upload import main, OAUTH_TOKEN_URL, GENERATE_UPLOAD_LINK_BASE_URL, TRANSCRIPT_UPLOAD_LINK MOCK_UPLOAD_LINK_ROOT = "example.com/upload" @@ -35,6 +35,10 @@ def post_side_effect(*args, **kwargs): } mock.json.return_value = json return mock + elif args[0] == TRANSCRIPT_UPLOAD_LINK: + mock = Mock() + mock.status_code = 201 + return mock else: return mock() From 225557d1a5d0fc05795d3e8718d033f23603450d Mon Sep 17 00:00:00 2001 From: Jani Monoses Date: Mon, 17 May 2021 13:19:20 +0300 Subject: [PATCH 02/13] Add more tests for video upload tool. --- src/cc2olx/tools/video_upload.py | 4 +-- tests/test_tools.py | 48 +++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/cc2olx/tools/video_upload.py b/src/cc2olx/tools/video_upload.py index 587c5f72..bfb87a70 100644 --- a/src/cc2olx/tools/video_upload.py +++ b/src/cc2olx/tools/video_upload.py @@ -58,7 +58,7 @@ def get_access_token(): return data["access_token"] -def parse_args(): +def parse_args(args=None): """Set up and return command line arguments for the video upload tool.""" parser = argparse.ArgumentParser(description="Upload video files to edX via Studio's video encoding pipeline.") parser.add_argument( @@ -79,7 +79,7 @@ def parse_args(): "--output-csv", help="path to where the output CSV should be stored; this will overwrite existing files", ) - return parser.parse_args() + return parser.parse_args(args) def make_generate_upload_link_request(url, data, filename, access_token): diff --git a/tests/test_tools.py b/tests/test_tools.py index b8b14ba9..72da18ac 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,6 +1,14 @@ +from argparse import Namespace from unittest.mock import Mock, call -from cc2olx.tools.video_upload import main, OAUTH_TOKEN_URL, GENERATE_UPLOAD_LINK_BASE_URL, TRANSCRIPT_UPLOAD_LINK +from cc2olx.tools.video_upload import ( + main, + parse_args, + upload_transcript, + OAUTH_TOKEN_URL, + GENERATE_UPLOAD_LINK_BASE_URL, + TRANSCRIPT_UPLOAD_LINK, +) MOCK_UPLOAD_LINK_ROOT = "example.com/upload" @@ -134,3 +142,41 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args): ), ] csv_writerow_mock.assert_has_calls(expected_csv_writerow_call_args, any_order=True) + + +def test_parse_args(): + """ + Basic cli test. + """ + + parsed_args = parse_args(["courseid", "dirname", "input.csv", "--output-csv", "output.csv"]) + + assert parsed_args == Namespace( + course_id="courseid", directory="dirname", input_csv="input.csv", output_csv="output.csv" + ) + + +def upload_transcript_side_effect(*args, **kwargs): + print(kwargs["data"]) + mock = Mock() + + if kwargs["data"].get("edx_video_id") and kwargs["data"].get("language_code"): + mock.status_code = 201 + else: + mock.status_code = 400 + + return mock + + +def test_transcript_upload(mocker): + mocker.patch("cc2olx.tools.video_upload.open") + mocker.patch("cc2olx.tools.video_upload.requests.post", side_effect=upload_transcript_side_effect) + + response = upload_transcript("filename", "edxid", "en") + assert response.status_code == 201 + + response = upload_transcript("filename", "edxid", None) + assert response.status_code == 400 + + response = upload_transcript("filename", None, "en") + assert response.status_code == 400 From 47fec5dee1fdf4caa82be4334a8157e1c42b1bb9 Mon Sep 17 00:00:00 2001 From: Jani Monoses Date: Mon, 17 May 2021 14:44:45 +0300 Subject: [PATCH 03/13] Fix an old typo --- tests/test_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tools.py b/tests/test_tools.py index 72da18ac..ae9b3dfe 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -48,7 +48,7 @@ def post_side_effect(*args, **kwargs): mock.status_code = 201 return mock else: - return mock() + return Mock() def put_side_effect(*args, **kwargs): From 45483c8731cb705da36bf26d632da66ad34a3019 Mon Sep 17 00:00:00 2001 From: Jani Monoses Date: Mon, 24 May 2021 16:10:39 +0300 Subject: [PATCH 04/13] Update readme to mention transcript uploads --- src/cc2olx/tools/README.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cc2olx/tools/README.rst b/src/cc2olx/tools/README.rst index 5130695d..071e7178 100644 --- a/src/cc2olx/tools/README.rst +++ b/src/cc2olx/tools/README.rst @@ -6,7 +6,7 @@ Tools Video Upload Tool ----------------- -The video upload tool uploads video files to edX's video encoding pipeline. +The video upload tool uploads video files and associated transcripts to edX's video encoding pipeline. It also generates an output CSV that can be used as input to the video xBlock conversion tool. Input @@ -16,8 +16,10 @@ The tool also optionally takes as a keyword argument an ``--output-csv``. The ``course-id`` argument is the ID of the course as it appears in Studio. For example, ``course-v1:edX+111222+111222``. -The ``directory`` argument is a directory containing video files that will be uploaded to edX's video encoding pipeline. +The ``directory`` argument is a directory (such as the one created by the video download tool) containing video files and transcripts, that will be uploaded to edX's video encoding pipeline. The currently supported formats for the videos are ``mp4`` and ``mov``, which are the files supported by the edX video encoding pipeline. +The only supported format for transcripts is ``srt``, and transcript files must be named similarly to the videos they belong to. +For example for a video file named video_name.mp4 the tool will find transcripts in the same directory, named like video_name.XX.srt where XX is a language code. Note that files of other types will be ignored by the tool. The ``input-csv`` argument is a file that is meant to establish a relationship between a particular video, its externally hosted URL, and the edX video ID that is generated by the video encoding pipeline when the video is uploaded. From 87f65eedac8e7ea2eb24febc4f48abc1d17f8ab8 Mon Sep 17 00:00:00 2001 From: Jani Monoses Date: Thu, 27 May 2021 11:03:42 +0300 Subject: [PATCH 05/13] Add transcript languages to video upload output CSV This add the new column that contains dash separated, alphabetically ordered language codes to the output CSV. This is used by cc2olx to add transcript tags in the generated video block. --- src/cc2olx/tools/README.rst | 5 ++++- src/cc2olx/tools/video_upload.py | 9 ++++++++- tests/test_tools.py | 5 +++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/cc2olx/tools/README.rst b/src/cc2olx/tools/README.rst index 071e7178..e46639cd 100644 --- a/src/cc2olx/tools/README.rst +++ b/src/cc2olx/tools/README.rst @@ -38,7 +38,10 @@ Output ------ The tool will generate a CSV file. If the optional command line argument ``--output-csv`` is supplied, then the output CSV will be saved to that path. Otherwise, if the command line argument is not supplied, then the file will be saved to the same directory as the input file as specified by the ``input-csv`` command line argument, and the name of the CSV fill will be the name of the original CSV file supplied as a command line argument with "upload-results" appended to the end of the name, i.e. ``-upload-results.csv``. -A new column, "Edx Id", which represents the edX Video ID for each video, will be appended to the end of the file. +Two new columns will be appended to the end of the file: +* "Edx Id", which represents the edX Video ID for each video +* "Languages", a list of dash separated, alphabetically ordered language codes for which transcripts were uploaded for each video. + Otherwise, the file is identical to the file supplied as a command line argument. Logs diff --git a/src/cc2olx/tools/video_upload.py b/src/cc2olx/tools/video_upload.py index bfb87a70..9b69f4bb 100644 --- a/src/cc2olx/tools/video_upload.py +++ b/src/cc2olx/tools/video_upload.py @@ -77,6 +77,7 @@ def parse_args(args=None): ) parser.add_argument( "--output-csv", + "-o", help="path to where the output CSV should be stored; this will overwrite existing files", ) return parser.parse_args(args) @@ -220,6 +221,7 @@ def write_upload_results_csv(input_csv_path, output_csv_path, file_data): new_fieldnames = reader.fieldnames.copy() new_fieldnames.append("Edx Id") + new_fieldnames.append("Languages") writer = csv.DictWriter(output_csv, new_fieldnames) writer.writeheader() @@ -234,6 +236,7 @@ def write_upload_results_csv(input_csv_path, output_csv_path, file_data): new_row = row.copy() new_row["Edx Id"] = data["edx_video_id"] + new_row["Languages"] = data["lang"] writer.writerow(new_row) @@ -295,12 +298,16 @@ def main(): make_upload_video_request(upload_url, data, headers, filename) files_data[str(relative_path)] = {"edx_video_id": edx_video_id} + langs = [] # Look for files with the same name as our video but with a ${LANG}.srt suffix - for srt_path in full_path.parent.glob(full_path.stem + "*.srt"): + for srt_path in sorted(full_path.parent.glob(full_path.stem + "*.srt")): lang = srt_path.suffixes[0][1:] + langs.append(lang) upload_transcript(srt_path, edx_video_id, lang) + files_data[str(relative_path)]["lang"] = "-".join(langs) + input_csv_path = Path(args.input_csv) output_csv_path = args.output_csv diff --git a/tests/test_tools.py b/tests/test_tools.py index ae9b3dfe..ecee59d7 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -98,6 +98,7 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args): "Additional Notes": "Additional Notes", "Youtube ID": "Youtube ID", "External Video Link": "External Video Link", + "Languages": "Languages", } ), call( @@ -108,6 +109,7 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args): "Relative File Path": "01___Intro_to_Knowledge_Based_AI/0 - Introductions.mp4", "Additional Notes": "This is the first video.", "Youtube ID": "onRUvL2SBG8", + "Languages": "en", } ), call( @@ -118,6 +120,7 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args): "Relative File Path": "01___Intro_to_Knowledge_Based_AI/1 - Preview.mp4", "Additional Notes": "", "Youtube ID": "NXlG00JYX-o", + "Languages": "en-fr", } ), call( @@ -128,6 +131,7 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args): "Relative File Path": "01___Intro_to_Knowledge_Based_AI/2 - Conundrums in AI.mov", "Additional Notes": "", "Youtube ID": "_SIvUj7xUKc", + "Languages": "fr", } ), call( @@ -138,6 +142,7 @@ def test_video_upload(self, mocker, monkeypatch, video_upload_args): "Relative File Path": "01___Intro_to_Knowledge_Based_AI/3 - Characteristics of AI Problems.mov", "Additional Notes": "This is the last video.", "Youtube ID": "3pT8dh4ftbc", + "Languages": "", } ), ] From a5a041e154f8226a146d533f4b98ab99f3f00265 Mon Sep 17 00:00:00 2001 From: Josh McLaughlin Date: Mon, 7 Jun 2021 07:15:52 -0700 Subject: [PATCH 06/13] Use Authorization header for transcript uploads --- src/cc2olx/tools/video_upload.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/cc2olx/tools/video_upload.py b/src/cc2olx/tools/video_upload.py index 9b69f4bb..355ceb05 100644 --- a/src/cc2olx/tools/video_upload.py +++ b/src/cc2olx/tools/video_upload.py @@ -127,7 +127,7 @@ def make_generate_upload_link_request(url, data, filename, access_token): return response -def upload_transcript(filename, edx_video_id, language_code): +def upload_transcript(filename, edx_video_id, language_code, access_token): """ Make a POST request against the Studio upload transcript API and return the response. If errors occur during the API call, log to the console. @@ -140,11 +140,14 @@ def upload_transcript(filename, edx_video_id, language_code): Returns: * response: the response object from the POST API call """ + s = requests.Session() + s.auth = SuppliedJwtAuth(access_token) + data = {"edx_video_id": edx_video_id, "language_code": language_code, "new_language_code": language_code} files = {"file": open(filename, "rb")} try: - response = requests.post(TRANSCRIPT_UPLOAD_LINK, data=data, files=files) + response = s.post(TRANSCRIPT_UPLOAD_LINK, data=data, files=files) response.raise_for_status() except requests.exceptions.HTTPError as error: print( @@ -304,7 +307,7 @@ def main(): for srt_path in sorted(full_path.parent.glob(full_path.stem + "*.srt")): lang = srt_path.suffixes[0][1:] langs.append(lang) - upload_transcript(srt_path, edx_video_id, lang) + upload_transcript(srt_path, edx_video_id, lang, access_token) files_data[str(relative_path)]["lang"] = "-".join(langs) From 8489087650233e72503d5b6c3d97ee2c180da821 Mon Sep 17 00:00:00 2001 From: Josh McLaughlin Date: Mon, 7 Jun 2021 07:16:49 -0700 Subject: [PATCH 07/13] Use newly created transcript upload endpoint --- src/cc2olx/tools/video_upload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cc2olx/tools/video_upload.py b/src/cc2olx/tools/video_upload.py index 355ceb05..11e8e024 100644 --- a/src/cc2olx/tools/video_upload.py +++ b/src/cc2olx/tools/video_upload.py @@ -8,10 +8,10 @@ OAUTH_TOKEN_URL = "https://courses.edx.org/oauth2/access_token" GENERATE_UPLOAD_LINK_BASE_URL = "https://studio.edx.org/generate_video_upload_link/" -TRANSCRIPT_UPLOAD_LINK = "https://studio.edx.org/transcript_upload/" +TRANSCRIPT_UPLOAD_LINK = "https://studio.edx.org/transcript_upload_api/" # OAUTH_TOKEN_URL = "https://courses.stage.edx.org/oauth2/access_token" # GENERATE_UPLOAD_LINK_BASE_URL = "https://studio.stage.edx.org/generate_video_upload_link/" -# TRANSCRIPT_UPLOAD_LINK = "https://studio.stage.edx.org/transcript_upload/" +# TRANSCRIPT_UPLOAD_LINK = "https://studio.stage.edx.org/transcript_upload_api/" VIDEO_EXTENSION_CONTENT_TYPES = { ".mp4": "video/mp4", From e770f1f3a15afe7bccf47749e676a404ede85404 Mon Sep 17 00:00:00 2001 From: Josh McLaughlin Date: Wed, 9 Jun 2021 14:26:52 -0700 Subject: [PATCH 08/13] Update tests for new transcript_upload function --- tests/conftest.py | 5 +++++ tests/test_tools.py | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3ee17b22..7c93b47e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -219,3 +219,8 @@ def bad_passports_csv(fixtures_data_dir): """ bad_passports_csv = str(fixtures_data_dir / "bad_passports.csv") return bad_passports_csv + +@pytest.fixture(scope="session") +def transcript_file(fixtures_data_dir): + transcript_file_path = str(fixtures_data_dir / "video_files/01___Intro_to_Knowledge_Based_AI/0 - Introductions.en.srt") + return transcript_file_path diff --git a/tests/test_tools.py b/tests/test_tools.py index ecee59d7..57b5721f 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -173,15 +173,15 @@ def upload_transcript_side_effect(*args, **kwargs): return mock -def test_transcript_upload(mocker): - mocker.patch("cc2olx.tools.video_upload.open") +def test_transcript_upload(mocker, transcript_file): mocker.patch("cc2olx.tools.video_upload.requests.post", side_effect=upload_transcript_side_effect) + mocker.patch("cc2olx.tools.video_upload.requests.Session.post", side_effect=upload_transcript_side_effect) - response = upload_transcript("filename", "edxid", "en") + response = upload_transcript(transcript_file, "edxid", "en", "test_access_token") assert response.status_code == 201 - response = upload_transcript("filename", "edxid", None) + response = upload_transcript(transcript_file, "edxid", None, "test_access_token") assert response.status_code == 400 - response = upload_transcript("filename", None, "en") + response = upload_transcript(transcript_file, None, "en", "test_access_token") assert response.status_code == 400 From 7a20adf3063630a065be3b92e5e85c8cbfd0b842 Mon Sep 17 00:00:00 2001 From: Josh McLaughlin Date: Wed, 9 Jun 2021 14:58:13 -0700 Subject: [PATCH 09/13] Update documentation for video download tool --- src/cc2olx/tools/README.rst | 38 +++++++++++++++++++++++++++++++- src/cc2olx/tools/video_upload.py | 1 + 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/cc2olx/tools/README.rst b/src/cc2olx/tools/README.rst index e46639cd..733c476e 100644 --- a/src/cc2olx/tools/README.rst +++ b/src/cc2olx/tools/README.rst @@ -18,7 +18,8 @@ The ``course-id`` argument is the ID of the course as it appears in Studio. For The ``directory`` argument is a directory (such as the one created by the video download tool) containing video files and transcripts, that will be uploaded to edX's video encoding pipeline. The currently supported formats for the videos are ``mp4`` and ``mov``, which are the files supported by the edX video encoding pipeline. -The only supported format for transcripts is ``srt``, and transcript files must be named similarly to the videos they belong to. +The tool will recurse into the provided directory and files at any depth, and matching the supported formats will be uploaded. +The only supported format for transcripts is ``srt``, and transcript files must have the same location and base name as their associated video, apart from the suffix. For example for a video file named video_name.mp4 the tool will find transcripts in the same directory, named like video_name.XX.srt where XX is a language code. Note that files of other types will be ignored by the tool. @@ -77,3 +78,38 @@ Once these are done, you can run the tool as follows:: Or, if you would like to specify the path to the output CSV file:: python src/cc2olx/tools/video_upload.py course-v1:edX+111222+111222 /Users/example/workspaces/videos /Users/example/workspaces/video-data.csv --output-csv /Users/example/workspaces/video-data-output.csv +.. _video_download_tool: + +Video Download Tool +------------------- +The download tool will accept a common cartridge format course, and search for matching videos to download, in preparation for use with the video upload tool. +It also generates a CSV in a format that can be used by the video upload tool for inclusion in the edX video pipeline (by default "out.csv"). + +Input +----- +The tool has one required parameter ``-i`` or ``--input``, which is the Common Cartridge Course (IMSCC) format archive, or single HTML file. This is searched for iframes with video links. +The tool will download the raw video for any embedded video links, as well as associated transcripts, and store these in the `downloads` folder in the current working directory. + +The ``--output`` argument allows for specifying an alternative filename to store the CSV metadata generated for downloaded videos. + +The ``--downloads`` argument will specify an alternative directory for storing downloaded video data. + +The ``--simulate`` argument allows for a dry-run of parsing and extracting URLs for download, without actually downloading the video content. + +The ``--config`` argument allows for providing extra configuration to youtube-dl which is used to handle video downloading. + +Output +------ +Unless otherwise specified with the options above, the tool will generate a file ``out.csv`` containing the URL to the video, file path of the downloaded video, and a YouTube ID if the video was originally hosted on YouTube. +The tool will also create a `downloads` directory with the raw videos downloaded, and any transcripts or subtitles associated with the videos. + +Use +--- +The video download tool is a basic command-line Python 3 program. +This can be run simply against a Common Cartridge Format Course (IMSCC) file as so:: + + python src/cc2olx/tools/video_download.py -i ~/ushistory.imscc + +This tool is most useful by chaining it with the video upload tool, in which case the video upload tool can be run in the same directory after download is complete like so:: + + python src/cc2olx/tools/video_upload.py course-v1:edX+111222+111222 ./downloads ./out.csv diff --git a/src/cc2olx/tools/video_upload.py b/src/cc2olx/tools/video_upload.py index 11e8e024..4878d091 100644 --- a/src/cc2olx/tools/video_upload.py +++ b/src/cc2olx/tools/video_upload.py @@ -136,6 +136,7 @@ def upload_transcript(filename, edx_video_id, language_code, access_token): * filename: the transcript filename * edx_video_id: the video ID of the video this transcript is for * language_code: the language of the transcript + * access_token: access token to be able to make authenticated calls to the Studio API Returns: * response: the response object from the POST API call From a3cdb574aebce9d882150323884b9a3be1cb86c5 Mon Sep 17 00:00:00 2001 From: Josh McLaughlin Date: Wed, 9 Jun 2021 15:03:12 -0700 Subject: [PATCH 10/13] Add clearer heading hierarchy in tools README --- src/cc2olx/tools/README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cc2olx/tools/README.rst b/src/cc2olx/tools/README.rst index 733c476e..193e7b4d 100644 --- a/src/cc2olx/tools/README.rst +++ b/src/cc2olx/tools/README.rst @@ -4,7 +4,7 @@ Tools .. _video_upload_tool: Video Upload Tool ------------------ +================= The video upload tool uploads video files and associated transcripts to edX's video encoding pipeline. It also generates an output CSV that can be used as input to the video xBlock conversion tool. @@ -81,7 +81,7 @@ Or, if you would like to specify the path to the output CSV file:: .. _video_download_tool: Video Download Tool -------------------- +=================== The download tool will accept a common cartridge format course, and search for matching videos to download, in preparation for use with the video upload tool. It also generates a CSV in a format that can be used by the video upload tool for inclusion in the edX video pipeline (by default "out.csv"). From c663606ecd7ffce71de243db06bb35a617fe119d Mon Sep 17 00:00:00 2001 From: Josh McLaughlin Date: Fri, 11 Jun 2021 18:38:16 -0700 Subject: [PATCH 11/13] Fix tests formatting --- tests/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7c93b47e..95dab887 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -220,7 +220,10 @@ def bad_passports_csv(fixtures_data_dir): bad_passports_csv = str(fixtures_data_dir / "bad_passports.csv") return bad_passports_csv + @pytest.fixture(scope="session") def transcript_file(fixtures_data_dir): - transcript_file_path = str(fixtures_data_dir / "video_files/01___Intro_to_Knowledge_Based_AI/0 - Introductions.en.srt") + transcript_file_path = str( + fixtures_data_dir / "video_files/01___Intro_to_Knowledge_Based_AI/0 - Introductions.en.srt" + ) return transcript_file_path From 3edcf732bffd63ac53f7cc23c5234c05242a0069 Mon Sep 17 00:00:00 2001 From: Jani Monoses Date: Thu, 27 May 2021 11:38:16 +0300 Subject: [PATCH 12/13] Add transcript nodes to video blocks --- src/cc2olx/iframe_link_parser.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/cc2olx/iframe_link_parser.py b/src/cc2olx/iframe_link_parser.py index eedd33a1..64edf1c8 100644 --- a/src/cc2olx/iframe_link_parser.py +++ b/src/cc2olx/iframe_link_parser.py @@ -99,6 +99,7 @@ def _create_video_olx(self, doc, url_row): attributes = {} edx_id = url_row.get("Edx Id", "") youtube_id = url_row.get("Youtube Id", "") + languages = url_row.get("Languages", "") if edx_id.strip() != "": attributes["edx_video_id"] = edx_id elif youtube_id.strip() != "": @@ -106,7 +107,16 @@ def _create_video_olx(self, doc, url_row): attributes["youtube_id_1_0"] = youtube_id else: raise IframeLinkParserError("Missing Edx Id or Youtube Id for video conversion.") - child = xml_element("video", children=None, attributes=attributes) + + children = [] + + # For rows that contain languages generate transcript nodes + if languages != "": + for lang in languages.split("-"): + src = f"{edx_id}-{lang}.srt" + transcript = xml_element("transcript", children=None, attributes={"language": lang, "src": src}) + children.append(transcript) + child = xml_element("video", children=children, attributes=attributes) return child From cdd933c63a221e487d6c4e8909c09cde24fe7530 Mon Sep 17 00:00:00 2001 From: Jani Monoses Date: Thu, 27 May 2021 12:31:03 +0300 Subject: [PATCH 13/13] Add tests for language column in link files --- tests/conftest.py | 15 +++++++++++++++ tests/fixtures_data/link_map_languages.csv | 6 ++++++ tests/test_kaltura_link_parser.py | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 tests/fixtures_data/link_map_languages.csv diff --git a/tests/conftest.py b/tests/conftest.py index 95dab887..4503120d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -127,6 +127,21 @@ def link_map_csv(fixtures_data_dir): return link_map_csv_file_path +@pytest.fixture(scope="session") +def link_map_languages_csv(fixtures_data_dir): + """ + This fixture helps to provide csv file path with transcript languages included + Args: + fixtures_data_dir ([str]): Path to the directory where fixture data is present. + + Returns: + [str]: Path to the csv + """ + + link_map_csv_file_path = str(fixtures_data_dir / "link_map_languages.csv") + return link_map_csv_file_path + + @pytest.fixture(scope="session") def link_map_edx_only_csv(fixtures_data_dir): """ diff --git a/tests/fixtures_data/link_map_languages.csv b/tests/fixtures_data/link_map_languages.csv new file mode 100644 index 00000000..1ae2e89a --- /dev/null +++ b/tests/fixtures_data/link_map_languages.csv @@ -0,0 +1,6 @@ +External Video Link,Edx Id,Youtube Id,Languages +https://cdnapisec.kaltura.com/p/2019031/sp/201903100/playManifest/entryId/1_zeqnrfgw/format/url/protocol/https,42d2a5e2-bced-45d6-b8dc-2f5901c9fdd0,onRUvL2SBG8,en-fr +https://cdnapisec.kaltura.com/p/2019031/sp/201903100/playManifest/entryId/1_9if7cth0/format/url/protocol/https,42d2a5e2-bced-45d6-b8dc-2f5901c9fdd1,NXlG00JYX-o,en +https://cdnapisec.kaltura.com/p/2019031/sp/201903100/playManifest/entryId/1_c7j5va21/format/url/protocol/https,42d2a5e2-bced-45d6-b8dc-2f5901c9fdd2,_SIvUj7xUKc,fr +https://cdnapisec.kaltura.com/p/2019031/sp/201903100/playManifest/entryId/1_xcjzc0q5/format/url/protocol/https,42d2a5e2-bced-45d6-b8dc-2f5901c9fdd3,3pT8dh4ftbc +https://cdnapisec.kaltura.com/p/2019031/sp/201903100/playManifest/entryId/1_t1jur7p3/format/url/protocol/https,42d2a5e2-bced-45d6-b8dc-2f5901c9fdd4,uwuZKEKpq8k,de-en-fr diff --git a/tests/test_kaltura_link_parser.py b/tests/test_kaltura_link_parser.py index 543cceb9..78b4ea71 100644 --- a/tests/test_kaltura_link_parser.py +++ b/tests/test_kaltura_link_parser.py @@ -152,3 +152,22 @@ def test_video_olx_bad_link_map(self, iframes, link_map_bad_csv): with pytest.raises(Exception): video_olx, _ = iframe_link_parser.get_video_olx(doc, iframes) + + def test_video_olx_languages(self, iframes, link_map_languages_csv): + """ + Test that video olx is generated and produced when transcript languages are provided + """ + iframe_link_parser = KalturaIframeLinkParser(link_map_languages_csv) + doc = xml.dom.minidom.Document() + video_olx, _ = iframe_link_parser.get_video_olx(doc, iframes) + + assert len(video_olx) == 1 + + actual_video_olx = video_olx[0] + + # The first line in the fixtures file has two languages listed + assert len(actual_video_olx.childNodes) == 2 + + assert actual_video_olx.firstChild.nodeName == "transcript" + assert actual_video_olx.firstChild.hasAttribute("language") + assert actual_video_olx.firstChild.hasAttribute("src")