From 6e114c9a4ced838815e14307225a5ee63ddfe8ac Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 18 Oct 2018 15:11:02 -0600 Subject: [PATCH 01/29] Test what the value of $TRAVIS_SECURE_ENV_VARS is on Travis on a fork --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5e0cee53..f17cd0b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,7 @@ install: script: - set -e - set -x + - echo $TRAVIS_SECURE_ENV_VARS # Note, the commands below need --sync to actually sync, to override sync: False below. Also, every deploy to the main repo needs to have --key-path deploy_key.enc. - | if [[ "${DOCS}" == "true" ]]; then From 688db82d219e7856e2756b6013105b13b0d896ad Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 18 Oct 2018 15:11:36 -0600 Subject: [PATCH 02/29] Trigger build From 28ba6e496c42c43af32639e8a00dfbed455c9e83 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 18 Oct 2018 15:40:49 -0600 Subject: [PATCH 03/29] Check if a repo is a fork on Travis Travis does not make encrypted environment variables available to forks, so doctr should not attempt to push from them. Fixes #110 Closes #331 Closes #236 --- doctr/tests/test_travis.py | 75 +++++++++++++++++++++----------------- doctr/travis.py | 15 +++++++- 2 files changed, 56 insertions(+), 34 deletions(-) diff --git a/doctr/tests/test_travis.py b/doctr/tests/test_travis.py index 09b9e5bf..9efada88 100644 --- a/doctr/tests/test_travis.py +++ b/doctr/tests/test_travis.py @@ -222,45 +222,53 @@ def test_sync_from_log(src, dst): @pytest.mark.parametrize("""branch_whitelist, TRAVIS_BRANCH, - TRAVIS_PULL_REQUEST, TRAVIS_TAG, build_tags, + TRAVIS_PULL_REQUEST, TRAVIS_TAG, fork, build_tags, canpush""", [ - ('master', 'doctr', 'true', "", False, False), - ('master', 'doctr', 'false', "", False, False), - ('master', 'master', 'true', "", False, False), - ('master', 'master', 'false', "", False, True), - ('doctr', 'doctr', 'True', "", False, False), - ('doctr', 'doctr', 'false', "", False, True), - ('set()', 'doctr', 'false', "", False, False), - - ('master', 'doctr', 'true', "tagname", False, False), - ('master', 'doctr', 'false', "tagname", False, False), - ('master', 'master', 'true', "tagname", False, False), - ('master', 'master', 'false', "tagname", False, False), - ('doctr', 'doctr', 'True', "tagname", False, False), - ('doctr', 'doctr', 'false', "tagname", False, False), - ('set()', 'doctr', 'false', "tagname", False, False), - - ('master', 'doctr', 'true', "", True, False), - ('master', 'doctr', 'false', "", True, False), - ('master', 'master', 'true', "", True, False), - ('master', 'master', 'false', "", True, True), - ('doctr', 'doctr', 'True', "", True, False), - ('doctr', 'doctr', 'false', "", True, True), - ('set()', 'doctr', 'false', "", True, False), - - ('master', 'doctr', 'true', "tagname", True, True), - ('master', 'doctr', 'false', "tagname", True, True), - ('master', 'master', 'true', "tagname", True, True), - ('master', 'master', 'false', "tagname", True, True), - ('doctr', 'doctr', 'True', "tagname", True, True), - ('doctr', 'doctr', 'false', "tagname", True, True), - ('set()', 'doctr', 'false', "tagname", True, True), + ('master', 'doctr', 'true', "", False, False, False), + ('master', 'doctr', 'false', "", False, False, False), + ('master', 'master', 'true', "", False, False, False), + ('master', 'master', 'false', "", False, False, True), + ('doctr', 'doctr', 'True', "", False, False, False), + ('doctr', 'doctr', 'false', "", False, False, True), + ('set()', 'doctr', 'false', "", False, False, False), + + ('master', 'doctr', 'true', "tagname", False, False, False), + ('master', 'doctr', 'false', "tagname", False, False, False), + ('master', 'master', 'true', "tagname", False, False, False), + ('master', 'master', 'false', "tagname", False, False, False), + ('doctr', 'doctr', 'True', "tagname", False, False, False), + ('doctr', 'doctr', 'false', "tagname", False, False, False), + ('set()', 'doctr', 'false', "tagname", False, False, False), + + ('master', 'doctr', 'true', "", False, True, False), + ('master', 'doctr', 'false', "", False, True, False), + ('master', 'master', 'true', "", False, True, False), + ('master', 'master', 'false', "", False, True, True), + ('doctr', 'doctr', 'True', "", False, True, False), + ('doctr', 'doctr', 'false', "", False, True, True), + ('set()', 'doctr', 'false', "", False, True, False), + + ('master', 'doctr', 'true', "tagname", False, True, True), + ('master', 'doctr', 'false', "tagname", False, True, True), + ('master', 'master', 'true', "tagname", False, True, True), + ('master', 'master', 'false', "tagname", False, True, True), + ('doctr', 'doctr', 'True', "tagname", False, True, True), + ('doctr', 'doctr', 'false', "tagname", False, True, True), + ('set()', 'doctr', 'false', "tagname", False, True, True), + + ('master', 'doctr', 'true', "", True, False, False), + ('master', 'doctr', 'false', "", True, False, False), + ('master', 'master', 'true', "", True, False, False), + ('master', 'master', 'false', "", True, False, False), + ('doctr', 'doctr', 'True', "", True, False, False), + ('doctr', 'doctr', 'false', "", True, False, False), + ('set()', 'doctr', 'false', "", True, False, False), ]) def test_determine_push_rights(branch_whitelist, TRAVIS_BRANCH, - TRAVIS_PULL_REQUEST, TRAVIS_TAG, build_tags, canpush, monkeypatch): + TRAVIS_PULL_REQUEST, TRAVIS_TAG, build_tags, fork, canpush, monkeypatch): branch_whitelist = {branch_whitelist} assert determine_push_rights( @@ -268,6 +276,7 @@ def test_determine_push_rights(branch_whitelist, TRAVIS_BRANCH, TRAVIS_BRANCH=TRAVIS_BRANCH, TRAVIS_PULL_REQUEST=TRAVIS_PULL_REQUEST, TRAVIS_TAG=TRAVIS_TAG, + fork=fork, build_tags=build_tags) == canpush @pytest.mark.parametrize("src", ["src", "."]) diff --git a/doctr/travis.py b/doctr/travis.py index 80a72c66..5a0c8a9e 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -13,6 +13,8 @@ import tempfile import time +import requests + from cryptography.fernet import Fernet from .common import red, blue @@ -214,10 +216,17 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', TRAVIS_BRANCH = os.environ.get("TRAVIS_BRANCH", "") TRAVIS_PULL_REQUEST = os.environ.get("TRAVIS_PULL_REQUEST", "") + # Check if the repo is a fork + TRAVIS_REPO_SLUG = os.environ["TRAVIS_REPO_SLUG"] + REPO_URL = 'https://api.github.com/repos/{slug}' + r = requests.get(REPO_URL.format(slug=TRAVIS_REPO_SLUG)) + fork = r.json()['fork'] + canpush = determine_push_rights( branch_whitelist=branch_whitelist, TRAVIS_BRANCH=TRAVIS_BRANCH, TRAVIS_PULL_REQUEST=TRAVIS_PULL_REQUEST, + fork=fork, TRAVIS_TAG=TRAVIS_TAG, build_tags=build_tags) @@ -549,7 +558,7 @@ def last_commit_by_doctr(): return False def determine_push_rights(*, branch_whitelist, TRAVIS_BRANCH, - TRAVIS_PULL_REQUEST, TRAVIS_TAG, build_tags): + TRAVIS_PULL_REQUEST, TRAVIS_TAG, build_tags, fork): """Check if Travis is running on ``master`` (or a whitelisted branch) to determine if we can/should push the docs to the deploy repo """ @@ -570,6 +579,10 @@ def determine_push_rights(*, branch_whitelist, TRAVIS_BRANCH, print("The website and docs are not pushed to gh-pages on pull requests", file=sys.stderr) canpush = False + if fork: + print("The website and docs are not pushed to gh-pages on fork builds.", file=sys.stderr) + canpush = False + if last_commit_by_doctr(): print(red("The last commit on this branch was pushed by doctr. Not pushing to " "avoid an infinite build-loop."), file=sys.stderr) From 82381b72f437238032614705519a2c8bd91053ad Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 18 Oct 2018 15:45:53 -0600 Subject: [PATCH 04/29] Remove test echo from .travis.yml --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f17cd0b2..5e0cee53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,6 @@ install: script: - set -e - set -x - - echo $TRAVIS_SECURE_ENV_VARS # Note, the commands below need --sync to actually sync, to override sync: False below. Also, every deploy to the main repo needs to have --key-path deploy_key.enc. - | if [[ "${DOCS}" == "true" ]]; then From e910c8fa46231bd986df3784622b458519e2e9fd Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 18 Oct 2018 15:54:14 -0600 Subject: [PATCH 05/29] Assume a repo is not a fork if the GitHub API doesn't have the fork key I think this might happen when the request fails. Since we can't really control this, handle the common case of the repo not being a fork. --- doctr/travis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctr/travis.py b/doctr/travis.py index 5a0c8a9e..225cc2bb 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -220,7 +220,7 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', TRAVIS_REPO_SLUG = os.environ["TRAVIS_REPO_SLUG"] REPO_URL = 'https://api.github.com/repos/{slug}' r = requests.get(REPO_URL.format(slug=TRAVIS_REPO_SLUG)) - fork = r.json()['fork'] + fork = r.json().get('fork', False) canpush = determine_push_rights( branch_whitelist=branch_whitelist, From 969243d341554d3bddc35965e9a63a2dc33548fd Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 11:58:42 -0700 Subject: [PATCH 06/29] Don't run test_guess_github_repo() on fork builds --- doctr/tests/test_local.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doctr/tests/test_local.py b/doctr/tests/test_local.py index 368d0f75..6c63a8c7 100644 --- a/doctr/tests/test_local.py +++ b/doctr/tests/test_local.py @@ -67,7 +67,7 @@ def test_GIT_URL(): def test_guess_github_repo(): """ Only works if run in this repo, and if cloned from origin. For safety, - only run on Travis + only run on Travis and not run on fork builds. """ - if on_travis(): + if on_travis() and os.environ.get('TRAVIS_SECURE_ENV_VARS', 'false') == 'true': assert guess_github_repo() == 'drdoctr/doctr' From 77d0baceee021418291a56d2c0c7d9d6e44a0084 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 12:00:59 -0700 Subject: [PATCH 07/29] Use pytest to skip test_guess_github_repo That way we can see if it is being skipped properly. --- doctr/tests/test_local.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doctr/tests/test_local.py b/doctr/tests/test_local.py index 6c63a8c7..b60aef2d 100644 --- a/doctr/tests/test_local.py +++ b/doctr/tests/test_local.py @@ -64,10 +64,11 @@ def test_GIT_URL(): assert not GIT_URL.fullmatch('https://gitlab.com/drdoctr/doctr.git') +@pytest.mark.skipif(not on_travis(), reason="Not on Travis") +@pytest.mark.skipif(os.environ.get('TRAVIS_SECURE_ENV_VARS', 'false') != 'true', reason="Not run on Travis fork builds") def test_guess_github_repo(): """ Only works if run in this repo, and if cloned from origin. For safety, only run on Travis and not run on fork builds. """ - if on_travis() and os.environ.get('TRAVIS_SECURE_ENV_VARS', 'false') == 'true': - assert guess_github_repo() == 'drdoctr/doctr' + assert guess_github_repo() == 'drdoctr/doctr' From 40fe8eeaf1bddac100f6786404836d985164ac6e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 12:01:26 -0700 Subject: [PATCH 08/29] Run py.test with verbose mode on Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5e0cee53..e7f2445a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ script: fi - if [[ "${TESTS}" == "true" ]]; then pyflakes doctr; - py.test doctr; + py.test doctr -v; fi doctr: From 4f343240a2ebbb82e154f8f2a44a6fbbacfd385a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 12:04:01 -0700 Subject: [PATCH 09/29] Fix skipif order --- doctr/tests/test_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctr/tests/test_local.py b/doctr/tests/test_local.py index b60aef2d..9678207f 100644 --- a/doctr/tests/test_local.py +++ b/doctr/tests/test_local.py @@ -64,8 +64,8 @@ def test_GIT_URL(): assert not GIT_URL.fullmatch('https://gitlab.com/drdoctr/doctr.git') -@pytest.mark.skipif(not on_travis(), reason="Not on Travis") @pytest.mark.skipif(os.environ.get('TRAVIS_SECURE_ENV_VARS', 'false') != 'true', reason="Not run on Travis fork builds") +@pytest.mark.skipif(not on_travis(), reason="Not on Travis") def test_guess_github_repo(): """ Only works if run in this repo, and if cloned from origin. For safety, From ab7713c8daf316ec6a410ad4c02ec43dd4f4bb0b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 12:04:23 -0700 Subject: [PATCH 10/29] Show the skip reasons on Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e7f2445a..4c822b0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ script: fi - if [[ "${TESTS}" == "true" ]]; then pyflakes doctr; - py.test doctr -v; + py.test doctr -v -rs; fi doctr: From 0cffa6a3de915e34adf23bf959f87fb4bc55cd34 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 12:12:29 -0700 Subject: [PATCH 11/29] Fix check if we are on a Travis fork build for test_guess_github_repo --- doctr/tests/test_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctr/tests/test_local.py b/doctr/tests/test_local.py index 9678207f..3d76f74d 100644 --- a/doctr/tests/test_local.py +++ b/doctr/tests/test_local.py @@ -64,7 +64,7 @@ def test_GIT_URL(): assert not GIT_URL.fullmatch('https://gitlab.com/drdoctr/doctr.git') -@pytest.mark.skipif(os.environ.get('TRAVIS_SECURE_ENV_VARS', 'false') != 'true', reason="Not run on Travis fork builds") +@pytest.mark.skipif(os.environ.get('TRAVIS_REPO_SLUG', '') != 'drdoctr/doctr', reason="Not run on Travis fork builds") @pytest.mark.skipif(not on_travis(), reason="Not on Travis") def test_guess_github_repo(): """ From c929ffc23c621447ca0f9361fb159955286f9975 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 12:32:26 -0700 Subject: [PATCH 12/29] Make sure stash-test is a directory If it doesn't yet exist doctr will assume it is a file because it is syncing a single file. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4c822b0b..a3479c7c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ script: echo `date` >> test python -m doctr deploy --key-path deploy_key.enc --no-require-master docs; # Test syncing a tracked file with a change - python -m doctr deploy --sync --key-path deploy_key.enc stash-test --built-docs test; + python -m doctr deploy --sync --key-path deploy_key.enc stash-test/ --built-docs test; # Test deploy branch creation. Delete the branch gh-pages-testing on drdoctr/doctr whenever you want to test this. python -m doctr deploy --sync --key-path deploy_key.enc --no-require-master --deploy-branch gh-pages-testing docs; # Test pushing to .github.io From d84ae9b068a6dcfa6994821af4f8da4b896527f8 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 12:38:32 -0700 Subject: [PATCH 13/29] See if things are failing because the request failed --- doctr/travis.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doctr/travis.py b/doctr/travis.py index 225cc2bb..304c1c4c 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -221,6 +221,9 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', REPO_URL = 'https://api.github.com/repos/{slug}' r = requests.get(REPO_URL.format(slug=TRAVIS_REPO_SLUG)) fork = r.json().get('fork', False) + if 'fork' not in r.json(): + r.raise_for_status() + raise RuntimeError("fork key not found") canpush = determine_push_rights( branch_whitelist=branch_whitelist, From a552efbd2552e51ba649f0141645be4125775daa Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 12:44:15 -0700 Subject: [PATCH 14/29] isdir only returns True if the directory already exists --- doctr/travis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctr/travis.py b/doctr/travis.py index 304c1c4c..acbd0b56 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -453,7 +453,7 @@ def sync_from_log(src, dst, log_file, exclude=()): if any(is_subdir(f, os.path.join(src, i)) for i in exclude): continue new_f = join(dst, f[len(src):]) - if isdir(f): + if isdir(f) or f.endswith('/'): os.makedirs(new_f, exist_ok=True) else: shutil.copy2(f, new_f) From 5d8d2a22f09338da7c10fa014a5d4a19aaef8696 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 13:07:54 -0700 Subject: [PATCH 15/29] Improve handling of syncing single files If the dst is an existing directory, or ends in /, it is treated as a directory and the file is synced to that directory. Otherwise, it is treated as a file and the file is synced to that filename. --- doctr/tests/test_travis.py | 80 ++++++++++++++++++++++++++++++++++++++ doctr/travis.py | 16 +++++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/doctr/tests/test_travis.py b/doctr/tests/test_travis.py index 9efada88..12a8e701 100644 --- a/doctr/tests/test_travis.py +++ b/doctr/tests/test_travis.py @@ -221,6 +221,86 @@ def test_sync_from_log(src, dst): os.chdir(old_curdir) +def test_sync_from_log_file_to_file(): + with tempfile.TemporaryDirectory() as dir: + try: + old_curdir = os.path.abspath(os.curdir) + os.chdir(dir) + + # src is a file and dst doesn't exist as a directory or end in /, + # so should sync to a file called 'dst' + src = 'file' + dst = 'dst' + + with open(src, 'w') as f: + f.write('test1') + + # Test that the sync happens + added, removed = sync_from_log(src, dst, 'logfile') + + assert added == [ + 'dst', + 'logfile', + ] + + assert removed == [] + + # Make sure dst is a file + with open(dst) as f: + assert f.read() == 'test1' + + + with open('logfile') as f: + assert f.read() == '\n'.join([ + dst, + ]) + + # Now make sure that the dst is created as a directory if it + # ends in / + + finally: + os.chdir(old_curdir) + + +def test_sync_from_log_file_to_dir(): + with tempfile.TemporaryDirectory() as dir: + try: + old_curdir = os.path.abspath(os.curdir) + os.chdir(dir) + + # src is a file but dst ends in / (even though it doesn't exist), + # so it should sync to dst/file + src = 'file' + dst = 'dst/' + + with open(src, 'w') as f: + f.write('test1') + + # Test that the sync happens + added, removed = sync_from_log(src, dst, 'logfile') + + assert added == [ + os.path.join('dst', 'file'), + 'logfile', + ] + + assert removed == [] + + assert os.path.isdir(dst) + # Make sure dst is a file + with open(os.path.join('dst', 'file')) as f: + assert f.read() == 'test1' + + + with open('logfile') as f: + assert f.read() == '\n'.join([ + os.path.join('dst', 'file') + ]) + + finally: + os.chdir(old_curdir) + + @pytest.mark.parametrize("""branch_whitelist, TRAVIS_BRANCH, TRAVIS_PULL_REQUEST, TRAVIS_TAG, fork, build_tags, canpush""", diff --git a/doctr/travis.py b/doctr/travis.py index acbd0b56..6c4deab5 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -446,14 +446,26 @@ def sync_from_log(src, dst, log_file, exclude=()): files = glob.iglob(join(src, '**'), recursive=True) else: files = [src] - src = os.path.dirname(src) + os.sep + + if dst.endswith(os.sep): + os.makedirs(dst, exist_ok=True) + elif not isdir(dst) and os.sep not in src: + # src is a file and dst doesn't end in / or exist as a directory, + # so make it a file + shutil.copy2(src, dst) + added.append(dst) + if dst in removed: + removed.remove(dst) + files = [] + + src = '' # sorted makes this easier to test for f in sorted(files): if any(is_subdir(f, os.path.join(src, i)) for i in exclude): continue new_f = join(dst, f[len(src):]) - if isdir(f) or f.endswith('/'): + if isdir(f) or f.endswith(os.sep): os.makedirs(new_f, exist_ok=True) else: shutil.copy2(f, new_f) From 47395f2e808e89fc1252df5b87c733ccb14da21e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 13:27:51 -0700 Subject: [PATCH 16/29] Debug print --- doctr/travis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doctr/travis.py b/doctr/travis.py index 6c4deab5..7df6022b 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -417,6 +417,7 @@ def sync_from_log(src, dst, log_file, exclude=()): ``exclude`` may be a list of paths from ``src`` that should be ignored. Such paths are neither added nor removed, even if they are in the logfile. """ + print('dst is', dst) from os.path import join, exists, isdir exclude = [os.path.normpath(i) for i in exclude] From 61ba79acbde68f516f4bdbd9d065195155d1edda Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 13:36:37 -0700 Subject: [PATCH 17/29] Debug print --- doctr/travis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctr/travis.py b/doctr/travis.py index 7df6022b..2c551dc5 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -417,7 +417,6 @@ def sync_from_log(src, dst, log_file, exclude=()): ``exclude`` may be a list of paths from ``src`` that should be ignored. Such paths are neither added nor removed, even if they are in the logfile. """ - print('dst is', dst) from os.path import join, exists, isdir exclude = [os.path.normpath(i) for i in exclude] @@ -466,6 +465,7 @@ def sync_from_log(src, dst, log_file, exclude=()): if any(is_subdir(f, os.path.join(src, i)) for i in exclude): continue new_f = join(dst, f[len(src):]) + print(src, dst, f, new_f) if isdir(f) or f.endswith(os.sep): os.makedirs(new_f, exist_ok=True) else: From a35d7e6b4bd80a4cdcf974500d459d1c7bb0f2c9 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 13:41:16 -0700 Subject: [PATCH 18/29] Debug: remove raise_for_status on fork check --- doctr/travis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doctr/travis.py b/doctr/travis.py index 2c551dc5..f609e4eb 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -221,9 +221,9 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', REPO_URL = 'https://api.github.com/repos/{slug}' r = requests.get(REPO_URL.format(slug=TRAVIS_REPO_SLUG)) fork = r.json().get('fork', False) - if 'fork' not in r.json(): - r.raise_for_status() - raise RuntimeError("fork key not found") + # if 'fork' not in r.json(): + # r.raise_for_status() + # raise RuntimeError("fork key not found") canpush = determine_push_rights( branch_whitelist=branch_whitelist, From f1827ee16f0737d79a49df29996c0af38b91c74e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 13:46:42 -0700 Subject: [PATCH 19/29] Print a warning message if the fork test failed because of rate limits --- doctr/travis.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doctr/travis.py b/doctr/travis.py index f609e4eb..7865a1f7 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -221,9 +221,8 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', REPO_URL = 'https://api.github.com/repos/{slug}' r = requests.get(REPO_URL.format(slug=TRAVIS_REPO_SLUG)) fork = r.json().get('fork', False) - # if 'fork' not in r.json(): - # r.raise_for_status() - # raise RuntimeError("fork key not found") + if r.status_code == 403: + print(red("Warning: GitHub's API rate limits prevented us from detecting if this build is a fork. If it is, doctr will fail with an error like 'DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set'. This error can be safely ignored. If this is not a fork build, the error likely indicates you need to run 'doctr configure' again for this repo.")) canpush = determine_push_rights( branch_whitelist=branch_whitelist, From 1bdf65c2e52fa9b90e8fab943c6da70fd28f61a1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 13:48:26 -0700 Subject: [PATCH 20/29] Fix the API check warning message --- doctr/travis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doctr/travis.py b/doctr/travis.py index 7865a1f7..938f786f 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -221,8 +221,10 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', REPO_URL = 'https://api.github.com/repos/{slug}' r = requests.get(REPO_URL.format(slug=TRAVIS_REPO_SLUG)) fork = r.json().get('fork', False) + # Rate limits prevent this check from working every time. By default, we + # assume it isn't a fork so that things just work on non-fork builds. if r.status_code == 403: - print(red("Warning: GitHub's API rate limits prevented us from detecting if this build is a fork. If it is, doctr will fail with an error like 'DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set'. This error can be safely ignored. If this is not a fork build, the error likely indicates you need to run 'doctr configure' again for this repo.")) + print(red("Warning: GitHub's API rate limits prevented us from detecting if this build is a fork. If it is, doctr will fail with an error like 'DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set'. This error can be safely ignored. If this is not a fork build, you can ignore this warning.")) canpush = determine_push_rights( branch_whitelist=branch_whitelist, From 9edb5cd29696c16d73ded8c1a3c29d37be9ec400 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 14:43:19 -0700 Subject: [PATCH 21/29] Print the warning to stderr --- doctr/travis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctr/travis.py b/doctr/travis.py index 938f786f..ec7f553a 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -224,7 +224,7 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', # Rate limits prevent this check from working every time. By default, we # assume it isn't a fork so that things just work on non-fork builds. if r.status_code == 403: - print(red("Warning: GitHub's API rate limits prevented us from detecting if this build is a fork. If it is, doctr will fail with an error like 'DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set'. This error can be safely ignored. If this is not a fork build, you can ignore this warning.")) + print(red("Warning: GitHub's API rate limits prevented us from detecting if this build is a fork. If it is, doctr will fail with an error like 'DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set'. This error can be safely ignored. If this is not a fork build, you can ignore this warning."), file=sys.stderr) canpush = determine_push_rights( branch_whitelist=branch_whitelist, From 2629f248bdecf871cdc445754c69d9bc765c9392 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 14:48:58 -0700 Subject: [PATCH 22/29] Debugging --- doctr/travis.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doctr/travis.py b/doctr/travis.py index ec7f553a..bb53bf0b 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -466,11 +466,15 @@ def sync_from_log(src, dst, log_file, exclude=()): if any(is_subdir(f, os.path.join(src, i)) for i in exclude): continue new_f = join(dst, f[len(src):]) - print(src, dst, f, new_f) + if isdir(f) or f.endswith(os.sep): os.makedirs(new_f, exist_ok=True) else: - shutil.copy2(f, new_f) + try: + shutil.copy2(f, new_f) + except: + print("DEBUG:", src, dst, f, new_f) + raise added.append(new_f) if new_f in removed: removed.remove(new_f) From 4eebca17b732434e5bb3689835ca4a496c1bfefa Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 14:49:31 -0700 Subject: [PATCH 23/29] Better wording in the warning message --- doctr/travis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctr/travis.py b/doctr/travis.py index bb53bf0b..27fca1a1 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -224,7 +224,7 @@ def setup_GitHub_push(deploy_repo, *, auth_type='deploy_key', # Rate limits prevent this check from working every time. By default, we # assume it isn't a fork so that things just work on non-fork builds. if r.status_code == 403: - print(red("Warning: GitHub's API rate limits prevented us from detecting if this build is a fork. If it is, doctr will fail with an error like 'DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set'. This error can be safely ignored. If this is not a fork build, you can ignore this warning."), file=sys.stderr) + print(red("Warning: GitHub's API rate limits prevented doctr from detecting if this build is a fork. If it is, doctr will fail with an error like 'DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set'. This error can be safely ignored. If this is not a fork build, you can ignore this warning."), file=sys.stderr) canpush = determine_push_rights( branch_whitelist=branch_whitelist, From d7292d9f05a81585b141cdb1ccfc62d6c89f80e0 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 14:51:53 -0700 Subject: [PATCH 24/29] Test syncing a file to a file on Travis --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index a3479c7c..0bbd5a06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,9 @@ script: python -m doctr deploy --key-path deploy_key.enc --no-require-master docs; # Test syncing a tracked file with a change python -m doctr deploy --sync --key-path deploy_key.enc stash-test/ --built-docs test; + # Test syncing a single file + echo `date` >> test-file + python -m doctr deploy --sync --key-path deploy_key.enc test-file --built-docs test-file; # Test deploy branch creation. Delete the branch gh-pages-testing on drdoctr/doctr whenever you want to test this. python -m doctr deploy --sync --key-path deploy_key.enc --no-require-master --deploy-branch gh-pages-testing docs; # Test pushing to .github.io From 8ec3ed8e77b17cc572dd6b202988a794bdc4b140 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 14:53:15 -0700 Subject: [PATCH 25/29] Print the debug message to standard error --- doctr/travis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctr/travis.py b/doctr/travis.py index 27fca1a1..e4d6c32c 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -473,7 +473,7 @@ def sync_from_log(src, dst, log_file, exclude=()): try: shutil.copy2(f, new_f) except: - print("DEBUG:", src, dst, f, new_f) + print("DEBUG:", src, dst, f, new_f, file=sys.stderr) raise added.append(new_f) if new_f in removed: From 282f92f13dbf244d89ad5b819efeae4d7e9f1d57 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 15:06:36 -0700 Subject: [PATCH 26/29] Fix handling of absolute vs. relative paths for single file sync in sync_from_log --- doctr/travis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctr/travis.py b/doctr/travis.py index e4d6c32c..074365a1 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -459,7 +459,7 @@ def sync_from_log(src, dst, log_file, exclude=()): removed.remove(dst) files = [] - src = '' + src = os.path.dirname(src) + os.sep if os.path.isabs(src) else '' # sorted makes this easier to test for f in sorted(files): From 14e49f7cacd236fe4677da8413741e150b502726 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 15:07:29 -0700 Subject: [PATCH 27/29] Remove debug print --- doctr/travis.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doctr/travis.py b/doctr/travis.py index 074365a1..f0f9289b 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -470,11 +470,7 @@ def sync_from_log(src, dst, log_file, exclude=()): if isdir(f) or f.endswith(os.sep): os.makedirs(new_f, exist_ok=True) else: - try: - shutil.copy2(f, new_f) - except: - print("DEBUG:", src, dst, f, new_f, file=sys.stderr) - raise + shutil.copy2(f, new_f) added.append(new_f) if new_f in removed: removed.remove(new_f) From 504580b2eedc55e50a1346f5b4813971740ae731 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 15:41:43 -0700 Subject: [PATCH 28/29] Correct test if dst is a directory --- doctr/travis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doctr/travis.py b/doctr/travis.py index f0f9289b..d54dd4aa 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -450,7 +450,7 @@ def sync_from_log(src, dst, log_file, exclude=()): if dst.endswith(os.sep): os.makedirs(dst, exist_ok=True) - elif not isdir(dst) and os.sep not in src: + elif not isdir(dst) and os.sep not in dst: # src is a file and dst doesn't end in / or exist as a directory, # so make it a file shutil.copy2(src, dst) From a9fcc2e1cfafaab40a86118919d45faa80ffc457 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 27 Nov 2018 16:12:37 -0700 Subject: [PATCH 29/29] Revert logic for syncing a file to a file The deploy-dir should always be a directory. If you want to sync to the current directory use '.'. --- .travis.yml | 2 +- doctr/tests/test_travis.py | 47 ++------------------------------------ doctr/travis.py | 14 ++---------- 3 files changed, 5 insertions(+), 58 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0bbd5a06..5badcd81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ script: python -m doctr deploy --sync --key-path deploy_key.enc stash-test/ --built-docs test; # Test syncing a single file echo `date` >> test-file - python -m doctr deploy --sync --key-path deploy_key.enc test-file --built-docs test-file; + python -m doctr deploy --sync --key-path deploy_key.enc . --built-docs test-file; # Test deploy branch creation. Delete the branch gh-pages-testing on drdoctr/doctr whenever you want to test this. python -m doctr deploy --sync --key-path deploy_key.enc --no-require-master --deploy-branch gh-pages-testing docs; # Test pushing to .github.io diff --git a/doctr/tests/test_travis.py b/doctr/tests/test_travis.py index 12a8e701..81286cec 100644 --- a/doctr/tests/test_travis.py +++ b/doctr/tests/test_travis.py @@ -221,57 +221,14 @@ def test_sync_from_log(src, dst): os.chdir(old_curdir) -def test_sync_from_log_file_to_file(): +@pytest.mark.parametrize("dst", ['dst', 'dst/']) +def test_sync_from_log_file_to_dir(dst): with tempfile.TemporaryDirectory() as dir: try: old_curdir = os.path.abspath(os.curdir) os.chdir(dir) - # src is a file and dst doesn't exist as a directory or end in /, - # so should sync to a file called 'dst' src = 'file' - dst = 'dst' - - with open(src, 'w') as f: - f.write('test1') - - # Test that the sync happens - added, removed = sync_from_log(src, dst, 'logfile') - - assert added == [ - 'dst', - 'logfile', - ] - - assert removed == [] - - # Make sure dst is a file - with open(dst) as f: - assert f.read() == 'test1' - - - with open('logfile') as f: - assert f.read() == '\n'.join([ - dst, - ]) - - # Now make sure that the dst is created as a directory if it - # ends in / - - finally: - os.chdir(old_curdir) - - -def test_sync_from_log_file_to_dir(): - with tempfile.TemporaryDirectory() as dir: - try: - old_curdir = os.path.abspath(os.curdir) - os.chdir(dir) - - # src is a file but dst ends in / (even though it doesn't exist), - # so it should sync to dst/file - src = 'file' - dst = 'dst/' with open(src, 'w') as f: f.write('test1') diff --git a/doctr/travis.py b/doctr/travis.py index d54dd4aa..27f46b64 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -447,19 +447,9 @@ def sync_from_log(src, dst, log_file, exclude=()): files = glob.iglob(join(src, '**'), recursive=True) else: files = [src] + src = os.path.dirname(src) + os.sep if os.sep in src else '' - if dst.endswith(os.sep): - os.makedirs(dst, exist_ok=True) - elif not isdir(dst) and os.sep not in dst: - # src is a file and dst doesn't end in / or exist as a directory, - # so make it a file - shutil.copy2(src, dst) - added.append(dst) - if dst in removed: - removed.remove(dst) - files = [] - - src = os.path.dirname(src) + os.sep if os.path.isabs(src) else '' + os.makedirs(dst, exist_ok=True) # sorted makes this easier to test for f in sorted(files):