From 1a3e6a100b07172f5d6df8f6e95ef8bc021bdd66 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sat, 15 Jul 2017 10:21:38 -0500 Subject: [PATCH 1/7] Copy the --built-docs to a temporary directory This makes it possible to deploy committed files, and avoids issues with checkout out gh-pages if tracked files would be modified by checkout. Fixes #202 Fixes #214 Closes #211 --- doctr/__init__.py | 7 ++++--- doctr/__main__.py | 9 ++++++++- doctr/travis.py | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/doctr/__init__.py b/doctr/__init__.py index 5245b4e6..dad31e5b 100644 --- a/doctr/__init__.py +++ b/doctr/__init__.py @@ -2,8 +2,9 @@ generate_GitHub_token, upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists) from .travis import (decrypt_file, setup_deploy_key, get_token, run, - setup_GitHub_push, deploy_branch_exists, create_deploy_branch, sync_from_log, - commit_docs, push_docs, get_current_repo, find_sphinx_build_dir) + setup_GitHub_push, deploy_branch_exists, create_deploy_branch, + copy_to_tmp, sync_from_log, commit_docs, push_docs, get_current_repo, + find_sphinx_build_dir) __all__ = [ 'encrypt_variable', 'encrypt_file', 'GitHub_post', @@ -11,7 +12,7 @@ 'check_repo_exists', 'decrypt_file', 'setup_deploy_key', 'get_token', 'run', 'setup_GitHub_push', 'deploy_branch_exists', - 'create_deploy_branch', 'sync_from_log', 'commit_docs', 'push_docs', 'get_current_repo', 'find_sphinx_build_dir' + 'create_deploy_branch', 'copy_to_tmp', 'sync_from_log', 'commit_docs', 'push_docs', 'get_current_repo', 'find_sphinx_build_dir' ] from ._version import get_versions diff --git a/doctr/__main__.py b/doctr/__main__.py index 22871182..167446a6 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -36,7 +36,8 @@ from .local import (generate_GitHub_token, encrypt_variable, encrypt_file, upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists, GitHub_login) from .travis import (setup_GitHub_push, commit_docs, push_docs, - get_current_repo, sync_from_log, find_sphinx_build_dir, run, get_travis_branch) + get_current_repo, sync_from_log, find_sphinx_build_dir, run, + get_travis_branch, copy_to_tmp) from . import __version__ def make_parser_with_config_adder(parser, config): @@ -152,6 +153,9 @@ def get_parser(config=None): conjunction with the --command flag, for instance, if the command syncs the files for you. Any files you wish to commit should be added to the index.""") + deploy_parser.add_argument('--no-temp-dir', dest='temp_dir', + action='store_false', default=True, help="""Don't copy the + --built-docs directory to a temporary directory.""") deploy_parser_add_argument('--no-push', dest='push', action='store_false', default=True, help="Run all the steps except the last push step. " "Useful for debugging") @@ -161,6 +165,7 @@ def get_parser(config=None): the first argument to 'doctr deploy'. This flag is kept for backwards compatibility.""") + if config: print('Warning, The following options in `.travis.yml` were not recognized:\n%s' % json.dumps(config, indent=2)) @@ -256,6 +261,8 @@ def deploy(args, parser): if args.sync: built_docs = args.built_docs or find_sphinx_build_dir() + if args.temp_dir: + built_docs = copy_to_tmp(built_docs) log_file = os.path.join(deploy_dir, '.doctr-files') diff --git a/doctr/travis.py b/doctr/travis.py index bf073c27..d5c42588 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -9,6 +9,8 @@ import sys import glob import re +import pathlib +import tempfile from cryptography.fernet import Fernet @@ -294,6 +296,19 @@ def find_sphinx_build_dir(): # TRAVIS_JOB_NUMBER = os.environ.get("TRAVIS_JOB_NUMBER", '') # ACTUAL_TRAVIS_JOB_NUMBER = TRAVIS_JOB_NUMBER.split('.')[1] +def copy_to_tmp(directory): + """ + Copies the contents of directory to a temporary directory, and returns the + copied location. + """ + tmp_dir = tempfile.mkdtemp() + # Use pathlib because os.path.basename is different depending on whether + # the path ends in a / + p = pathlib.Path(directory) + new_dir = os.path.join(tmp_dir, p.name) + shutil.copytree(directory, new_dir) + return new_dir + def sync_from_log(src, dst, log_file): """ Sync the files in ``src`` to ``dst``. From 8ec983d02d3020094ec62a63c1d5ec946c3dc1fe Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sat, 15 Jul 2017 10:36:19 -0500 Subject: [PATCH 2/7] Run the command before syncing That way it can affect the build docs files before they are synced. --- doctr/__main__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doctr/__main__.py b/doctr/__main__.py index 167446a6..b3c80f36 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -259,6 +259,11 @@ def deploy(args, parser): full_key_path=args.key_path, branch_whitelist=branch_whitelist) + if args.command: + run(['git', 'checkout', get_travis_branch()]) + run(args.command, shell=True) + run(['git', 'checkout', deploy_branch]) + if args.sync: built_docs = args.built_docs or find_sphinx_build_dir() if args.temp_dir: @@ -273,11 +278,6 @@ def deploy(args, parser): else: added, removed = [], [] - if args.command: - run(['git', 'checkout', get_travis_branch()]) - run(args.command, shell=True) - run(['git', 'checkout', deploy_branch]) - changes = commit_docs(added=added, removed=removed) if changes: if can_push and args.push: From 925406546a2cbe4d9bb3be520c8b2f335639d0d0 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sat, 15 Jul 2017 10:37:45 -0500 Subject: [PATCH 3/7] Add a test for the --command creating a file that should be synced --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 94e13f47..40863080 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ script: python -m doctr deploy --sync .; python -m doctr deploy --sync --gh-pages-docs docs; python -m doctr deploy --sync --no-require-master --built-docs docs/_build/html "docs-$TRAVIS_BRANCH"; - python -m doctr deploy --no-require-master --command "echo test; ls" docs; + python -m doctr deploy --no-require-master --command "echo test; ls; touch docs/_build/html/test" docs; fi - if [[ "${TESTS}" == "true" ]]; then pyflakes doctr; From 1ac8d71574fff7f3d4c8853e1305103af9de5d1e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sat, 15 Jul 2017 10:43:25 -0500 Subject: [PATCH 4/7] Use same variable name spelling across files --- doctr/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doctr/__main__.py b/doctr/__main__.py index b3c80f36..50960550 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -254,7 +254,7 @@ def deploy(args, parser): branch_whitelist = {'master'} if args.require_master else set(get_travis_branch()) branch_whitelist.update(set(config.get('branches',set({})))) - can_push = setup_GitHub_push(deploy_repo, deploy_branch=deploy_branch, + canpush = setup_GitHub_push(deploy_repo, deploy_branch=deploy_branch, auth_type='token' if args.token else 'deploy_key', full_key_path=args.key_path, branch_whitelist=branch_whitelist) @@ -280,7 +280,7 @@ def deploy(args, parser): changes = commit_docs(added=added, removed=removed) if changes: - if can_push and args.push: + if canpush and args.push: push_docs(deploy_branch) else: print("Don't have permission to push. Not trying.") From ca2d8c63d084aa54734f2b7f48d4321a295bfd41 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sat, 15 Jul 2017 10:46:42 -0500 Subject: [PATCH 5/7] Separate out checking out the deploy branch into a separate function That way we can run the --command in the base branch without checking out back and forth, and we can do it before syncing and copying to the temporary directory (next commit). --- doctr/__init__.py | 9 +++++---- doctr/__main__.py | 6 +++--- doctr/travis.py | 8 ++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/doctr/__init__.py b/doctr/__init__.py index dad31e5b..38687811 100644 --- a/doctr/__init__.py +++ b/doctr/__init__.py @@ -2,16 +2,17 @@ generate_GitHub_token, upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists) from .travis import (decrypt_file, setup_deploy_key, get_token, run, - setup_GitHub_push, deploy_branch_exists, create_deploy_branch, - copy_to_tmp, sync_from_log, commit_docs, push_docs, get_current_repo, - find_sphinx_build_dir) + setup_GitHub_push, checkout_deploy_branch, deploy_branch_exists, + create_deploy_branch, copy_to_tmp, sync_from_log, commit_docs, push_docs, + get_current_repo, find_sphinx_build_dir) __all__ = [ 'encrypt_variable', 'encrypt_file', 'GitHub_post', 'generate_GitHub_token', 'upload_GitHub_deploy_key', 'generate_ssh_key', 'check_repo_exists', - 'decrypt_file', 'setup_deploy_key', 'get_token', 'run', 'setup_GitHub_push', 'deploy_branch_exists', + 'decrypt_file', 'setup_deploy_key', 'get_token', 'run', + 'setup_GitHub_push', 'checkout_deploy_branch', 'deploy_branch_exists', 'create_deploy_branch', 'copy_to_tmp', 'sync_from_log', 'commit_docs', 'push_docs', 'get_current_repo', 'find_sphinx_build_dir' ] diff --git a/doctr/__main__.py b/doctr/__main__.py index 50960550..52bba801 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -37,7 +37,7 @@ upload_GitHub_deploy_key, generate_ssh_key, check_repo_exists, GitHub_login) from .travis import (setup_GitHub_push, commit_docs, push_docs, get_current_repo, sync_from_log, find_sphinx_build_dir, run, - get_travis_branch, copy_to_tmp) + get_travis_branch, copy_to_tmp, checkout_deploy_branch) from . import __version__ def make_parser_with_config_adder(parser, config): @@ -260,9 +260,9 @@ def deploy(args, parser): branch_whitelist=branch_whitelist) if args.command: - run(['git', 'checkout', get_travis_branch()]) run(args.command, shell=True) - run(['git', 'checkout', deploy_branch]) + + checkout_deploy_branch(deploy_branch, canpush=canpush) if args.sync: built_docs = args.built_docs or find_sphinx_build_dir() diff --git a/doctr/travis.py b/doctr/travis.py index d5c42588..af31bfce 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -220,6 +220,12 @@ def setup_GitHub_push(deploy_repo, auth_type='deploy_key', full_key_path='github print("Fetching doctr remote") run(['git', 'fetch', 'doctr_remote']) + return canpush + +def checkout_deploy_branch(deploy_branch, canpush=True): + """ + Checkout the deploy branch, creating it if it doesn't exist. + """ #create empty branch with .nojekyll if it doesn't already exist new_deploy_branch = create_deploy_branch(deploy_branch, push=canpush) print("Checking out {}".format(deploy_branch)) @@ -230,8 +236,6 @@ def setup_GitHub_push(deploy_repo, auth_type='deploy_key', full_key_path='github run(['git', 'checkout', '-b', deploy_branch, '--track', 'doctr_remote/{}'.format(deploy_branch)]) print("Done") - return canpush - def deploy_branch_exists(deploy_branch): """ Check if there is a remote branch with name specified in ``deploy_branch``. From 38f9fe0a3973d7edb75e529a0aa324b64486f8a4 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sat, 15 Jul 2017 10:48:25 -0500 Subject: [PATCH 6/7] Copy files to the temporary directory before checking out the deploy branch --- doctr/__main__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doctr/__main__.py b/doctr/__main__.py index 52bba801..f5d1e65c 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -262,13 +262,14 @@ def deploy(args, parser): if args.command: run(args.command, shell=True) - checkout_deploy_branch(deploy_branch, canpush=canpush) - if args.sync: built_docs = args.built_docs or find_sphinx_build_dir() if args.temp_dir: built_docs = copy_to_tmp(built_docs) + checkout_deploy_branch(deploy_branch, canpush=canpush) + + if args.sync: log_file = os.path.join(deploy_dir, '.doctr-files') print("Moving built docs into place") From e377b6d328e8501eda3f0bc694a9760b71687723 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sat, 15 Jul 2017 10:50:23 -0500 Subject: [PATCH 7/7] Run git reset --hard before checking out the deploy branch --- doctr/__main__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doctr/__main__.py b/doctr/__main__.py index f5d1e65c..e734bc18 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -267,6 +267,9 @@ def deploy(args, parser): if args.temp_dir: built_docs = copy_to_tmp(built_docs) + # Reset in case there are modified files that are tracked in the + # dpeloy branch. + run(['git', 'reset', '--hard']) checkout_deploy_branch(deploy_branch, canpush=canpush) if args.sync: