From f091d4791d53d8744c0e5a55ff6bb67cea854c98 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 13 Mar 2017 18:57:31 -0700 Subject: [PATCH 1/7] Use travis.yml as a config file - Change require master to branch_whitelist --- .travis.yml | 15 ++++-- docs/changelog.rst | 8 +++- docs/commandline.rst | 39 +++++++++++++++ doctr/__main__.py | 110 +++++++++++++++++++++++++++++++++++++++---- doctr/travis.py | 16 ++++++- setup.py | 1 + 6 files changed, 173 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e921986..b9734eef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ install: - conda config --add channels conda-forge # For sphinxcontrib.autoprogram - conda update -q conda - conda info -a - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION requests cryptography sphinx pyflakes sphinxcontrib-autoprogram pytest sphinx-issues + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION requests cryptography sphinx pyflakes sphinxcontrib-autoprogram pytest sphinx-issues pyyaml - source activate test-environment script: @@ -31,12 +31,17 @@ script: cd docs; make html; cd ..; - python -m doctr deploy --key-path deploy_key.enc .; - python -m doctr deploy --key-path deploy_key.enc --gh-pages-docs docs; - python -m doctr deploy --no-require-master --built-docs docs/_build/html --key-path deploy_key.enc "docs-$TRAVIS_BRANCH"; - python -m doctr deploy --no-require-master --key-path deploy_key.enc --no-sync --command "echo test" docs; + python -m doctr deploy --sync .; + python -m doctr deploy --sync --gh-pages-docs docs; + python -m doctr deploy --sync --no-require-master --built-docs docs/_build/html "docs-$TRAVIS_BRANCH"; + python -m doctr deploy --no-require-master --command "echo test" docs; fi - if [[ "${TESTS}" == "true" ]]; then pyflakes doctr; py.test doctr; fi +doctr: + key-path: deploy_key.enc + require-master: true + gh-pages-docs: 'docs' + sync: False diff --git a/docs/changelog.rst b/docs/changelog.rst index 526b58a1..4f41c8c9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,13 @@ Current - The ``--gh-pages-docs`` flag of ``doctr deploy`` has been deprecated. Specify the deploy directory like ``doctr deploy .`` or ``doctr deploy docs``. There is also no longer a default deploy directory. (:issue:`128`) - +- ``setup_GitHub_push`` now takes a ``branch_whitelist`` parameter instead of + of a ``require_master`` +- ``.travis.yml`` can be used to store some of doctr configuration in addition + to the command line flags. Write doctr configuration under the ``doctr`` key. + :issue:`137` +- All boolean command line flags now have a their counterpart that can overwrite + the config values set in ``.travis.yml`` 1.4.1 (2017-01-11) ================== diff --git a/docs/commandline.rst b/docs/commandline.rst index facf703c..53080365 100644 --- a/docs/commandline.rst +++ b/docs/commandline.rst @@ -4,3 +4,42 @@ .. autoprogram:: doctr.__main__:get_parser() :prog: doctr + + +Configuration +------------- + +In addition to command line arguments you can configure ``doctr`` using the +``.travis.yml`` files. Command line arguments take precedence over the value +present in the configuration file. + +The configuration parameters available from the ``.travis.yml`` file mirror +their command line siblings except doubledashes ``--`` and ``--no-`` prefix are +dropped. + +Use a ``doctr`` section in your ``travis.yml`` file to store your doctr +configuration: + +.. code:: yaml + + - language: python + - script: + - set -e + - py.test + - cd docs + - make html + - cd .. + - doctr deploy + - doctr: + - key-path : 'path/to/key/from/repo/root/path.key' + - deploy-repo : 'myorg/myrepo' + + +The following options are available from the configuration file and not from +the command line: + +``branches``: + A list of regular expression that matches branches on which ``doctr`` should + still deploy the documentation. For example ``.*\.x`` will deploy all + branches like ``3.x``, ``4.x`` ... + diff --git a/doctr/__main__.py b/doctr/__main__.py index 6a9a73bf..d2ba4351 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -23,9 +23,13 @@ import sys import os +import os.path import argparse import shlex import subprocess +import yaml + +from pathlib import Path from textwrap import dedent @@ -35,7 +39,65 @@ get_current_repo, sync_from_log, find_sphinx_build_dir, run) from . import __version__ -def get_parser(): +def make_parser_with_config_adder(parser, config): + """factory function for a smarter parser: + + return an utility function that pull default from the config as well. + + Pull the default for parser not only from the ``default`` kwarg, + but also if an identical value is find in ``config`` where leading + ``--`` or ``--no`` is removed. + + If the option is a boolean flag, automatically register an opposite, + exclusive option by prepending or removing the `--no-`. This is useful + to overwrite config in ``.travis.yml`` + """ + + def internal(arg, **kwargs): + invert = { + 'store_true':'store_false', + 'store_false':'store_true', + } + if arg.startswith('--no-'): + key = arg[5:] + else: + key = arg[2:] + if 'default' in kwargs: + old_default = kwargs['default'] + kwargs['default'] = config.get(key, old_default) + action = kwargs.get('action') + if action in invert: + exclusive_grp = parser.add_mutually_exclusive_group() + exclusive_grp.add_argument(arg, **kwargs) + kwargs['action'] = invert[action] + kwargs['help'] = 'Inverse of "%s"' % arg + if arg.startswith('--no-'): + arg = '--%s' % arg[5:] + else: + arg = '--no-%s' % arg[2:] + exclusive_grp.add_argument(arg, **kwargs) + else: + parser.add_argument(arg, **kwargs) + + return internal + + +def get_parser(config=None): + """ + return a parser suitable to parse CL arguments. + + Parameters + ---------- + + config: dict + Default values to fall back on, if not given. + + Returns + ------- + + An argparse parser configured to parse the command lines arguments of + sys.argv which will default on values present in ``config``. + """ # This uses RawTextHelpFormatter so that the description (the docstring of # this module) is formatted correctly. Unfortunately, that means that # parser help is not text wrapped (but all other help is). @@ -45,15 +107,21 @@ def get_parser(): options available. """, ) + + if not config: + config={} parser.add_argument('-V', '--version', action='version', version='doctr ' + __version__) subcommand = parser.add_subparsers(title='subcommand', dest='subcommand') + deploy_parser = subcommand.add_parser('deploy', help="""Deploy the docs to GitHub from Travis.""") deploy_parser.set_defaults(func=deploy) - deploy_parser.add_argument('deploy_directory', type=str, nargs='?', - help="""Directory to deploy the html documentation to on gh-pages.""") + deploy_parser.add_argument = make_parser_with_config_adder(deploy_parser, config) deploy_parser.add_argument('--force', action='store_true', help="""Run the deploy command even if we do not appear to be on Travis.""") + deploy_parser_add_argument('deploy_directory', type=str, nargs='?', + help="""Directory to deploy the html documentation to on gh-pages.""") + deploy_parser.add_argument('--token', action='store_true', default=False, help="""Push to GitHub using a personal access token. Use this if you used 'doctr configure --token'.""") @@ -62,6 +130,9 @@ def get_parser(): deploy_parser.add_argument('--built-docs', default=None, help="""Location of the built html documentation to be deployed to gh-pages. If not specified, Doctr will try to automatically detect build location""") + deploy_parser.add_argument('--gh-pages-docs', default='docs', + help="""Directory to deploy the html documentation to on gh-pages. The + default is %(default)r.""") deploy_parser.add_argument('--tmp-dir', default=None, help=argparse.SUPPRESS) deploy_parser.add_argument('--deploy-repo', default=None, help="""Repo to @@ -78,7 +149,7 @@ def get_parser(): the files for you. Any files you wish to commit should be added to the index.""") deploy_parser.add_argument('--no-push', dest='push', action='store_false', - default=True, help="Run all the steps except the last push step." + default=True, help="Run all the steps except the last push step. " "Useful for debugging") deploy_parser.add_argument('--gh-pages-docs', default=None, help="""!!DEPRECATED!! Directory to deploy the html documentation to on gh-pages. @@ -86,7 +157,6 @@ def get_parser(): the first argument to 'doctr deploy'. This flag is kept for backwards compatibility.""") - configure_parser = subcommand.add_parser('configure', help="Configure doctr. This command should be run locally (not on Travis).") configure_parser.set_defaults(func=configure) configure_parser.add_argument('--force', action='store_true', help="""Run the configure command even @@ -106,6 +176,23 @@ def get_parser(): return parser +def get_config(): + """ + This load some configuration from the ``.travis.yml``, if file is present, + ``doctr`` key if present. + """ + p = Path('.travis.yml') + if not p.exists(): + return {} + with p.open() as f: + travis_config = yaml.safe_load(f.read()) + + config = travis_config.get('doctr', {}) + + if not isinstance(config, dict): + raise ValueError('config is not a dict: {}'.format(config)) + return config + def process_args(parser): args = parser.parse_args() @@ -126,6 +213,8 @@ def deploy(args, parser): parser.error("doctr does not appear to be running on Travis. Use " "doctr deploy --force to run anyway.") + config = get_config() + if args.tmp_dir: parser.error("The --tmp-dir flag has been removed (doctr no longer uses a temporary directory when deploying).") @@ -138,16 +227,20 @@ def deploy(args, parser): if not args.gh_pages_docs and not args.deploy_directory: parser.error("No deploy directory specified. Specify the directory to deploy to using `doctr deploy `") - deploy_dir = args.gh_pages_docs or args.deploy_directory + deploy_dir = args.gh_pages_docs or args.deploy_directory or config.get('deploy_directory') build_repo = get_current_repo() deploy_repo = args.deploy_repo or build_repo current_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf-8').strip() try: + + branch_whitelist = {'master'} if args.require_master else set({}) + branch_whitelist.update(set(config.get('branches',set({})))) + can_push = setup_GitHub_push(deploy_repo, auth_type='token' if args.token else 'deploy_key', full_key_path=args.key_path, - require_master=args.require_master) + branch_whitelist=branch_whitelist) if args.sync: built_docs = args.built_docs or find_sphinx_build_dir() @@ -284,7 +377,8 @@ def configure(args, parser): """.format(encrypted_variable=encrypted_variable.decode('utf-8'), N=N))) def main(): - return process_args(get_parser()) + config = get_config() + return process_args(get_parser(config=config)) if __name__ == '__main__': sys.exit(main()) diff --git a/doctr/travis.py b/doctr/travis.py index 3011199a..6ef9dcca 100644 --- a/doctr/travis.py +++ b/doctr/travis.py @@ -8,6 +8,7 @@ import subprocess import sys import glob +import re from cryptography.fernet import Fernet @@ -132,7 +133,7 @@ def get_current_repo(): _, org, git_repo = remote_url.rsplit('.git', 1)[0].rsplit('/', 2) return (org + '/' + git_repo) -def setup_GitHub_push(deploy_repo, auth_type='deploy_key', full_key_path='github_deploy_key.enc', require_master=True): +def setup_GitHub_push(deploy_repo, auth_type='deploy_key', full_key_path='github_deploy_key.enc', require_master=None, branch_whitelist=None): """ Setup the remote to push to GitHub (to be run on Travis). @@ -144,6 +145,17 @@ def setup_GitHub_push(deploy_repo, auth_type='deploy_key', full_key_path='github For ``auth_type='deploy_key'``, this sets up the remote with ssh access. """ + + if branch_whitelist is None: + branch_whitelist={'master'} + + if require_master is not None: + import warnings + warnings.warn("`setup_GitHub_push`'s `require_master` argument in favor of `branch_whitelist=['master']`", + DeprecationWarning, + stacklevel=2) + branch_whitelist.add('master') + canpush = True if auth_type not in ['deploy_key', 'token']: raise ValueError("auth_type must be 'deploy_key' or 'token'") @@ -151,7 +163,7 @@ def setup_GitHub_push(deploy_repo, auth_type='deploy_key', full_key_path='github TRAVIS_BRANCH = os.environ.get("TRAVIS_BRANCH", "") TRAVIS_PULL_REQUEST = os.environ.get("TRAVIS_PULL_REQUEST", "") - if TRAVIS_BRANCH != "master" and require_master: + if any([re.compile(x).match(TRAVIS_BRANCH) for x in branch_whitelist]): print("The docs are only pushed to gh-pages from master. To allow pushing from " "a non-master branch, use the --no-require-master flag", file=sys.stderr) print("This is the {TRAVIS_BRANCH} branch".format(TRAVIS_BRANCH=TRAVIS_BRANCH), file=sys.stderr) diff --git a/setup.py b/setup.py index 7ec8b44e..5d7ae1ec 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ entry_points={'console_scripts': [ 'doctr = doctr.__main__:main']}, python_requires= '>=3.5', install_requires=[ + 'pyyaml', 'requests', 'cryptography', ], From 84e29e946932745e6170cb2e3dae0eca95e3469f Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 13 Mar 2017 19:07:01 -0700 Subject: [PATCH 2/7] 'rename gh-pages-docs to deploy-directory' --- .travis.yml | 2 +- doctr/__main__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b9734eef..2c0a2c11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,5 +43,5 @@ script: doctr: key-path: deploy_key.enc require-master: true - gh-pages-docs: 'docs' + deploy-directory: 'docs' sync: False diff --git a/doctr/__main__.py b/doctr/__main__.py index d2ba4351..541d588a 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -227,7 +227,7 @@ def deploy(args, parser): if not args.gh_pages_docs and not args.deploy_directory: parser.error("No deploy directory specified. Specify the directory to deploy to using `doctr deploy `") - deploy_dir = args.gh_pages_docs or args.deploy_directory or config.get('deploy_directory') + deploy_dir = args.gh_pages_docs or args.deploy_directory or config.get('deploy-directory') build_repo = get_current_repo() deploy_repo = args.deploy_repo or build_repo From 0a4bf028debe7720740ff50116808f6a003c5d43 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 13 Mar 2017 19:04:49 -0700 Subject: [PATCH 3/7] 'replace . with _ in deploy_parser_add_argument' --- doctr/__main__.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doctr/__main__.py b/doctr/__main__.py index 541d588a..aebedc15 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -116,42 +116,42 @@ def get_parser(config=None): deploy_parser = subcommand.add_parser('deploy', help="""Deploy the docs to GitHub from Travis.""") deploy_parser.set_defaults(func=deploy) - deploy_parser.add_argument = make_parser_with_config_adder(deploy_parser, config) - deploy_parser.add_argument('--force', action='store_true', help="""Run the deploy command even + deploy_parser_add_argument = make_parser_with_config_adder(deploy_parser, config) + deploy_parser_add_argument('--force', action='store_true', help="""Run the deploy command even if we do not appear to be on Travis.""") deploy_parser_add_argument('deploy_directory', type=str, nargs='?', help="""Directory to deploy the html documentation to on gh-pages.""") - deploy_parser.add_argument('--token', action='store_true', default=False, + deploy_parser_add_argument('--token', action='store_true', default=False, help="""Push to GitHub using a personal access token. Use this if you used 'doctr configure --token'.""") - deploy_parser.add_argument('--key-path', default='github_deploy_key.enc', + deploy_parser_add_argument('--key-path', default='github_deploy_key.enc', help="""Path of the encrypted GitHub deploy key. The default is %(default)r.""") - deploy_parser.add_argument('--built-docs', default=None, + deploy_parser_add_argument('--built-docs', default=None, help="""Location of the built html documentation to be deployed to gh-pages. If not specified, Doctr will try to automatically detect build location""") - deploy_parser.add_argument('--gh-pages-docs', default='docs', + deploy_parser_add_argument('--gh-pages-docs', default='docs', help="""Directory to deploy the html documentation to on gh-pages. The default is %(default)r.""") - deploy_parser.add_argument('--tmp-dir', default=None, + deploy_parser_add_argument('--tmp-dir', default=None, help=argparse.SUPPRESS) - deploy_parser.add_argument('--deploy-repo', default=None, help="""Repo to + deploy_parser_add_argument('--deploy-repo', default=None, help="""Repo to deploy the docs to. By default, it deploys to the repo Doctr is run from.""") - deploy_parser.add_argument('--no-require-master', dest='require_master', action='store_false', + deploy_parser_add_argument('--no-require-master', dest='require_master', action='store_false', default=True, help="""Allow docs to be pushed from a branch other than master""") - deploy_parser.add_argument('--command', default=None, help="""Command to + deploy_parser_add_argument('--command', default=None, help="""Command to be run before committing and pushing. If the command creates additional files that should be deployed, they should be added to the index.""") - deploy_parser.add_argument('--no-sync', dest='sync', action='store_false', + deploy_parser_add_argument('--no-sync', dest='sync', action='store_false', default=True, help="""Don't sync any files. This is generally used in conjunction with the --command flag, for instance, if the command syncs the files for you. Any files you wish to commit should be added to the index.""") - deploy_parser.add_argument('--no-push', dest='push', action='store_false', + deploy_parser_add_argument('--no-push', dest='push', action='store_false', default=True, help="Run all the steps except the last push step. " "Useful for debugging") - deploy_parser.add_argument('--gh-pages-docs', default=None, + deploy_parser_add_argument('--gh-pages-docs', default=None, help="""!!DEPRECATED!! Directory to deploy the html documentation to on gh-pages. The default is %(default)r. The deploy directory should be passed as the first argument to 'doctr deploy'. This flag is kept for backwards From edea11afd0f23c05301134f530ccbe91307463e2 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 13 Mar 2017 19:13:29 -0700 Subject: [PATCH 4/7] Remove deploy-directory/gh-pages-docs from config --- .travis.yml | 1 - doctr/__main__.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2c0a2c11..16a206f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,5 +43,4 @@ script: doctr: key-path: deploy_key.enc require-master: true - deploy-directory: 'docs' sync: False diff --git a/doctr/__main__.py b/doctr/__main__.py index aebedc15..a140f8ed 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -130,9 +130,6 @@ def get_parser(config=None): deploy_parser_add_argument('--built-docs', default=None, help="""Location of the built html documentation to be deployed to gh-pages. If not specified, Doctr will try to automatically detect build location""") - deploy_parser_add_argument('--gh-pages-docs', default='docs', - help="""Directory to deploy the html documentation to on gh-pages. The - default is %(default)r.""") deploy_parser_add_argument('--tmp-dir', default=None, help=argparse.SUPPRESS) deploy_parser_add_argument('--deploy-repo', default=None, help="""Repo to @@ -227,7 +224,7 @@ def deploy(args, parser): if not args.gh_pages_docs and not args.deploy_directory: parser.error("No deploy directory specified. Specify the directory to deploy to using `doctr deploy `") - deploy_dir = args.gh_pages_docs or args.deploy_directory or config.get('deploy-directory') + deploy_dir = args.gh_pages_docs or args.deploy_directory build_repo = get_current_repo() deploy_repo = args.deploy_repo or build_repo From eb62a70159cf370f38cc03171579155e1b239c2a Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 13 Mar 2017 19:13:57 -0700 Subject: [PATCH 5/7] Mutate config object to find unused options. --- doctr/__main__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/doctr/__main__.py b/doctr/__main__.py index a140f8ed..242c4237 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -51,6 +51,9 @@ def make_parser_with_config_adder(parser, config): If the option is a boolean flag, automatically register an opposite, exclusive option by prepending or removing the `--no-`. This is useful to overwrite config in ``.travis.yml`` + + Mutate the config object and remove know keys in order to detect unused + options afterwoard. """ def internal(arg, **kwargs): @@ -63,8 +66,9 @@ def internal(arg, **kwargs): else: key = arg[2:] if 'default' in kwargs: - old_default = kwargs['default'] - kwargs['default'] = config.get(key, old_default) + if key in config: + kwargs['default'] = config[key] + del config[key] action = kwargs.get('action') if action in invert: exclusive_grp = parser.add_mutually_exclusive_group() @@ -154,6 +158,9 @@ def get_parser(config=None): the first argument to 'doctr deploy'. This flag is kept for backwards compatibility.""") + if config: + print('warning, following option in `.travis.yml` not recognised', config) + configure_parser = subcommand.add_parser('configure', help="Configure doctr. This command should be run locally (not on Travis).") configure_parser.set_defaults(func=configure) configure_parser.add_argument('--force', action='store_true', help="""Run the configure command even From be9d76937f19807c8920f0685e33d4804831fb41 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 13 Mar 2017 19:16:28 -0700 Subject: [PATCH 6/7] fix docs forgotten now mandatory deploy-directory --- docs/commandline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commandline.rst b/docs/commandline.rst index 53080365..58cef768 100644 --- a/docs/commandline.rst +++ b/docs/commandline.rst @@ -29,7 +29,7 @@ configuration: - cd docs - make html - cd .. - - doctr deploy + - doctr deploy . - doctr: - key-path : 'path/to/key/from/repo/root/path.key' - deploy-repo : 'myorg/myrepo' From f3e5697a5dfa06e411ca2e7631fdef9d06917465 Mon Sep 17 00:00:00 2001 From: Matthias Bussonnier Date: Mon, 13 Mar 2017 19:20:56 -0700 Subject: [PATCH 7/7] Pretty print non-regcognized options in travis.yml. --- .travis.yml | 1 + doctr/__main__.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 16a206f8..c52c2474 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,3 +44,4 @@ doctr: key-path: deploy_key.enc require-master: true sync: False + lubalubadubdub: False diff --git a/doctr/__main__.py b/doctr/__main__.py index 242c4237..bf4f7c71 100644 --- a/doctr/__main__.py +++ b/doctr/__main__.py @@ -28,6 +28,7 @@ import shlex import subprocess import yaml +import json from pathlib import Path @@ -159,7 +160,7 @@ def get_parser(config=None): compatibility.""") if config: - print('warning, following option in `.travis.yml` not recognised', config) + print('Warning, The following options in `.travis.yml` were not recognised:\n%s' % json.dumps(config, indent=2)) configure_parser = subcommand.add_parser('configure', help="Configure doctr. This command should be run locally (not on Travis).") configure_parser.set_defaults(func=configure)