Skip to content

Add support for multiple deploy key and GitHub Wiki #276

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ylemkimon wants to merge 3 commits into from
Closed

Add support for multiple deploy key and GitHub Wiki #276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0a5402c
Add support for deployment to GitHub wiki
ylemkimon Nov 10, 2017
df056ce
Add support for multiple deploy key encryption key
ylemkimon Nov 13, 2017
9971dbd
Fix Sphinx autoprogram error
ylemkimon Nov 13, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 41 additions & 18 deletions doctr/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,15 +131,16 @@ def get_parser(config=None):
deploy_parser_add_argument('--token', action='store_true', default=False,
help="""Push to GitHub using a personal access token. Use this if you
used 'doctr configure --token'.""")
deploy_parser_add_argument('--key-path', default='github_deploy_key.enc',
help="""Path of the encrypted GitHub deploy key. The default is %(default)r.""")
deploy_parser_add_argument('--key-path', default=None,
help="""Path of the encrypted GitHub deploy key. The default is github_deploy_key_+
deploy respository name + .enc.""")
deploy_parser_add_argument('--built-docs', default=None,
help="""Location of the built html documentation to be deployed to gh-pages. If not
specified, Doctr will try to automatically detect build location
(right now only works for Sphinx docs).""")
deploy_parser.add_argument('--deploy-branch-name', default=None,
help="""Name of the branch to deploy to (default: 'master' for ``*.github.io``
repos, 'gh-pages' otherwise)""")
and wiki repos, 'gh-pages' otherwise)""")
deploy_parser_add_argument('--tmp-dir', default=None,
help=argparse.SUPPRESS)
deploy_parser_add_argument('--deploy-repo', default=None, help="""Repo to
Expand Down Expand Up @@ -185,9 +186,9 @@ def get_parser(config=None):
dest="upload_key", help="""Don't automatically upload the deploy key to GitHub. If you select this
option, you will not be prompted for your GitHub credentials, so this option is not compatible with
private repositories.""")
configure_parser.add_argument('--key-path', default='github_deploy_key',
help="""Path to save the encrypted GitHub deploy key. The default is %(default)r.
The .enc extension is added to the file automatically.""")
configure_parser.add_argument('--key-path', default=None,
help="""Path to save the encrypted GitHub deploy key. The default is github_deploy_key_+
deploy respository name. The .enc extension is added to the file automatically.""")

return parser

Expand All @@ -208,6 +209,22 @@ def get_config():
raise ValueError('config is not a dict: {}'.format(config))
return config

def get_deploy_key_repo(deploy_repo, key_path, key_ext=''):
"""
Return (repository of which deploy key is used, environment variable to store
the encryption key of deploy key, path of deploy key file)
"""
# deploy key of the original repo has write access to the wiki
deploy_key_repo = deploy_repo[:-5] if deploy_repo.endswith('.wiki') else deploy_repo

# Automatically determine environment variable and key file name from deploy repo name
# Special characters are substituted with a hyphen(-) by GitHub
snake_case_name = deploy_key_repo.replace('-', '_').replace('.', '_').replace('/', '_').lower()
env_name = 'DOCTR_DEPLOY_KEY_' + snake_case_name.upper()
key_path = key_path or 'github_deploy_key_' + snake_case_name + key_ext

return (deploy_key_repo, env_name, key_path)

def process_args(parser):
args = parser.parse_args()

Expand Down Expand Up @@ -250,7 +267,9 @@ def deploy(args, parser):
if args.deploy_branch_name:
deploy_branch = args.deploy_branch_name
else:
deploy_branch = 'master' if deploy_repo.endswith(('.github.io', '.github.com')) else 'gh-pages'
deploy_branch = 'master' if deploy_repo.endswith(('.github.io', '.github.com', '.wiki')) else 'gh-pages'

_, env_name, key_path = get_deploy_key_repo(deploy_repo, args.key_path, key_ext='.enc')

current_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf-8').strip()
try:
Expand All @@ -259,8 +278,8 @@ def deploy(args, parser):

canpush = setup_GitHub_push(deploy_repo, deploy_branch=deploy_branch,
auth_type='token' if args.token else 'deploy_key',
full_key_path=args.key_path,
branch_whitelist=branch_whitelist)
full_key_path=key_path,
branch_whitelist=branch_whitelist, env_name=env_name)

if args.sync:
built_docs = args.built_docs or find_sphinx_build_dir()
Expand Down Expand Up @@ -369,21 +388,25 @@ def configure(args, parser):

print(header)
else:
ssh_key = generate_ssh_key("doctr deploy key for {deploy_repo}".format(deploy_repo=deploy_repo), keypath=args.key_path)
key = encrypt_file(args.key_path, delete=True)
encrypted_variable = encrypt_variable(b"DOCTR_DEPLOY_ENCRYPTION_KEY=" + key, build_repo=build_repo, is_private=is_private, **login_kwargs)
deploy_key_repo, env_name, key_path = get_deploy_key_repo(deploy_repo, args.key_path)

ssh_key = generate_ssh_key("doctr deploy key for {deploy_repo}".format(
deploy_repo=deploy_key_repo), keypath=key_path)
key = encrypt_file(key_path, delete=True)
encrypted_variable = encrypt_variable(env_name.encode('utf-8') + b"=" + key,
build_repo=build_repo, is_private=is_private, **login_kwargs)

deploy_keys_url = 'https://github.com/{deploy_repo}/settings/keys'.format(deploy_repo=deploy_repo)
deploy_keys_url = 'https://github.com/{deploy_repo}/settings/keys'.format(deploy_repo=deploy_key_repo)

if args.upload_key:

upload_GitHub_deploy_key(deploy_repo, ssh_key, **login_kwargs)
upload_GitHub_deploy_key(deploy_key_repo, ssh_key, **login_kwargs)

print(dedent("""
The deploy key has been added for {deploy_repo}.

You can go to {deploy_keys_url} to revoke the deploy key.\
""".format(deploy_repo=deploy_repo, deploy_keys_url=deploy_keys_url, keypath=args.key_path)))
""".format(deploy_repo=deploy_key_repo, deploy_keys_url=deploy_keys_url, keypath=key_path)))
print(header)
else:
print(header)
Expand All @@ -401,11 +424,11 @@ def configure(args, parser):

git add {keypath}.enc

""".format(keypath=args.key_path, N=N)))
""".format(keypath=key_path, N=N)))

options = '--built-docs path/to/built/html/'
if args.key_path != 'github_deploy_key':
options += ' --key-path {keypath}.enc'.format(keypath=args.key_path)
if args.key_path:
options += ' --key-path {keypath}.enc'.format(keypath=key_path)
if deploy_repo != build_repo:
options += ' --deploy-repo {deploy_repo}'.format(deploy_repo=deploy_repo)

Expand Down
16 changes: 15 additions & 1 deletion doctr/local.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,11 @@ def check_repo_exists(deploy_repo, service='github', *, auth=None, headers=None)
else:
raise RuntimeError('Invalid service specified for repo check (neither "travis" nor "github")')

wiki = False
if repo.endswith('.wiki') and service == 'github':
wiki = True
repo = repo[:-5]

r = requests.get(REPO_URL.format(user=user, repo=repo), auth=auth, headers=headers)

if r.status_code == requests.codes.not_found:
Expand All @@ -253,8 +258,17 @@ def check_repo_exists(deploy_repo, service='github', *, auth=None, headers=None)
service=service))

r.raise_for_status()
private = r.json().get('private', False)

if wiki and not private:
# private wiki needs authentication, so skip check for existence
p = subprocess.run(['git', 'ls-remote', '-h', 'https://github.com/{user}/{repo}.wiki'.format(
user=user, repo=repo)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)

@ylemkimon ylemkimon Nov 12, 2017

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we can do:

p = subprocess.run(['git', 'ls-remote', '-h', 'https://{auth}@github.com/{user}/{repo}.wiki'.format(
    auth=base64.urlsafe_b64encode(bytes(auth.username + ':' + auth.password, 'utf8')).decode(),
    user=user, repo=repo)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)

if p.stderr or p.returncode:
raise RuntimeError('Wiki not found. Please create a wiki')
return False

return r.json().get('private', False)
return private

GIT_URL = re.compile(r'(?:git@|https://|git://)github\.com[:/](.*?)(?:\.git)?')

Expand Down
3 changes: 3 additions & 0 deletions doctr/tests/test_local.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,13 @@ def test_github_bad_user():
def test_github_bad_repo():
with raises(RuntimeError):
check_repo_exists('drdoctr/---', headers=HEADERS)
with raises(RuntimeError):
check_repo_exists('drdoctr/---.wiki', headers=HEADERS)

@pytest.mark.skipif(not TEST_TOKEN, reason="No API token present")
def test_github_repo_exists():
assert not check_repo_exists('drdoctr/doctr', headers=HEADERS)
assert not check_repo_exists('drdoctr/doctr.wiki', headers=HEADERS)

@pytest.mark.skipif(not TEST_TOKEN, reason="No API token present")
def test_github_invalid_repo():
Expand Down
20 changes: 12 additions & 8 deletions doctr/travis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def decrypt_file(file, key):

os.chmod(file[:-4], 0o600)

def setup_deploy_key(keypath='github_deploy_key', key_ext='.enc'):
def setup_deploy_key(keypath='github_deploy_key', key_ext='.enc', env_name='DOCTR_DEPLOY_ENCRYPTION_KEY'):
"""
Decrypts the deploy key and configures it with ssh

Expand All @@ -58,10 +58,13 @@ def setup_deploy_key(keypath='github_deploy_key', key_ext='.enc'):
DOCTR_DEPLOY_ENCRYPTION_KEY.

"""
key = os.environ.get("DOCTR_DEPLOY_ENCRYPTION_KEY", None)
key = os.environ.get(env_name, os.environ.get("DOCTR_DEPLOY_ENCRYPTION_KEY", None))
if not key:
raise RuntimeError("DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set")
raise RuntimeError("{env_name} or DOCTR_DEPLOY_ENCRYPTION_KEY environment variable is not set"
.format(env_name=env_name))

if not os.path.isfile(keypath + key_ext):
keypath = 'github_deploy_key'
key_filename = os.path.basename(keypath)
key = key.encode('utf-8')
decrypt_file(keypath + key_ext, key)
Expand Down Expand Up @@ -131,10 +134,10 @@ def run(args, shell=False, exit=True):
If exit=True, it exits on nonzero returncode. Otherwise it returns the
returncode.
"""
if "DOCTR_DEPLOY_ENCRYPTION_KEY" in os.environ:
token = b''
else:
if "GH_TOKEN" in os.environ:
token = get_token()
else:
token = b''

if not shell:
command = ' '.join(map(shlex.quote, args))
Expand Down Expand Up @@ -177,7 +180,8 @@ def get_travis_branch():
else:
return os.environ.get("TRAVIS_BRANCH", "")

def setup_GitHub_push(deploy_repo, auth_type='deploy_key', full_key_path='github_deploy_key.enc', require_master=None, branch_whitelist=None, deploy_branch='gh-pages'):
def setup_GitHub_push(deploy_repo, auth_type='deploy_key', full_key_path='github_deploy_key.enc',
require_master=None, branch_whitelist=None, deploy_branch='gh-pages', env_name='DOCTR_DEPLOY_ENCRYPTION_KEY'):
"""
Setup the remote to push to GitHub (to be run on Travis).

Expand Down Expand Up @@ -226,7 +230,7 @@ def setup_GitHub_push(deploy_repo, auth_type='deploy_key', full_key_path='github
else:
keypath, key_ext = full_key_path.rsplit('.', 1)
key_ext = '.' + key_ext
setup_deploy_key(keypath=keypath, key_ext=key_ext)
setup_deploy_key(keypath=keypath, key_ext=key_ext, env_name=env_name)
run(['git', 'remote', 'add', 'doctr_remote',
'git@github.com:{deploy_repo}.git'.format(deploy_repo=deploy_repo)])
else:
Expand Down