Skip to content

Add Forked repos stream #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
joaopamaral merged 8 commits into from
Jan 17, 2025
Merged

Add Forked repos stream #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c91b035
Add Forked repos stream
joaopamaral Jan 13, 2025
55bbe38
Fix
joaopamaral Jan 14, 2025
5464ab1
Fix tests
joaopamaral Jan 14, 2025
0d4347c
Bump version
joaopamaral Jan 14, 2025
3a94da0
Update tap_github/streams.py
joaopamaral Jan 16, 2025
1183f66
Refactor auth get pages
joaopamaral Jan 16, 2025
abae8ec
Merge remote-tracking branch 'origin/repos-forked-streams' into repos…
joaopamaral Jan 16, 2025
9db65db
Add error_message
joaopamaral Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from setuptools import setup, find_packages

setup(name='tap-github',
version='2.0.8',
version='2.0.9',
description='Singer.io tap for extracting data from the GitHub API',
author='Stitch',
url='http://singer.io',
Expand Down
12 changes: 9 additions & 3 deletions tap_github/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ def set_auth_in_session(self):
@backoff.on_exception(backoff.expo, (requests.Timeout, requests.ConnectionError, Server5xxError, TooManyRequests),
max_tries=5, factor=2)
@backoff.on_exception(backoff.expo, (BadCredentialsException, ), max_tries=3, factor=2)
def authed_get(self, source, url, headers={}, stream="", should_skip_404 = True):
def authed_get_single_page(self, source, url, headers={}, stream="", should_skip_404 = True):
"""
Call rest API and return the response in case of status code 200.
"""
Expand All @@ -228,11 +228,17 @@ def authed_get_all_pages(self, source, url, headers={}, stream="", should_skip_4
"""
next_url = self.prepare_url(url)
while next_url:
response = self.authed_get(source, next_url, headers, stream, should_skip_404)
response = self.authed_get_single_page(source, next_url, headers, stream, should_skip_404)
yield response

next_url = response.links.get('next', {}).get('url', None)

def authed_get(self, source, url, headers={}, stream="", should_skip_404=True, single_page=False):
if single_page:
yield self.authed_get_single_page(source, url, headers, stream, should_skip_404)
else:
yield from self.authed_get_all_pages(source, url, headers, stream, should_skip_404)

def prepare_url(self, url):
"""
Prepare the URL with some additional parameters
Expand All @@ -247,7 +253,7 @@ def verify_repo_access(self, url_for_repo, repo):
Call rest API to verify that the user has sufficient permissions to access this repository.
"""
try:
self.authed_get("verifying repository access", url_for_repo, should_skip_404=False)
self.authed_get_single_page("verifying repository access", url_for_repo, should_skip_404=False)
except NotFoundException:
# Throwing user-friendly error message as it checks token access
message = "HTTP-error-code: 404, Error: Please check the repository name \'{}\' or you do not have sufficient permissions to access this repository.".format(repo)
Expand Down
77 changes: 77 additions & 0 deletions tap_github/schemas/repo_forked_compares.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
{
"type": ["null", "object"],
"properties": {
"_sdc_repository": {
"type": ["null", "string"]
},
"name": {
"type": ["null", "string"]
},
"full_name": {
"type": ["null", "string"]
},
"owner_login": {
"type": ["null", "string"]
},
"default_branch": {
"type": ["null", "string"]
},
"fork_name": {
"type": ["null", "string"]
},
"fork_owner_login": {
"type": ["null", "string"]
},
"fork_default_branch": {
"type": ["null", "string"]
},
"url": {
"type": ["null", "string"]
},
"html_url": {
"type": ["null", "string"]
},
"permalink_url": {
"type": ["null", "string"]
},
"diff_url": {
"type": ["null", "string"]
},
"patch_url": {
"type": ["null", "string"]
},
"base_commit": {
"type": ["null", "object"]
},
"merge_base_commit": {
"type": ["null", "object"]
},
"status": {
"type": ["null", "string"]
},
"ahead_by": {
"type": ["null", "number"]
},
"behind_by": {
"type": ["null", "number"]
},
"total_commits": {
"type": ["null", "number"]
},
"commits": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"]
}
},
"files": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"]
}
},
"error_message": {
"type": ["null", "string"]
}
}
}
260 changes: 260 additions & 0 deletions tap_github/schemas/repo_forked_parents.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,260 @@
{
"type": ["null", "object"],
"properties": {
"_sdc_repository": {
"type": ["null", "string"]
},
"id": {
"type": ["null", "number"]
},
"node_id": {
"type": ["null", "string"]
},
"url": {
"type": ["null", "string"]
},
"html_url": {
"type": ["null", "string"]
},
"name": {
"type": ["null", "string"]
},
"full_name": {
"type": ["null", "string"]
},
"private": {
"type": ["null", "boolean"]
},
"owner": {
"$ref": "shared/user.json#/"
},
"description": {
"type": ["null", "string"]
},
"fork": {
"type": ["null", "boolean"]
},
"fork_name": {
"type": ["null", "string"]
},
"fork_owner_login": {
"type": ["null", "string"]
},
"fork_default_branch": {
"type": ["null", "string"]
},
"keys_url": {
"type": ["null", "string"]
},
"collaborators_url": {
"type": ["null", "string"]
},
"teams_url": {
"type": ["null", "string"]
},
"hooks_url": {
"type": ["null", "string"]
},
"issue_events_url": {
"type": ["null", "string"]
},
"events_url": {
"type": ["null", "string"]
},
"assignees_url": {
"type": ["null", "string"]
},
"branches_url": {
"type": ["null", "string"]
},
"tags_url": {
"type": ["null", "string"]
},
"blobs_url": {
"type": ["null", "string"]
},
"git_tags_url": {
"type": ["null", "string"]
},
"git_refs_url": {
"type": ["null", "string"]
},
"trees_url": {
"type": ["null", "string"]
},
"statuses_url": {
"type": ["null", "string"]
},
"languages_url": {
"type": ["null", "string"]
},
"stargazers_url": {
"type": ["null", "string"]
},
"contributors_url": {
"type": ["null", "string"]
},
"subscribers_url": {
"type": ["null", "string"]
},
"subscription_url": {
"type": ["null", "string"]
},
"commits_url": {
"type": ["null", "string"]
},
"git_commits_url": {
"type": ["null", "string"]
},
"comments_url": {
"type": ["null", "string"]
},
"issue_comment_url": {
"type": ["null", "string"]
},
"contents_url": {
"type": ["null", "string"]
},
"compare_url": {
"type": ["null", "string"]
},
"merges_url": {
"type": ["null", "string"]
},
"archive_url": {
"type": ["null", "string"]
},
"downloads_url": {
"type": ["null", "string"]
},
"issues_url": {
"type": ["null", "string"]
},
"pulls_url": {
"type": ["null", "string"]
},
"milestones_url": {
"type": ["null", "string"]
},
"notifications_url": {
"type": ["null", "string"]
},
"labels_url": {
"type": ["null", "string"]
},
"releases_url": {
"type": ["null", "string"]
},
"deployments_url": {
"type": ["null", "string"]
},
"created_at": {
"type": ["null", "string"]
},
"updated_at": {
"type": ["null", "string"]
},
"pushed_at": {
"type": ["null", "string"]
},
"git_url": {
"type": ["null", "string"]
},
"ssh_url": {
"type": ["null", "string"]
},
"clone_url": {
"type": ["null", "string"]
},
"svn_url": {
"type": ["null", "string"]
},
"homepage": {
"type": ["null", "string"]
},
"size": {
"type": ["null", "number"]
},
"stargazers_count": {
"type": ["null", "number"]
},
"watchers_count": {
"type": ["null", "number"]
},
"language": {
"type": ["null", "string"]
},
"has_issues": {
"type": ["null", "boolean"]
},
"has_projects": {
"type": ["null", "boolean"]
},
"has_downloads": {
"type": ["null", "boolean"]
},
"has_wiki": {
"type": ["null", "boolean"]
},
"has_pages": {
"type": ["null", "boolean"]
},
"has_discussions": {
"type": ["null", "boolean"]
},
"forks_count": {
"type": ["null", "number"]
},
"mirror_url": {
"type": ["null", "string"]
},
"archived": {
"type": ["null", "boolean"]
},
"disabled": {
"type": ["null", "boolean"]
},
"open_issues_count": {
"type": ["null", "number"]
},
"license": {
"type": ["null", "object"]
},
"allow_forking": {
"type": ["null", "boolean"]
},
"is_template": {
"type": ["null", "boolean"]
},
"web_commit_signoff_required": {
"type": ["null", "boolean"]
},
"topics": {
"type": ["null", "array"],
"items": {
"type": ["string"]
}
},
"visibility": {
"type": ["null", "string"]
},
"forks": {
"type": ["null", "number"]
},
"open_issues": {
"type": ["null", "number"]
},
"watchers": {
"type": ["null", "number"]
},
"default_branch": {
"type": ["null", "string"]
},
"permissions": {
"$ref": "shared/pull_permissions.json#/"
},
"parent": {
"$ref": "shared/repos.json#/"
}
}
}
Loading