diff --git a/setup.py b/setup.py index 2271e6e..3be29e1 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup(name='tap-github', - version='2.0.6', + version='2.0.7', description='Singer.io tap for extracting data from the GitHub API', author='Stitch', url='http://singer.io', diff --git a/tap_github/schemas/pr_files.json b/tap_github/schemas/pr_files.json new file mode 100644 index 0000000..1340f31 --- /dev/null +++ b/tap_github/schemas/pr_files.json @@ -0,0 +1,65 @@ +{ + "properties": { + "_sdc_repository": { + "type": ["string"] + }, + "sha": { + "type": ["null", "string"] + }, + "filename": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"], + "enum": [ + "added", + "removed", + "modified", + "renamed", + "copied", + "changed", + "unchanged" + ] + }, + "additions": { + "type": ["null", "integer"] + }, + "deletions": { + "type": ["null", "integer"] + }, + "changes": { + "type": ["null", "integer"] + }, + "blob_url": { + "type": ["null", "string"], + "format": "uri" + }, + "raw_url": { + "type": ["null", "string"], + "format": "uri" + }, + "contents_url": { + "type": ["null", "string"], + "format": "uri" + }, + "patch": { + "type": ["null", "string"] + }, + "previous_filename": { + "type": ["null", "string"] + }, + "pr_number": { + "type": ["null", "integer"] + }, + "pr_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "pr_updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + } diff --git a/tap_github/streams.py b/tap_github/streams.py index 3a75a00..84e8cdc 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -505,6 +505,28 @@ def add_fields_at_1st_level(self, record, parent_record = None): record['pr_id'] = parent_record.get('id') record['id'] = '{}-{}'.format(parent_record.get('id'), record.get('sha')) +class PRFiles(IncrementalStream): + ''' + https://docs.github.com/en/rest/pulls/pulls#list-pull-requests-files + ''' + tap_stream_id = "pr_files" + replication_method = "INCREMENTAL" + replication_keys = "pr_updated_at" + key_properties = ["id"] + path = "pulls/{}/files" + use_repository = True + id_keys = ['number'] + parent = 'pull_requests' + + def add_fields_at_1st_level(self, record, parent_record = None): + """ + Add fields in the record explicitly at the 1st level of JSON. + """ + record['pr_updated_at'] = parent_record['updated_at'] + record['pr_number'] = parent_record.get('number') + record['pr_id'] = str(parent_record.get('id')) + record['id'] = '{}-{}'.format(parent_record.get('id'), record.get('sha')) + class PullRequests(IncrementalOrderedStream): ''' https://developer.github.com/v3/pulls/#list-pull-requests @@ -514,7 +536,7 @@ class PullRequests(IncrementalOrderedStream): replication_keys = "updated_at" key_properties = ["id"] path = "pulls?state=all&sort=updated&direction=desc" - children = ['reviews', 'review_comments', 'pr_commits'] + children = ['reviews', 'review_comments', 'pr_commits', 'pr_files'] pk_child_fields = ["number"] class ProjectCards(IncrementalStream): @@ -773,6 +795,7 @@ def add_fields_at_1st_level(self, record, parent_record = None): "reviews": Reviews, "review_comments": ReviewComments, "pr_commits": PRCommits, + "pr_files": PRFiles, "teams": Teams, "team_members": TeamMembers, "team_memberships": TeamMemberships,