From ae85b68148760d181628d1b180c71e30a51913c8 Mon Sep 17 00:00:00 2001 From: sasi Date: Tue, 4 Jun 2024 23:09:48 +0530 Subject: [PATCH] cron func changes --- app.py | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- utils.py | 77 ++++++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+), 5 deletions(-) create mode 100644 utils.py diff --git a/app.py b/app.py index a58fb8d..fdf134a 100644 --- a/app.py +++ b/app.py @@ -7,6 +7,8 @@ from dotenv import load_dotenv from datetime import datetime +from utils import find_mentors, find_org_data + # Load environment variables from .env file load_dotenv() @@ -19,7 +21,7 @@ async def index(): return 'Hello, World!' -def define_pr_data(pr_val,issue_number,dmp): +def define_pr_data(pr_val,issue_number,dmp,owner,repo): try: pr_data = { "dmp_id":pr_val['id'], @@ -32,7 +34,9 @@ def define_pr_data(pr_val,issue_number,dmp): "url":dmp['repo_url'], "pr_updated_at":pr_val['updated_at'], "merged_at":pr_val['merged_at'], - "closed_at":pr_val['closed_at'] + "closed_at":pr_val['closed_at'], + "owner":owner, + "repo":repo } return pr_data @@ -41,7 +45,7 @@ def define_pr_data(pr_val,issue_number,dmp): return {} -def define_issue_data(val,owner,repo,issue_number): +def define_issue_data(val,owner,repo,issue_number,dmp): try: dmp_data = { "dmp_id":val['id'], @@ -54,7 +58,21 @@ def define_issue_data(val,owner,repo,issue_number): "comment_id":val['id'], "issue_url":val['issue_url'] if 'issue_url' in val else val['url'], "comment_url":val['comments_url'] if 'comments_url' in val else val['url'], - "comment_updated_at":val['updated_at'] + "comment_updated_at":val['updated_at'], + "mentor_name":val['mentor_usernames'], + "mentor_id":val['mentors'], + "contributor_name":val['cont_name'], + "contributor_id":val['cont_id'], + "title":val['title'], + "description":val['desc'], + "html_issue_url":val['html_issue_url'], + "org_id":val['org_id'], + "org_name":val['org_name'], + "org_link":val['org_link'], + "dmp_issue_url":dmp['repo_url'], + "org_description":val['org_desc'] + + } return dmp_data @@ -149,9 +167,107 @@ async def my_scheduled_job(): + +@app.route('/dmp_updates') +async def dmp_updates(): + print(f"job started --- {datetime.now()}") + # Define the GitHub API endpoint URL + GITHUB_TOKEN =os.getenv('GITHUB_TOKEN') + TARGET_DATE =os.getenv('TARGET_DATE') + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {GITHUB_TOKEN}", + "X-GitHub-Api-Version": "2022-11-28" + } + + try: + db = SupabaseInterface().get_instance() + dmp_tickets = db.readAll("dmp_issues") + + for dmp in dmp_tickets: + url_components = dmp["repo_url"].split('/') + issue_number = url_components[-1] + repo = url_components[-3] + owner = url_components[-4] + + GITHUB_API_URL = "https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}/comments" + #save first comment of issues + GITHUB_COMMENT_URL = "https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}" + comment_url = GITHUB_COMMENT_URL.format(owner=owner, repo=repo, issue_number=issue_number) + + async with httpx.AsyncClient() as client: + comment_response = await client.get(comment_url, headers=headers) + + #find mentors and organization datas + ment_data = find_mentors(comment_response.json()) + org_data = find_org_data(owner,repo,headers) + + issue_value = comment_response.json() + issue_value.update(ment_data) + issue_value.update(org_data) + + + dmp_data = define_issue_data(issue_value,owner,repo,issue_number,dmp) + #SAVE FIRST COMMENT OF THE ISSUE ONLY HERE + exist = db.client.table('dmp_issue_updates').select("*").eq('dmp_id',dmp_data['dmp_id']).execute() + if not exist.data: + add_data = db.add_data(dmp_data,'dmp_issue_updates') + else: + update_data =db.update_data(dmp_data,'dmp_issue_updates','dmp_id',dmp_data['dmp_id']) + + + url = GITHUB_API_URL.format(owner=owner, repo=repo, issue_number=issue_number) + + async with httpx.AsyncClient() as client: + response = await client.get(url, headers=headers) + # SAVE REMAINING COMMENT OF THE ISSUE BELOW + if response.status_code == 200: + + for val in response.json(): + val.update(org_data) + val.update(ment_data) + pr_created_at = val['created_at'] + if pr_created_at >= TARGET_DATE or 1==1: + dmp_data = define_issue_data(val,owner,repo,issue_number,dmp) + + exist = db.client.table('dmp_issue_updates').select("*").eq('dmp_id',dmp_data['dmp_id']).execute() + if not exist.data: + add_data = db.add_data(dmp_data,'dmp_issue_updates') + else: + update_data =db.update_data(dmp_data,'dmp_issue_updates','dmp_id',dmp_data['dmp_id']) + + #SECOND API CALL - PR + PR_API_URL = "https://api.github.com/repos/{owner}/{repo}/pulls" + + pr_url = PR_API_URL.format(owner=owner, repo=repo) + async with httpx.AsyncClient() as client: + pr_response = await client.get(pr_url, headers=headers) + if pr_response.status_code == 200: + + for pr_val in pr_response.json(): + pr_created_at = pr_val['created_at'] + if (pr_created_at >= TARGET_DATE) or 1==1: + pr_data = define_pr_data(pr_val,issue_number,dmp,owner,repo) + exist_pr = db.client.table('dmp_pr_updates').select("*").eq('pr_id',pr_data['pr_id']).execute() + if not exist_pr.data: + add_data = db.add_data(pr_data,'dmp_pr_updates') + else: + update_data = db.update_data(pr_data,'dmp_pr_updates','pr_id',pr_data['pr_id']) + + + + return "success" + + except Exception as e: + print(e) + return "Server Error" + + + + @app.before_serving async def start_scheduler(): - scheduler.add_job(my_scheduled_job, 'interval', minutes=2) + scheduler.add_job(dmp_updates, 'interval', minutes=2) scheduler.start() if __name__ == '__main__': diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..8b6fbd2 --- /dev/null +++ b/utils.py @@ -0,0 +1,77 @@ +import re,requests + + +def find_mentors(json): + try: + issue_details = json + + issue_body = issue_details['body'] + pattern = r"## Mentors\s*([\s\S]+?)\s*##" + disc_pattern = r"## Desc 1\s*([\s\S]+?)\s*##" + disc_match = re.search(disc_pattern, issue_body) + + disc_text = disc_match.group(1).strip() if disc_match else None + + match = re.search(pattern, issue_body) + + if match: + mentors_text = match.group(1).strip() + # Extract individual mentor usernames + mentors = [mentor.strip() for mentor in mentors_text.split(',')] + else: + mentors = [] + api_base_url = "https://api.github.com/users/" + + ment_username = [] + for val in mentors: + url = f"{api_base_url}{val[1:]}" + username = requests.get(url) + + ment_username.append(username.json()['login']) + return { + 'mentors': mentors, + 'mentor_usernames': ment_username, + 'desc':disc_text, + 'title':issue_details['title'], + 'issue_id':issue_details['id'], + 'html_issue_url':issue_details['html_url'], + 'cont_id':issue_details['user']['id'], + 'cont_name':issue_details['user']['login'] + } + except: + return { + 'mentors': [], + 'mentor_usernames': [], + 'desc':None, + 'title':None, + 'issue_id':None, + 'html_issue_url':None, + 'cont_id':None, + 'cont_name':None + } + + + +def find_org_data(owner,repo,headers): + try: + + # Fetch repository details to get organization info + repo_url = f"https://api.github.com/repos/{owner}/{repo}" + repo_response = requests.get(repo_url, headers=headers) + repo_data = repo_response.json() + if repo_data: + org_name = repo_data['owner']['login'] + org_id = repo_data['owner']['id'] + org_link = repo_data['owner']['html_url'] + org_desc = repo_data['description'] + else: + org_name = None + org_id = None + org_link = None + org_desc = repo_data['description'] + + return {"org_id":org_id,"org_name":org_name,'org_link':org_link,'org_desc':org_desc} + + except Exception as e: + return {"org_id":None,"org_name":None,'org_link':None} +