From 493729cae7134b9228e3d801583435b41d5ea1db Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sat, 10 Aug 2024 23:05:32 +0800 Subject: [PATCH 1/6] add webhook router to `Gateway`: fetch details when PR events emit Signed-off-by: Qihang Cai --- gateway/src/api/api_router.rs | 3 +- gateway/src/api/github_router.rs | 71 ++++++++++++++++++++++++++++++++ gateway/src/api/mod.rs | 1 + 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 gateway/src/api/github_router.rs diff --git a/gateway/src/api/api_router.rs b/gateway/src/api/api_router.rs index 3fff4acda..a788d702f 100644 --- a/gateway/src/api/api_router.rs +++ b/gateway/src/api/api_router.rs @@ -15,7 +15,7 @@ use ceres::model::{ }; use common::model::CommonResult; -use crate::api::mr_router; +use crate::api::{github_router, mr_router}; use crate::api::ApiServiceState; use super::ztm_router; @@ -34,6 +34,7 @@ pub fn routers() -> Router { .merge(router) .merge(mr_router::routers()) .merge(ztm_router::routers()) + .merge(github_router::routers()) } async fn get_blob_object( diff --git a/gateway/src/api/github_router.rs b/gateway/src/api/github_router.rs new file mode 100644 index 000000000..2e0b20193 --- /dev/null +++ b/gateway/src/api/github_router.rs @@ -0,0 +1,71 @@ +use std::fmt::format; +use axum::{Json, Router}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; +use axum::routing::{get, post}; +use lazy_static::lazy_static; +use reqwest::Client; +use reqwest::header::{ACCEPT, AUTHORIZATION}; +use serde_json::Value; +use common::model::CommonResult; +use crate::api::ApiServiceState; + +// const GITHUB_API_SERVER: &str = "https://api.github.com"; +// const OWNER: &str = "web3infra-foundation"; +// const REPO: &str = "mega"; + +lazy_static! { + static ref CLIENT: Client = Client::builder() + .user_agent("Mega/0.0.1") // IMPORTANT, or 403 Forbidden + .build() + .unwrap(); +} + +pub fn routers() -> Router { + Router::new() + .route("/github/webhook", post(webhook)) +} + +async fn webhook( + headers: HeaderMap, + Json(payload): Json +) -> Result { + let event_type = headers + .get("X-GitHub-Event") + .and_then(|v| v.to_str().ok()) + .expect("Missing X-GitHub-Event header"); + + tracing::debug!("WebHook Event Type: {}", event_type); + + if event_type == "pull_request" { + let action = payload["action"].as_str().unwrap(); + tracing::debug!("PR action: {}", action); + + if ["opened", "reopened", "synchronize"].contains(&action) { + let url = payload["pull_request"]["url"].as_str().unwrap(); + let files = get_pr_files(url).await; + let commits = get_pr_commits(url).await; + tracing::debug!("PR: {:#?}", files); + tracing::debug!("Commits: {:#?}", commits); + } else if action == "edited" { // PR title or body edited + let _title = payload["pull_request"]["title"].as_str().unwrap(); + let _body = payload["pull_request"]["body"].as_str().unwrap(); + } + } + + Ok("WebHook OK") +} + +async fn get_pr_files(pr_url: &str) -> Value { + get_request(&format!("{}/files", pr_url)).await +} + +async fn get_pr_commits(pr_url: &str) -> Value { + get_request(&format!("{}/commits", pr_url)).await +} + +/// Send a GET request to the given URL and return the JSON response. +async fn get_request(url: &str) -> Value { + let resp = CLIENT.get(url).send().await.unwrap(); + resp.json().await.unwrap() +} \ No newline at end of file diff --git a/gateway/src/api/mod.rs b/gateway/src/api/mod.rs index 070abfda8..073d5ee32 100644 --- a/gateway/src/api/mod.rs +++ b/gateway/src/api/mod.rs @@ -11,6 +11,7 @@ pub mod api_router; pub mod mr_router; pub mod oauth; pub mod ztm_router; +mod github_router; #[derive(Clone)] pub struct ApiServiceState { From e7ad2c4749467516af06142c7fab552d49c552d7 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sat, 10 Aug 2024 23:13:45 +0800 Subject: [PATCH 2/6] add Issues event to webhook router (`Gateway`) Signed-off-by: Qihang Cai --- gateway/src/api/github_router.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/gateway/src/api/github_router.rs b/gateway/src/api/github_router.rs index 2e0b20193..3c11f4712 100644 --- a/gateway/src/api/github_router.rs +++ b/gateway/src/api/github_router.rs @@ -1,19 +1,12 @@ -use std::fmt::format; use axum::{Json, Router}; use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; -use axum::routing::{get, post}; +use axum::routing::post; use lazy_static::lazy_static; use reqwest::Client; -use reqwest::header::{ACCEPT, AUTHORIZATION}; use serde_json::Value; -use common::model::CommonResult; use crate::api::ApiServiceState; -// const GITHUB_API_SERVER: &str = "https://api.github.com"; -// const OWNER: &str = "web3infra-foundation"; -// const REPO: &str = "mega"; - lazy_static! { static ref CLIENT: Client = Client::builder() .user_agent("Mega/0.0.1") // IMPORTANT, or 403 Forbidden @@ -51,6 +44,12 @@ async fn webhook( let _title = payload["pull_request"]["title"].as_str().unwrap(); let _body = payload["pull_request"]["body"].as_str().unwrap(); } + } else if event_type == "issues" { + let action = payload["action"].as_str().unwrap(); + tracing::debug!("Issue action: {}", action); + let title = payload["issue"]["title"].as_str().unwrap(); + let body = payload["issue"]["body"].as_str().unwrap(); + tracing::debug!("Issue: {} - {}", title, body); } Ok("WebHook OK") From e90afe042882731fae12bee6a38cc9eca555e1cb Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sun, 11 Aug 2024 19:35:05 +0800 Subject: [PATCH 3/6] add GitHub webhook endpoint to `aries` (unimpl) Signed-off-by: Qihang Cai --- aries/src/service/relay_server.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/aries/src/service/relay_server.rs b/aries/src/service/relay_server.rs index 4914309d7..d6e3b9c35 100644 --- a/aries/src/service/relay_server.rs +++ b/aries/src/service/relay_server.rs @@ -2,13 +2,14 @@ use std::net::SocketAddr; use std::str::FromStr; use std::time::{Duration, SystemTime}; -use axum::extract::{Query, State}; +use axum::extract::{Path, Query, State}; use axum::http::StatusCode; use axum::response::IntoResponse; use axum::routing::{get, post}; use axum::{Json, Router}; use callisto::{ztm_node, ztm_repo_info}; use clap::Parser; +use serde_json::Value; use common::config::Config; use gemini::ztm::hub::{LocalHub, ZTMUserPermit, ZTMCA}; use gemini::{Node, RelayGetParams, RelayResultRes, RepoInfo}; @@ -104,7 +105,8 @@ pub fn routers() -> Router { .route("/ping", get(ping)) .route("/node_list", get(node_list)) .route("/repo_provide", post(repo_provide)) - .route("/repo_list", get(repo_list)); + .route("/repo_list", get(repo_list)) + .route("/github/webhook/:peer_id", post(github_webhook)); Router::new().merge(router) } @@ -220,6 +222,14 @@ pub async fn repo_list( Ok(Json(repo_info_list_result)) } +/// Forwards the GitHub webhook to the corresponding peer. +async fn github_webhook( + Path(peer_id): Path, + Json(payload): Json, +) -> Result { + unimplemented!(); +} + async fn loop_running(context: Context) { let mut interval = tokio::time::interval(Duration::from_secs(60)); From fa9ea4a3501cf9b45f6273030b02345aacd83d8d Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sun, 11 Aug 2024 19:56:23 +0800 Subject: [PATCH 4/6] add `GithubWebhookEvent` in `taurus` Signed-off-by: Qihang Cai --- taurus/src/event/github_webhook.rs | 56 ++++++++++++++++++++++++++++++ taurus/src/event/mod.rs | 5 +++ 2 files changed, 61 insertions(+) create mode 100644 taurus/src/event/github_webhook.rs diff --git a/taurus/src/event/github_webhook.rs b/taurus/src/event/github_webhook.rs new file mode 100644 index 000000000..a1c23f3f2 --- /dev/null +++ b/taurus/src/event/github_webhook.rs @@ -0,0 +1,56 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use crate::event::{EventBase, EventType}; +use crate::queue::get_mq; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GithubWebhookEvent { + pub _type: WebhookType, + pub payload: Value, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub enum WebhookType { + PullRequest, + Issue, +} + +impl std::fmt::Display for GithubWebhookEvent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "GitHub Webhook Event: {:?}", self._type) + } +} + +#[async_trait] +impl EventBase for GithubWebhookEvent { + async fn process(&self) { + tracing::info!("Handling GitHub Webhook Event: [{}]", &self); + } +} + +impl GithubWebhookEvent { + // Create and enqueue this event. + pub fn notify(_type: WebhookType, payload: Value) { + get_mq().send(EventType::GithubWebhook(GithubWebhookEvent { + _type, + payload, + })); + } +} + +// For storing the data into database. +impl From for Value { + fn from(value: GithubWebhookEvent) -> Self { + serde_json::to_value(value).unwrap() + } +} + +impl TryFrom for GithubWebhookEvent { + type Error = crate::event::Error; + + fn try_from(value: Value) -> Result { + let res: GithubWebhookEvent = serde_json::from_value(value)?; + Ok(res) + } +} \ No newline at end of file diff --git a/taurus/src/event/mod.rs b/taurus/src/event/mod.rs index 455b555a7..79e1193be 100644 --- a/taurus/src/event/mod.rs +++ b/taurus/src/event/mod.rs @@ -7,13 +7,16 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use serde_json::Value; use thiserror::Error; +use github_webhook::GithubWebhookEvent; pub mod api_request; +mod github_webhook; #[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, Serialize, Deserialize)] pub enum EventType { ApiRequest(ApiRequestEvent), + GithubWebhook(GithubWebhookEvent), // Reserved ErrorEvent, @@ -54,6 +57,8 @@ impl EventType { EventType::ApiRequest(evt) => evt.process().await, // EventType::SomeOtherEvent(xxx) => xxx.process().await, + EventType::GithubWebhook(evt) => evt.process().await, + // This won't happen unless failed to load events from database. // And that's because of a event conversion error. // You should recheck yout conversion code logic. From 6da1ad0d4d8d20ebc9f2d288dd3e43a01a347cea Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 12 Aug 2024 14:59:12 +0800 Subject: [PATCH 5/6] use `GithubWebhookEvent` to notify webhook & add more details to payload Signed-off-by: Qihang Cai --- gateway/src/api/github_router.rs | 56 ++++++++++++++++++------------ taurus/src/event/github_webhook.rs | 16 +++++++-- taurus/src/event/mod.rs | 2 +- 3 files changed, 49 insertions(+), 25 deletions(-) diff --git a/gateway/src/api/github_router.rs b/gateway/src/api/github_router.rs index 3c11f4712..159a68748 100644 --- a/gateway/src/api/github_router.rs +++ b/gateway/src/api/github_router.rs @@ -5,6 +5,7 @@ use axum::routing::post; use lazy_static::lazy_static; use reqwest::Client; use serde_json::Value; +use taurus::event::github_webhook::{GithubWebhookEvent, WebhookType}; use crate::api::ApiServiceState; lazy_static! { @@ -19,47 +20,58 @@ pub fn routers() -> Router { .route("/github/webhook", post(webhook)) } +/// Handle the GitHub webhook event.
+/// For more details, see https://docs.github.com/zh/webhooks/webhook-events-and-payloads. async fn webhook( headers: HeaderMap, - Json(payload): Json + Json(mut payload): Json ) -> Result { let event_type = headers .get("X-GitHub-Event") .and_then(|v| v.to_str().ok()) .expect("Missing X-GitHub-Event header"); + payload["event_type"] = event_type.into(); - tracing::debug!("WebHook Event Type: {}", event_type); + let event_type = WebhookType::from(event_type); + match event_type { + WebhookType::PullRequest => { + let action = payload["action"].as_str().unwrap(); + tracing::debug!("PR action: {}", action); - if event_type == "pull_request" { - let action = payload["action"].as_str().unwrap(); - tracing::debug!("PR action: {}", action); + if ["opened", "reopened", "synchronize"].contains(&action) { // contents changed + let url = payload["pull_request"]["url"].as_str().unwrap(); + let files = get_pr_files(url).await; + let commits = get_pr_commits(url).await; + // Add details to the payload + payload["files"] = files; + payload["commits"] = commits; + } else if action == "edited" { // PR title or body edited + let _ = payload["pull_request"]["title"].as_str().unwrap(); + let _ = payload["pull_request"]["body"].as_str().unwrap(); + } - if ["opened", "reopened", "synchronize"].contains(&action) { - let url = payload["pull_request"]["url"].as_str().unwrap(); - let files = get_pr_files(url).await; - let commits = get_pr_commits(url).await; - tracing::debug!("PR: {:#?}", files); - tracing::debug!("Commits: {:#?}", commits); - } else if action == "edited" { // PR title or body edited - let _title = payload["pull_request"]["title"].as_str().unwrap(); - let _body = payload["pull_request"]["body"].as_str().unwrap(); + GithubWebhookEvent::notify(WebhookType::PullRequest, payload); + } + WebhookType::Issues => { + GithubWebhookEvent::notify(WebhookType::Issues, payload); + } + WebhookType::Unknown(_type) => { + tracing::warn!("Unknown event type: {}", _type); + GithubWebhookEvent::notify(WebhookType::Unknown(_type), payload); } - } else if event_type == "issues" { - let action = payload["action"].as_str().unwrap(); - tracing::debug!("Issue action: {}", action); - let title = payload["issue"]["title"].as_str().unwrap(); - let body = payload["issue"]["body"].as_str().unwrap(); - tracing::debug!("Issue: {} - {}", title, body); } Ok("WebHook OK") } -async fn get_pr_files(pr_url: &str) -> Value { +/// GitHub API: Get the files change of a pull request.
+/// For read-only operation of public repos, no authentication is required. +pub async fn get_pr_files(pr_url: &str) -> Value { get_request(&format!("{}/files", pr_url)).await } -async fn get_pr_commits(pr_url: &str) -> Value { +/// GitHub API: Get the commits of a pull request. +pub async fn get_pr_commits(pr_url: &str) -> Value { get_request(&format!("{}/commits", pr_url)).await } diff --git a/taurus/src/event/github_webhook.rs b/taurus/src/event/github_webhook.rs index a1c23f3f2..84bbb9fc5 100644 --- a/taurus/src/event/github_webhook.rs +++ b/taurus/src/event/github_webhook.rs @@ -13,7 +13,18 @@ pub struct GithubWebhookEvent { #[derive(Debug, Serialize, Deserialize, Clone)] pub enum WebhookType { PullRequest, - Issue, + Issues, + Unknown(String), +} + +impl From<&str> for WebhookType { + fn from(value: &str) -> Self { + match value { + "pull_request" => WebhookType::PullRequest, + "issues" => WebhookType::Issues, + _ => WebhookType::Unknown(value.to_string()), + } + } } impl std::fmt::Display for GithubWebhookEvent { @@ -25,7 +36,8 @@ impl std::fmt::Display for GithubWebhookEvent { #[async_trait] impl EventBase for GithubWebhookEvent { async fn process(&self) { - tracing::info!("Handling GitHub Webhook Event: [{}]", &self); + tracing::info!("Processing: [{}]", &self); + tracing::info!("Payload: {:#?}", &self.payload); } } diff --git a/taurus/src/event/mod.rs b/taurus/src/event/mod.rs index 79e1193be..8cb3cf688 100644 --- a/taurus/src/event/mod.rs +++ b/taurus/src/event/mod.rs @@ -10,7 +10,7 @@ use thiserror::Error; use github_webhook::GithubWebhookEvent; pub mod api_request; -mod github_webhook; +pub mod github_webhook; #[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, Serialize, Deserialize)] From ae30e0d6b122aa3080835dd6f37fb54ddd35d1d3 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 12 Aug 2024 15:21:38 +0800 Subject: [PATCH 6/6] clear clippy warning Signed-off-by: Qihang Cai --- aries/src/service/relay_server.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aries/src/service/relay_server.rs b/aries/src/service/relay_server.rs index d6e3b9c35..31fd6c3fc 100644 --- a/aries/src/service/relay_server.rs +++ b/aries/src/service/relay_server.rs @@ -226,7 +226,8 @@ pub async fn repo_list( async fn github_webhook( Path(peer_id): Path, Json(payload): Json, -) -> Result { +) -> Result { + tracing::info!("GitHub Webhook Event: {:?} \n {:?}", peer_id, payload); unimplemented!(); }