Skip to content

[orion-server] feat: add buck2 log segment function #1332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
benjamin-747 merged 5 commits into from
Aug 13, 2025
Merged

[orion-server] feat: add buck2 log segment function #1332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d7c8137
[orion-server] feat: add buck2 log segment function
Ivanbeethoven Aug 12, 2025
32ca31e
Merge branch 'main' into main
Ivanbeethoven Aug 12, 2025
0714d8e
[orion-server]: fix box leak
Ivanbeethoven Aug 13, 2025
be88786
[orion-server]: fix typo
Ivanbeethoven Aug 13, 2025
db24f0c
[orion-server] tasks' status interface
Ivanbeethoven Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 109 additions & 54 deletions ceres/src/api_service/mono_api_service.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ use async_trait::async_trait;
use callisto::sea_orm_active_enums::ConvTypeEnum;
use callisto::{mega_blob, mega_mr, mega_tree, raw_blob};
use common::errors::MegaError;
use neptune::neptune_engine::Diff;
use jupiter::storage::base_storage::StorageConnector;
use jupiter::storage::Storage;
use jupiter::utils::converter::generate_git_keep_with_timestamp;
Expand All @@ -53,11 +52,11 @@ use mercury::hash::SHA1;
use mercury::internal::object::blob::Blob;
use mercury::internal::object::commit::Commit;
use mercury::internal::object::tree::{Tree, TreeItem, TreeItemMode};
use neptune::neptune_engine::Diff;

use crate::api_service::ApiHandler;
use crate::model::git::CreateFileInfo;
use crate::model::mr::{MrDiffFile, MrDiff, MrPageInfo};

use crate::model::mr::{MrDiff, MrDiffFile, MrPageInfo};

#[derive(Clone)]
pub struct MonoApiService {
Expand Down Expand Up @@ -387,18 +386,23 @@ impl MonoApiService {
) -> Result<MrDiff, GitError> {
// old and new blobs for comparison
let stg = self.storage.mr_storage();
let mr = stg.get_mr(mr_link).await.unwrap().ok_or_else(|| {
GitError::CustomError(format!("Merge request not found: {mr_link}"))
})?;
let old_blobs = self.get_commit_blobs(&mr.from_hash).await.map_err(|e| {
GitError::CustomError(format!("Failed to get old commit blobs: {e}"))
})?;
let new_blobs = self.get_commit_blobs(&mr.to_hash).await.map_err(|e| {
GitError::CustomError(format!("Failed to get new commit blobs: {e}"))
})?;
let mr =
stg.get_mr(mr_link).await.unwrap().ok_or_else(|| {
GitError::CustomError(format!("Merge request not found: {mr_link}"))
})?;
let old_blobs = self
.get_commit_blobs(&mr.from_hash)
.await
.map_err(|e| GitError::CustomError(format!("Failed to get old commit blobs: {e}")))?;
let new_blobs = self
.get_commit_blobs(&mr.to_hash)
.await
.map_err(|e| GitError::CustomError(format!("Failed to get new commit blobs: {e}")))?;

// calculate pages
let sorted_changed_files = self.mr_files_list(old_blobs.clone(), new_blobs.clone()).await?;
let sorted_changed_files = self
.mr_files_list(old_blobs.clone(), new_blobs.clone())
.await?;

// ensure page_id is within bounds
let start = (page_id.saturating_sub(1)) * page_size;
Expand Down Expand Up @@ -441,7 +445,7 @@ impl MonoApiService {
}

// Simple synchronous closure that uses the pre-fetched cache
let read_content = |_file: &PathBuf, hash: &SHA1| -> Vec<u8>{
let read_content = |_file: &PathBuf, hash: &SHA1| -> Vec<u8> {
blob_cache.get(hash).cloned().unwrap_or_default()
};

Expand All @@ -452,15 +456,16 @@ impl MonoApiService {
"histogram".to_string(),
Vec::new(),
read_content,
).await;
)
.await;

Ok(MrDiff {
data: diff_output,
page_info: Some(MrPageInfo {
total_pages: (sorted_changed_files.len()-1).div_ceil(page_size),
total_pages: (sorted_changed_files.len() - 1).div_ceil(page_size),
current_page: page_id,
page_size,
})
}),
})
}

Expand Down Expand Up @@ -517,7 +522,9 @@ impl MonoApiService {

// Sort the results
res.sort_by(|a, b| {
a.path().cmp(b.path()).then_with(|| a.kind_weight().cmp(&b.kind_weight()))
a.path()
.cmp(b.path())
.then_with(|| a.kind_weight().cmp(&b.kind_weight()))
});
Ok(res)
}
Expand Down Expand Up @@ -566,10 +573,10 @@ impl MonoApiService {
#[cfg(test)]
mod test {
use super::*;
use std::path::PathBuf;
use crate::model::mr::{MrDiffFile, MrPageInfo};
use mercury::hash::SHA1;
use std::path::PathBuf;
use std::str::FromStr;
use crate::model::mr::{MrDiffFile, MrPageInfo};

#[test]
pub fn test_path() {
Expand All @@ -585,9 +592,19 @@ mod test {
#[test]
fn test_paging_calculation_basic() {
let files: Vec<MrDiffFile> = vec![
MrDiffFile::New(PathBuf::from("file1.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap()),
MrDiffFile::Modified(PathBuf::from("file2.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap(), SHA1::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap()),
MrDiffFile::Deleted(PathBuf::from("file3.txt"), SHA1::from_str("1111111111111111111111111111111111111111").unwrap()),
MrDiffFile::New(
PathBuf::from("file1.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
),
MrDiffFile::Modified(
PathBuf::from("file2.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
SHA1::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(),
),
MrDiffFile::Deleted(
PathBuf::from("file3.txt"),
SHA1::from_str("1111111111111111111111111111111111111111").unwrap(),
),
];

let page_size = 2u32;
Expand All @@ -613,10 +630,23 @@ mod test {
#[test]
fn test_paging_calculation_second_page() {
let files: Vec<MrDiffFile> = vec![
MrDiffFile::New(PathBuf::from("file1.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap()),
MrDiffFile::Modified(PathBuf::from("file2.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap(), SHA1::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap()),
MrDiffFile::Deleted(PathBuf::from("file3.txt"), SHA1::from_str("1111111111111111111111111111111111111111").unwrap()),
MrDiffFile::New(PathBuf::from("file4.txt"), SHA1::from_str("2222222222222222222222222222222222222222").unwrap()),
MrDiffFile::New(
PathBuf::from("file1.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
),
MrDiffFile::Modified(
PathBuf::from("file2.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
SHA1::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(),
),
MrDiffFile::Deleted(
PathBuf::from("file3.txt"),
SHA1::from_str("1111111111111111111111111111111111111111").unwrap(),
),
MrDiffFile::New(
PathBuf::from("file4.txt"),
SHA1::from_str("2222222222222222222222222222222222222222").unwrap(),
),
];

let page_size = 2u32;
Expand Down Expand Up @@ -644,9 +674,19 @@ mod test {
#[test]
fn test_paging_calculation_partial_page() {
let files: Vec<MrDiffFile> = vec![
MrDiffFile::New(PathBuf::from("file1.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap()),
MrDiffFile::Modified(PathBuf::from("file2.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap(), SHA1::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap()),
MrDiffFile::Deleted(PathBuf::from("file3.txt"), SHA1::from_str("1111111111111111111111111111111111111111").unwrap()),
MrDiffFile::New(
PathBuf::from("file1.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
),
MrDiffFile::Modified(
PathBuf::from("file2.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
SHA1::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(),
),
MrDiffFile::Deleted(
PathBuf::from("file3.txt"),
SHA1::from_str("1111111111111111111111111111111111111111").unwrap(),
),
];

let page_size = 5u32;
Expand All @@ -671,9 +711,10 @@ mod test {

#[test]
fn test_paging_calculation_out_of_bounds() {
let files: Vec<MrDiffFile> = vec![
MrDiffFile::New(PathBuf::from("file1.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap()),
];
let files: Vec<MrDiffFile> = vec![MrDiffFile::New(
PathBuf::from("file1.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
)];

let page_size = 2u32;
let page_id = 3u32; // Page that doesn't exist
Expand All @@ -697,9 +738,10 @@ mod test {

#[test]
fn test_paging_calculation_edge_case_zero_page_size() {
let files: Vec<MrDiffFile> = vec![
MrDiffFile::New(PathBuf::from("file1.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap()),
];
let files: Vec<MrDiffFile> = vec![MrDiffFile::New(
PathBuf::from("file1.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
)];

let page_size = 0u32;
let page_id = 1u32;
Expand All @@ -724,8 +766,15 @@ mod test {
#[test]
fn test_paging_calculation_zero_page_id() {
let files: Vec<MrDiffFile> = vec![
MrDiffFile::New(PathBuf::from("file1.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap()),
MrDiffFile::Modified(PathBuf::from("file2.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap(), SHA1::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap()),
MrDiffFile::New(
PathBuf::from("file1.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
),
MrDiffFile::Modified(
PathBuf::from("file2.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
SHA1::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(),
),
];

let page_size = 2u32;
Expand Down Expand Up @@ -771,9 +820,10 @@ mod test {
storage: Storage::mock(),
};

let files = vec![
MrDiffFile::New(PathBuf::from("new_file.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap()),
];
let files = vec![MrDiffFile::New(
PathBuf::from("new_file.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
)];

let mut old_blobs = Vec::new();
let mut new_blobs = Vec::new();
Expand All @@ -791,9 +841,10 @@ mod test {
storage: Storage::mock(),
};

let files = vec![
MrDiffFile::Deleted(PathBuf::from("deleted_file.txt"), SHA1::from_str("1234567890123456789012345678901234567890").unwrap()),
];
let files = vec![MrDiffFile::Deleted(
PathBuf::from("deleted_file.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
)];

let mut old_blobs = Vec::new();
let mut new_blobs = Vec::new();
Expand All @@ -811,13 +862,11 @@ mod test {
storage: Storage::mock(),
};

let files = vec![
MrDiffFile::Modified(
PathBuf::from("modified_file.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
SHA1::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap()
),
];
let files = vec![MrDiffFile::Modified(
PathBuf::from("modified_file.txt"),
SHA1::from_str("1234567890123456789012345678901234567890").unwrap(),
SHA1::from_str("abcdefabcdefabcdefabcdefabcdefabcdefabcd").unwrap(),
)];

let mut old_blobs = Vec::new();
let mut new_blobs = Vec::new();
Expand All @@ -837,12 +886,18 @@ mod test {
};

let files = vec![
MrDiffFile::New(PathBuf::from("new.txt"), SHA1::from_str("1111111111111111111111111111111111111111").unwrap()),
MrDiffFile::Deleted(PathBuf::from("deleted.txt"), SHA1::from_str("2222222222222222222222222222222222222222").unwrap()),
MrDiffFile::New(
PathBuf::from("new.txt"),
SHA1::from_str("1111111111111111111111111111111111111111").unwrap(),
),
MrDiffFile::Deleted(
PathBuf::from("deleted.txt"),
SHA1::from_str("2222222222222222222222222222222222222222").unwrap(),
),
MrDiffFile::Modified(
PathBuf::from("modified.txt"),
SHA1::from_str("3333333333333333333333333333333333333333").unwrap(),
SHA1::from_str("4444444444444444444444444444444444444444").unwrap()
SHA1::from_str("4444444444444444444444444444444444444444").unwrap(),
),
];

Expand Down
5 changes: 4 additions & 1 deletion orion-server/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,7 @@ utoipa.workspace = true
utoipa-swagger-ui = { workspace = true, features = ["axum"] }
chrono = { version = "0.4", features = ["serde"] }
reqwest.workspace = true
thiserror = "2.0"
thiserror = "2.0"

[dev-dependencies]
tempfile = "3"
Loading
Loading