feat: add load file while load the dirs from the server.

ceres/src/api_service/mod.rs

@@ -216,6 +216,92 @@ pub trait ApiHandler: Send + Sync {
         }
     }
 
+    /// the dir's hash as same as old,file's hash is the content hash
+    /// may think about change dir'hash as the content
+    /// for now,only change the file's hash
+    async fn get_tree_content_hash(&self, path: PathBuf) -> Result<Vec<TreeCommitItem>, GitError> {
+        let mut cache = GitObjectCache::default();
+        match self.search_tree_by_path(&path).await? {
+            Some(tree) => {
+                let mut item_to_commit = HashMap::new();
+
+                self.add_trees_to_map(
+                    &mut item_to_commit,
+                    tree.tree_items
+                        .iter()
+                        .filter(|x| x.mode == TreeItemMode::Tree)
+                        .map(|x| x.id.to_string())
+                        .collect(),
+                )
+                .await;
+
+                // self.add_blobs_to_map(
+                //     &mut item_to_commit,
+                //     tree.tree_items
+                //         .iter()
+                //         .filter(|x| x.mode == TreeItemMode::Blob)
+                //         .map(|x| x.id.to_string())
+                //         .collect(),
+                // )
+                // .await;
+                let content_items: Vec<TreeCommitItem> = tree
+                    .tree_items
+                    .iter()
+                    .filter(|x| x.mode == TreeItemMode::Blob)
+                    .map(|x| TreeCommitItem {
+                        oid: x.id.to_string(),
+                        name: x.name.clone(),
+                        content_type: "file".to_owned(),
+                        message: String::new(),
+                        date: String::new(),
+                    })
+                    .collect();
+                let commit_ids: HashSet<String> = item_to_commit.values().cloned().collect();
+                let commits = self
+                    .get_commits_by_hashes(commit_ids.into_iter().collect())
+                    .await
+                    .unwrap();
+                let commit_map: HashMap<String, Commit> =
+                    commits.into_iter().map(|x| (x.id.to_string(), x)).collect();
+
+                let mut root_commit: Option<Commit> = None;
+                let mut item_to_commit_map: HashMap<TreeItem, Option<Commit>> = HashMap::new();
+                for item in tree.tree_items {
+                    if item.mode == TreeItemMode::Blob {
+                        continue;
+                    }
+                    if let Some(commit_id) = item_to_commit.get(&item.id.to_string()) {
+                        let commit = if let Some(commit) = commit_map.get(commit_id) {
+                            commit.to_owned()
+                        } else {
+                            tracing::warn!("failed fetch from commit map: {}", commit_id);
+                            if root_commit.is_none() {
+                                root_commit = Some(self.get_root_commit().await);
+                            }
+                            let root_commit = root_commit.as_ref().unwrap().clone();
+                            self.traverse_commit_history(&path, &root_commit, &item, &mut cache)
+                                .await
+                        };
+                        item_to_commit_map.insert(item, Some(commit));
+                    }
+                }
+                let mut items: Vec<TreeCommitItem> = item_to_commit_map
+                    .into_iter()
+                    .map(TreeCommitItem::from)
+                    .collect();
+                // sort with type and date
+                items.sort_by(|a, b| {
+                    a.content_type
+                        .cmp(&b.content_type)
+                        .then(a.name.cmp(&b.name))
+                });
+                items.extend(content_items);
+                Ok(items)
+            }
+            None => Ok(Vec::new()),
+        }
+    }
+
     /// return the dir's hash only
     async fn get_tree_dir_hash(
         &self,

mono/src/api/api_router.rs

@@ -31,6 +31,7 @@ pub fn routers() -> OpenApiRouter<MonoApiServiceState> {
         .routes(routes!(create_file))
         .routes(routes!(get_latest_commit))
         .routes(routes!(get_tree_commit_info))
+        .routes(routes!(get_tree_content_hash))
         .routes(routes!(get_tree_dir_hash))
         .routes(routes!(path_can_be_cloned))
         .routes(routes!(get_tree_info))
@@ -173,6 +174,30 @@ async fn get_tree_commit_info(
     Ok(Json(CommonResult::success(Some(data))))
 }
 
+/// Get tree content hash,the dir's hash as same as old,file's hash is the content hash
+#[utoipa::path(
+    get,
+    path = "/tree/content-hash",
+    params(
+        CodePreviewQuery
+    ),
+    responses(
+        (status = 200, body = CommonResult<Vec<TreeCommitItem>>, content_type = "application/json")
+    ),
+    tag = GIT_TAG
+)]
+async fn get_tree_content_hash(
+    Query(query): Query<CodePreviewQuery>,
+    state: State<MonoApiServiceState>,
+) -> Result<Json<CommonResult<Vec<TreeCommitItem>>>, ApiError> {
+    let data = state
+        .api_handler(query.path.clone().into())
+        .await?
+        .get_tree_content_hash(query.path.into())
+        .await?;
+    Ok(Json(CommonResult::success(Some(data))))
+}
+
 /// return the dir's hash
 #[utoipa::path(
     get,

scorpio/src/dicfuse/async_io.rs

@@ -32,11 +32,8 @@ impl Filesystem for Dicfuse {
             .ok_or_else(|| std::io::Error::from_raw_os_error(libc::ENODATA))?;
 
         ppath.push(name.to_string_lossy().into_owned());
-        let chil = store.get_by_path(&ppath.to_string()).await?;
-        if self.open_buff.read().await.get(&chil.get_inode()).is_none() {
-            let _ = self.load_one_file(parent, name).await;
-        }
-        let re = self.get_stat(chil).await;
+        let child = store.get_by_path(&ppath.to_string()).await?;
+        let re = self.get_stat(child).await;
         Ok(re)
     }
     /// initialize filesystem. Called before any other filesystem method.
@@ -122,9 +119,9 @@ impl Filesystem for Dicfuse {
                 data: Bytes::from("".as_bytes()),
             });
         }
-        let read_lock = self.open_buff.read().await;
-        let datas = read_lock
-            .get(&inode)
+        let datas = self
+            .store
+            .get_file_content(inode)
             .ok_or_else(|| std::io::Error::from_raw_os_error(libc::ENOENT))?;
         let _offset = offset as usize;
         let end = (_offset + size as usize).min(datas.len());
@@ -201,8 +198,6 @@ impl Filesystem for Dicfuse {
         let items = self.store.do_readdir(parent, fh, offset).await?;
         let mut d: Vec<std::result::Result<DirectoryEntryPlus, Errno>> = Vec::new();
 
-        let parent_item = self.store.get_inode(parent).await?;
-        self.load_files(parent_item, &items).await;
         for (index, item) in items.into_iter().enumerate() {
             if index as u64 >= offset {
                 let attr = self.get_stat(item.clone()).await;

scorpio/src/dicfuse/content_store.rs

@@ -0,0 +1,37 @@
+use crate::util::config;
+use sled::Db;
+use std::io;
+use std::io::{Error, ErrorKind};
+
+pub struct ContentStorage {
+    db: Db,
+}
+#[allow(unused)]
+impl ContentStorage {
+    pub fn new_from_db(db: Db) -> Self {
+        ContentStorage { db }
+    }
+    pub fn new() -> io::Result<Self> {
+        let store_path = config::store_path();
+        let path = format!("{}/content.db", store_path);
+        let db = sled::open(path)?;
+        Ok(ContentStorage { db })
+    }
+    pub fn insert_file(&self, inode: u64, content: &[u8]) -> io::Result<()> {
+        self.db
+            .insert(inode.to_be_bytes(), content)
+            .map_err(Error::other)?;
+        Ok(())
+    }
+
+    pub fn get_file_content(&self, inode: u64) -> io::Result<Vec<u8>> {
+        match self.db.get(inode.to_be_bytes())? {
+            Some(value) => Ok(value.to_vec()),
+            None => Err(Error::new(ErrorKind::NotFound, "File not found")),
+        }
+    }
+    pub fn remove_file(&self, inode: u64) -> std::io::Result<()> {
+        self.db.remove(inode.to_be_bytes())?;
+        Ok(())
+    }
+}

scorpio/src/dicfuse/mod.rs

@@ -1,11 +1,11 @@
 mod abi;
 mod async_io;
+mod content_store;
 pub mod store;
 mod tree_store;
 use crate::manager::fetch::fetch_tree;
 use crate::util::config;
 use std::{
-    collections::HashMap,
     ffi::{OsStr, OsString},
     sync::Arc,
 };
@@ -19,23 +19,18 @@ use tree_store::StorageItem;
 pub struct Dicfuse {
     readable: bool,
     pub store: Arc<DictionaryStore>,
-    open_buff: Arc<tokio::sync::RwLock<HashMap<u64, Vec<u8>>>>,
 }
 #[allow(unused)]
 impl Dicfuse {
     pub async fn new() -> Self {
         Self {
             readable: config::dicfuse_readable(),
             store: DictionaryStore::new().await.into(), // Assuming DictionaryStore has a new() method
-            open_buff: Arc::new(tokio::sync::RwLock::new(HashMap::new())),
         }
     }
     pub async fn get_stat(&self, item: StorageItem) -> ReplyEntry {
         let mut e = item.get_stat();
-        let rl = self.open_buff.read().await;
-        if let Some(datas) = rl.get(&item.get_inode()) {
-            e.attr.size = datas.len() as u64;
-        }
+        e.attr.size = self.store.get_file_len(item.get_inode());
         e
     }
     async fn load_one_file(&self, parent: u64, name: &OsStr) -> std::io::Result<()> {
@@ -72,11 +67,7 @@ impl Dicfuse {
                 parent_item.push(i.name.clone());
 
                 let it_temp = self.store.get_by_path(&parent_item.to_string()).await?;
-
-                self.open_buff
-                    .write()
-                    .await
-                    .insert(it_temp.get_inode(), data);
+                self.store.save_file(it_temp.get_inode(), data);
             } else {
                 eprintln!("Request failed with status: {}", response.status());
             }
@@ -88,12 +79,7 @@ impl Dicfuse {
         if !self.readable {
             return;
         }
-        if self
-            .open_buff
-            .read()
-            .await
-            .contains_key(&parent_item.get_inode())
-        {
+        if self.store.file_exists(parent_item.get_inode()) {
             return;
         }
         let gpath = self.store.find_path(parent_item.get_inode()).await.unwrap();
@@ -131,26 +117,18 @@ impl Dicfuse {
 
                 // Look up the buff, find Loaded file.
                 if is_first {
-                    match self.open_buff.write().await.get(&hit_inodes) {
-                        Some(_) => {
-                            break;
-                            // is loaded , no need to reload ;
-                        }
-                        None => {
-                            // this dictionary is not loaded ,  just go ahead.
-                            is_first = false;
-                        }
+                    if self.store.file_exists(hit_inodes) {
+                        // if the file is already exists, no need to load again.
+                        break;
                     }
+                    self.store.save_file(hit_inodes, data);
+                    is_first = false;
                 }
-                self.open_buff.write().await.insert(hit_inodes, data);
             } else {
                 eprintln!("Request failed with status: {}", response.status());
             }
         }
-        self.open_buff
-            .write()
-            .await
-            .insert(parent_item.get_inode(), Vec::new());
+        self.store.save_file(parent_item.get_inode(), Vec::new());
     }
 }