fromost/sus-manager
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize rocksdb

Browse Source
This commit is contained in:
fromost
2025-10-26 01:31:57 +08:00
parent 9f6c81471e
commit da3acaaacb
6 changed files with 60 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/cli/sync.rs
View File

@@ -6,6 +6,7 @@ use colored::Colorize;
use crate::config::types::ApplicationConfig;
use crate::constants::{DB_CF_OPTIONS, DB_OPTIONS};
use crate::crawler::{dlsite, DLSiteCrawler};
use crate::helpers;
use crate::helpers::db::RocksDB;
use crate::models::DLSiteManiax;
@@ -63,29 +64,23 @@ impl SyncDLSiteCommand {
async fn sync_works(app_conf: &ApplicationConfig, db: &RocksDB) -> Result<()> {
let crawler = DLSiteCrawler::new();
let mut rj_nums: Vec<String> = Vec::new();
for path_str in app_conf.path_config.dlsite_paths.iter() {
let path = Path::new(path_str);
if !path.exists() {
return Err(eyre!("{} {}", path_str.blue(), "does not exist".red()));
let paths = app_conf.path_config.dlsite_paths.iter()
.map(|path| Path::new(path).to_path_buf())
.collect::<Vec<_>>();
let dirs = helpers::get_all_folders(&paths).await?;
for dir_path in dirs.iter() {
if !dir_path.is_dir() {
println!("{dir_path:?} is not a directory");
continue;
}
let dir_paths = path.read_dir()?
.filter_map(Result::ok)
.map(|e| e.path())
.collect::<Vec<_>>();
for dir_path in dir_paths.iter() {
if !dir_path.is_dir() {
println!("{dir_path:?} is not a directory");
continue;
}
let dir_name = dir_path
.file_name().unwrap()
.to_str().unwrap();
if !dlsite::is_valid_rj_number(dir_name) {
println!("{} {}", dir_path.to_str().unwrap().blue(), "is not a valid rj number, please add it manually".red());
continue;
}
rj_nums.push(dir_name.to_string());
let dir_name = dir_path
.file_name().unwrap()
.to_str().unwrap();
if !dlsite::is_valid_rj_number(dir_name) {
println!("{} {}", dir_path.to_str().unwrap().blue(), "is not a valid rj number, please add it manually".red());
continue;
}
rj_nums.push(dir_name.to_string());
}
let maniaxes = crawler.get_game_infos(rj_nums).await?;
db.set_values(&maniaxes)?;