From 3f5dee13f54af9605181581cbb73047429db8e2e Mon Sep 17 00:00:00 2001 From: fromost Date: Mon, 27 Oct 2025 20:37:42 +0800 Subject: [PATCH] Refactor config --- .config/config.toml | 3 + Cargo.toml | 13 +++- src/cli/mod.rs | 27 ++------- src/cli/sync.rs | 136 ++++++++++++++++++++++++++---------------- src/config/mod.rs | 9 +++ src/config/types.rs | 4 +- src/crawler/dlsite.rs | 42 +++++++------ src/helpers/db.rs | 9 ++- src/helpers/mod.rs | 15 +++-- src/lib.rs | 4 ++ src/models/game.rs | 40 +++++++++---- 11 files changed, 188 insertions(+), 114 deletions(-) create mode 100644 .config/config.toml create mode 100644 src/lib.rs diff --git a/.config/config.toml b/.config/config.toml new file mode 100644 index 0000000..8cc9c0b --- /dev/null +++ b/.config/config.toml @@ -0,0 +1,3 @@ +[env] +TERM = "xterm-256color" +RUST_BACKTRACE = 0 \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index b67deaa..ef68a05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,9 +21,18 @@ scraper = "0.24.0" rat-cursor = "1.2.1" serde_json = "1.0.145" image = "0.25.8" -colored = "3.0.0" log = "0.4.28" num_cpus = "1.17.0" +sys-locale = "0.3.2" +jemallocator = "0.5.4" + +[dependencies.language-tags] +version = "0.3.2" +features = ["serde"] + +[dependencies.indicatif] +version = "0.18.1" +features = ["futures", "tokio"] [dependencies.rocksdb] version = "0.24.0" @@ -52,7 +61,7 @@ features = ["derive", "cargo"] [dependencies.reqwest] version = "0.12.23" -features = ["blocking", "json"] +features = ["blocking", "json", "rustls-tls"] [dependencies.tokio] version = "1.47.1" diff --git a/src/cli/mod.rs b/src/cli/mod.rs index f0e7e1d..267b22c 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -6,12 +6,13 @@ use clap::{command, Args, Command, Parser, Subcommand}; use color_eyre::Result; use ratatui::crossterm; use crate::cli::folder::FolderCommand; -use crate::cli::sync::SyncCommand; +use crate::cli::sync::DLSiteCommand; #[derive(Parser, Debug)] enum CliSubCommand { Folder(FolderCommand), - Sync(SyncCommand), + #[command(name = "dlsite")] + DLSite(DLSiteCommand), } #[derive(Parser, Debug)] @@ -21,26 +22,6 @@ pub(crate) struct Cli { subcommand: Option, } -impl Subcommand for Cli { - fn augment_subcommands(cmd: Command) -> Command { - cmd.subcommand(FolderCommand::augment_args(Command::new("folder"))) - .subcommand_required(true) - .subcommand(SyncCommand::augment_args(Command::new("sync"))) - .subcommand_required(true) - } - - fn augment_subcommands_for_update(cmd: Command) -> Command { - cmd.subcommand(FolderCommand::augment_args(Command::new("folder"))) - .subcommand_required(true) - .subcommand(SyncCommand::augment_args(Command::new("sync"))) - .subcommand_required(true) - } - - fn has_subcommand(name: &str) -> bool { - matches!(name, "folder" | "sync") - } -} - impl Cli { pub async fn run(&self) -> Result<()> { helpers::initialize_folders().await?; @@ -70,7 +51,7 @@ impl CliSubCommand { pub async fn handle(&self) -> Result<()> { match self { CliSubCommand::Folder(cmd) => cmd.subcommand.handle().await, - CliSubCommand::Sync(cmd) => cmd.subcommand.handle().await, + CliSubCommand::DLSite(cmd) => cmd.subcommand.handle().await, } } } diff --git a/src/cli/sync.rs b/src/cli/sync.rs index 38074b9..167eebf 100644 --- a/src/cli/sync.rs +++ b/src/cli/sync.rs @@ -1,7 +1,10 @@ -use std::path::Path; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; use clap::{Args, Command, Parser, Subcommand}; use color_eyre::eyre::Result; -use colored::Colorize; +use crossterm::style::{style, Stylize}; +use futures::StreamExt; +use indicatif::{ProgressBar, ProgressStyle}; use tokio::time::Instant; use crate::models; use crate::config::types::ApplicationConfig; @@ -11,51 +14,43 @@ use crate::helpers; use crate::helpers::db::RocksDB; #[derive(Parser, Debug)] -pub(super) struct SyncCommand { +pub(super) struct DLSiteCommand { #[command(subcommand)] - pub(super) subcommand: SyncSubCommand, + pub(super) subcommand: DLSiteSubCommand, } #[derive(Parser, Debug)] -pub(super) enum SyncSubCommand { - DLSite(SyncDLSiteCommand) +pub(super) enum DLSiteSubCommand { + #[command(name = "sync")] + Sync(DLSiteSyncCommand) } #[derive(Parser, Debug)] -pub(super) struct SyncDLSiteCommand; - -impl Subcommand for SyncCommand { - fn augment_subcommands(cmd: Command) -> Command { - cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite"))) - .subcommand_required(true) - } - - fn augment_subcommands_for_update(cmd: Command) -> Command { - cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite"))) - .subcommand_required(true) - } - - fn has_subcommand(name: &str) -> bool { - matches!(name, "dlsite") - } +pub(super) struct DLSiteSyncCommand { + #[clap(long, short, action)] + missing: bool, + #[clap(long, short, action)] + genre: bool, } -impl SyncSubCommand { - pub async fn handle(&self) -> color_eyre::Result<()> { +impl DLSiteSubCommand { + pub async fn handle(&self) -> Result<()> { match self { - Self::DLSite(cmd) => cmd.handle().await, + Self::Sync(cmd) => cmd.handle().await, } } } -impl SyncDLSiteCommand { - pub async fn handle(&self) -> color_eyre::Result<()> { +impl DLSiteSyncCommand { + pub async fn handle(&self) -> Result<()> { let now = Instant::now(); let app_conf = ApplicationConfig::get_config()?; let mut db = RocksDB::new(DB_OPTIONS.clone(), DB_CF_OPTIONS.clone())?; - Self::sync_genres(&app_conf).await?; - Self::sync_works(&app_conf, &mut db).await?; - println!("{} Done in {:.2?}", "Syncing".green(), now.elapsed()); + if self.genre { + Self::sync_genres(&app_conf).await?; + } + self.sync_works(&app_conf, &mut db).await?; + println!("{} Done in {:.2?}", style("Syncing").green(), now.elapsed()); Ok(()) } @@ -63,31 +58,68 @@ impl SyncDLSiteCommand { Ok(()) } - async fn sync_works(app_conf: &ApplicationConfig, db: &mut RocksDB) -> Result<()> { + async fn sync_works(&self, app_conf: &ApplicationConfig, db: &mut RocksDB) -> Result<()> { let crawler = DLSiteCrawler::new(); - let mut rj_nums: Vec = Vec::new(); - let config_paths = app_conf.path_config.dlsite_paths.iter() - .map(|path| Path::new(path).to_path_buf()) - .collect::>(); - let dir_paths = helpers::get_all_folders(&config_paths).await?; - for dir_path in dir_paths.iter() { - if !dir_path.is_dir() { - println!("{dir_path:?} is not a directory"); - continue; - } - let dir_name = dir_path - .file_name().unwrap() - .to_str().unwrap(); - if !dlsite::is_valid_rj_number(dir_name) { - println!("{} {}", dir_path.to_str().unwrap().blue(), "is not a valid rj number, please add it manually".red()); - continue; - } - rj_nums.push(dir_name.to_string()); + let existing_works = db.get_all_values::()?; + + let work_list = self.get_work_list(&app_conf, existing_works).await?; + let rj_nums = work_list.clone().into_keys().collect::>(); + let mut maniaxes: Vec = Vec::new(); + let mut game_infos = crawler.get_game_infos(rj_nums).await?; + + let progress = ProgressBar::new(game_infos.len() as u64) + .with_style(ProgressStyle::default_bar()); + while let Some(info) = game_infos.next().await { + let mut value: models::DLSiteManiax = info?.into(); + let maniax_folder = work_list.get(&value.rj_num).unwrap().to_owned(); + value.folder_path = maniax_folder; + maniaxes.push(value); + progress.inc(1); } - let maniaxes: Vec = crawler.get_game_infos(rj_nums).await?.into_iter() - .map(|x| x.into()) - .collect::>(); db.set_values(&maniaxes)?; + Ok(()) } + + async fn get_work_list(&self, app_conf: &ApplicationConfig, existing_works: Vec) -> Result> { + let existing_nums = existing_works.iter() + .map(|x| x.rj_num.clone()) + .collect::>(); + let existing_folders = existing_works.iter() + .map(|x| x.folder_path.to_str().unwrap().to_string()) + .collect::>(); + let mut works_list: HashMap = HashMap::new(); + let config_paths = app_conf.path_config.dlsite_paths.iter() + .map(|path| Path::new(path)) + .collect::>(); + let dir_paths = helpers::get_all_folders(config_paths).await?; + for dir_path in dir_paths { + if !dir_path.is_dir() { + println!( + "{} {}", + style(dir_path.to_str().unwrap()).blue(), + style("is not a directory").red() + ); + continue; + } + let dir_path_str = dir_path.to_str().unwrap().to_string(); + let dir_name = dir_path + .file_name().unwrap() + .to_str().unwrap() + .to_string(); + if !dlsite::is_valid_rj_number(&dir_name) && !existing_folders.contains(&dir_path_str) { + println!( + "{} {}", + style(dir_path.to_str().unwrap()).blue(), + style("is not a valid rj number, please add it manually").red() + ); + continue; + } + if self.missing && existing_nums.contains(&dir_name) { + continue; + } + works_list.insert(dir_name, dir_path); + } + Ok(works_list) + } } \ No newline at end of file diff --git a/src/config/mod.rs b/src/config/mod.rs index 60934c8..ef05e53 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -2,10 +2,17 @@ use crate::config::types::{ApplicationConfig, BasicConfig, PathConfig}; use crate::constants::{APP_CONIFG_FILE_PATH, APP_DB_DATA_DIR}; use color_eyre::Result; use std::path::PathBuf; +use language_tags::LanguageTag; +use ratatui::widgets::ListState; use serde_json; pub mod types; +pub(crate) struct GameList { + games: Vec, + state: ListState, +} + impl ApplicationConfig { pub fn get_config() -> Result { if APP_CONIFG_FILE_PATH.exists() { @@ -22,10 +29,12 @@ impl ApplicationConfig { } fn new() -> Self { + let default_locale = sys_locale::get_locale().unwrap_or(String::from("ja-JP")); let conf = Self { basic_config: BasicConfig { db_path: APP_DB_DATA_DIR.to_str().unwrap().to_string(), tick_rate: 250, + locale: LanguageTag::parse(&default_locale).unwrap(), }, path_config: PathConfig { dlsite_paths: vec![], diff --git a/src/config/types.rs b/src/config/types.rs index 84a294d..0562a6c 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -1,3 +1,4 @@ +use language_tags::LanguageTag; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Serialize, Deserialize)] @@ -9,10 +10,11 @@ pub struct ApplicationConfig { #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct BasicConfig { pub db_path: String, + pub locale: LanguageTag, pub tick_rate: u64, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct PathConfig { pub dlsite_paths: Vec -} +} \ No newline at end of file diff --git a/src/crawler/dlsite.rs b/src/crawler/dlsite.rs index dbf2da8..6e60a1f 100644 --- a/src/crawler/dlsite.rs +++ b/src/crawler/dlsite.rs @@ -1,16 +1,14 @@ + use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use color_eyre::eyre::eyre; use color_eyre::owo_colors::OwoColorize; -use reqwest::Url; +use reqwest::{StatusCode, Url}; use color_eyre::{Report, Result}; -use colored::Colorize; use futures::stream::FuturesUnordered; -use futures::StreamExt; use lazy_static::lazy_static; use scraper::{Html, Selector}; use serde::{Deserialize, Serialize}; -use tokio::time::Instant; use crate::constants::{APP_DATA_DIR}; use crate::crawler::Crawler; @@ -40,6 +38,13 @@ pub(crate) struct DLSiteManiax { pub(crate) genre_ids: Vec, #[serde(skip)] pub(crate) rj_num: String, + #[serde(skip)] + pub(crate) folder_path: PathBuf, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub(crate) struct DLSiteGenreCategory { + } impl DLSiteCrawler { @@ -49,7 +54,8 @@ impl DLSiteCrawler { } } - pub async fn get_game_infos(&self, rj_nums: Vec) -> Result> { + pub async fn get_game_infos(&self, rj_nums: Vec) -> Result>>> + { let invalid_nums = rj_nums.iter() .filter(|&n| !is_valid_rj_number(n)) .map(|n| n.to_string()) @@ -61,17 +67,23 @@ impl DLSiteCrawler { } let query = &format!("product_id={}", rj_nums.join(",")); - let (maniax_result, _) = self.crawler - .get_json::>(DLSITE_PRODUCT_API_ENDPOINT, Some(query)) + let (value, _) = self.crawler + .get_json::(DLSITE_PRODUCT_API_ENDPOINT, Some(query)) .await?; + // try to catch '[]' empty result from the api + let value_downcast_result: Result, _> = serde_json::from_value(value); + let maniax_result = value_downcast_result.unwrap_or(HashMap::new()); Self::verify_all_works_exists(&maniax_result, rj_nums); - let mut tasks = FuturesUnordered::new(); + let tasks = FuturesUnordered::new(); for (rj_num, mut info) in maniax_result { tasks.push(async { let html_path = format!("{DLSITE_MANIAX_PATH}{rj_num}"); - let (_, html_result) = tokio::join!(self.save_main_image(&info, &rj_num), self.crawler.get_html(&html_path)); + let (_, html_result) = tokio::join!( + self.save_main_image(&info, &rj_num), + self.crawler.get_html(&html_path) + ); let (html, _) = html_result?; let genres = self.get_genres(&html).await?; info.genre_ids = genres; @@ -79,12 +91,7 @@ impl DLSiteCrawler { Ok::(info) }) } - let mut maniax_infos = Vec::new(); - while let Some(result) = tasks.next().await { - maniax_infos.push(result?); - } - - Ok(maniax_infos) + Ok(tasks) } fn verify_all_works_exists(maniax_result: &HashMap, rj_nums: Vec) { @@ -123,8 +130,9 @@ impl DLSiteCrawler { ); let result = html.select(&selector).next().unwrap(); let genre_rows = result.child_elements().collect::>(); - let genre_len = genre_rows.iter().count(); - let genre_row = genre_rows.iter().skip(genre_len - 2).next().unwrap(); + let genre_rows_len = genre_rows.iter().count(); + // get second last row for genre + let genre_row = genre_rows.iter().skip(genre_rows_len - 2).next().unwrap(); let data = genre_row .child_elements().skip(1).next().unwrap() .child_elements().next().unwrap(); diff --git a/src/helpers/db.rs b/src/helpers/db.rs index a21c21d..f067c2f 100644 --- a/src/helpers/db.rs +++ b/src/helpers/db.rs @@ -1,9 +1,11 @@ -use crate::constants::{APP_DB_DATA_DIR, DB_COLUMNS}; -use rocksdb::{ColumnFamilyDescriptor, IteratorMode, OptimisticTransactionDB, Options, ReadOptions}; +use std::path::Path; +use crate::constants::{DB_COLUMNS}; +use rocksdb::{ColumnFamilyDescriptor, IteratorMode, OptimisticTransactionDB, Options}; use serde::{Serialize}; use serde::de::DeserializeOwned; use crate::models::{RocksColumn, RocksReference, RocksReferences}; use color_eyre::Result; +use crate::config::types::ApplicationConfig; pub struct RocksDB { db: OptimisticTransactionDB, @@ -11,12 +13,13 @@ pub struct RocksDB { impl RocksDB { pub fn new(db_opts: Options, cf_opts: Options) -> Result { + let app_conf = ApplicationConfig::get_config()?; let cfs = DB_COLUMNS.iter() .map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), cf_opts.clone())) .collect::>(); let db = OptimisticTransactionDB::open_cf_descriptors( &db_opts, - APP_DB_DATA_DIR.as_path(), + Path::new(&app_conf.basic_config.db_path), cfs )?; let rocks = Self { diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index 72ef5d1..dc3f760 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -1,14 +1,16 @@ pub mod db; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use color_eyre::eyre::eyre; use color_eyre::owo_colors::OwoColorize; use tokio::fs; -use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR, APP_DB_DATA_DIR}; +use crate::config::types::ApplicationConfig; +use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR}; use crate::crawler::DLSITE_IMG_FOLDER; pub async fn initialize_folders() -> color_eyre::Result<()> { + let app_conf = ApplicationConfig::get_config()?; if !APP_CONFIG_DIR.exists() { fs::create_dir_all(APP_CONFIG_DIR.as_path()).await?; } @@ -18,16 +20,17 @@ pub async fn initialize_folders() -> color_eyre::Result<()> { if !DLSITE_IMG_FOLDER.exists() { fs::create_dir_all(DLSITE_IMG_FOLDER.as_path()).await?; } - if !APP_DB_DATA_DIR.exists() { - fs::create_dir_all(APP_DB_DATA_DIR.as_path()).await?; + let db_path = Path::new(&app_conf.basic_config.db_path); + if !db_path.exists() { + fs::create_dir_all(db_path).await?; } Ok(()) } -pub async fn get_all_folders(paths: &Vec) -> color_eyre::Result> { +pub async fn get_all_folders(paths: Vec<&Path>) -> color_eyre::Result> { let mut folders: Vec = Vec::new(); for path in paths { - let path = path.as_path(); + let path = path.to_path_buf(); if !path.exists() { return Err(eyre!("{:?} {}", path.blue(), "does not exist".red())); } diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..5536e89 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,4 @@ +extern crate jemallocator; + +#[global_allocator] +static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc; \ No newline at end of file diff --git a/src/models/game.rs b/src/models/game.rs index 65bd85e..dd0d738 100644 --- a/src/models/game.rs +++ b/src/models/game.rs @@ -1,19 +1,19 @@ -use ratatui::widgets::ListState; +use std::path::PathBuf; +use color_eyre::{eyre, Report}; +use language_tags::LanguageTag; use serde::{Deserialize, Serialize}; +use crate::config::types::ApplicationConfig; use crate::models::{RocksColumn, RocksReference, RocksReferences}; -pub(crate) struct GameList { - games: Vec, - state: ListState, -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct DLSiteManiax { #[serde(skip)] pub rj_num: String, pub genre_ids: Vec, - pub name: String, - pub sells_count: u32 + pub name: Vec, + pub sells_count: u32, + pub folder_path: PathBuf, + pub version: Option } impl From for DLSiteManiax { @@ -21,8 +21,10 @@ impl From for DLSiteManiax { Self { rj_num: value.rj_num, genre_ids: value.genre_ids, - name: value.title, - sells_count: value.sells_count + name: vec![], + sells_count: value.sells_count, + folder_path: value.folder_path, + version: None } } } @@ -104,4 +106,22 @@ impl RocksColumn for DLSiteCategory { #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) enum DLSiteTranslation { EN(String), JP(String) +} + +impl TryFrom<&str> for DLSiteTranslation { + type Error = Report; + fn try_from(value: &str) -> color_eyre::Result { + let app_conf = ApplicationConfig::get_config()?; + let locale = app_conf.basic_config.locale; + + let en_locale = LanguageTag::parse("en-US")?; + if locale.matches(&en_locale) { + return Ok(DLSiteTranslation::EN(value.to_string())); + } + let jp_locale = LanguageTag::parse("ja-JP")?; + if locale.matches(&jp_locale) { + return Ok(DLSiteTranslation::JP(value.to_string())); + } + Err(eyre::eyre!("Invalid Locale: {:?}; Support {:?}", locale, [en_locale, jp_locale])) + } } \ No newline at end of file