Refactor config

.config/config.toml

@@ -0,0 +1,3 @@
+[env]
+TERM = "xterm-256color"
+RUST_BACKTRACE = 0

Cargo.toml

@@ -21,9 +21,18 @@ scraper = "0.24.0"
 rat-cursor = "1.2.1"
 serde_json = "1.0.145"
 image = "0.25.8"
-colored = "3.0.0"
 log = "0.4.28"
 num_cpus = "1.17.0"
+sys-locale = "0.3.2"
+jemallocator = "0.5.4"
+
+[dependencies.language-tags]
+version = "0.3.2"
+features = ["serde"]
+
+[dependencies.indicatif]
+version = "0.18.1"
+features = ["futures", "tokio"]
 
 [dependencies.rocksdb]
 version = "0.24.0"
@@ -52,7 +61,7 @@ features = ["derive", "cargo"]
 
 [dependencies.reqwest]
 version = "0.12.23"
-features = ["blocking", "json"]
+features = ["blocking", "json", "rustls-tls"]
 
 [dependencies.tokio]
 version = "1.47.1"

src/cli/mod.rs

@@ -6,12 +6,13 @@ use clap::{command, Args, Command, Parser, Subcommand};
 use color_eyre::Result;
 use ratatui::crossterm;
 use crate::cli::folder::FolderCommand;
-use crate::cli::sync::SyncCommand;
+use crate::cli::sync::DLSiteCommand;
 
 #[derive(Parser, Debug)]
 enum CliSubCommand {
     Folder(FolderCommand),
-    Sync(SyncCommand),
+    #[command(name = "dlsite")]
+    DLSite(DLSiteCommand),
 }
 
 #[derive(Parser, Debug)]
@@ -21,26 +22,6 @@ pub(crate) struct Cli {
     subcommand: Option<CliSubCommand>,
 }
 
-impl Subcommand for Cli {
-    fn augment_subcommands(cmd: Command) -> Command {
-        cmd.subcommand(FolderCommand::augment_args(Command::new("folder")))
-            .subcommand_required(true)
-            .subcommand(SyncCommand::augment_args(Command::new("sync")))
-            .subcommand_required(true)
-    }
-
-    fn augment_subcommands_for_update(cmd: Command) -> Command {
-        cmd.subcommand(FolderCommand::augment_args(Command::new("folder")))
-            .subcommand_required(true)
-            .subcommand(SyncCommand::augment_args(Command::new("sync")))
-            .subcommand_required(true)
-    }
-
-    fn has_subcommand(name: &str) -> bool {
-        matches!(name, "folder" | "sync")
-    }
-}
-
 impl Cli {
     pub async fn run(&self) -> Result<()> {
         helpers::initialize_folders().await?;
@@ -70,7 +51,7 @@ impl CliSubCommand {
     pub async fn handle(&self) -> Result<()> {
         match self {
             CliSubCommand::Folder(cmd) => cmd.subcommand.handle().await,
-            CliSubCommand::Sync(cmd) => cmd.subcommand.handle().await,
+            CliSubCommand::DLSite(cmd) => cmd.subcommand.handle().await,
         }
     }
 }

src/cli/sync.rs

@@ -1,7 +1,10 @@
-use std::path::Path;
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
 use clap::{Args, Command, Parser, Subcommand};
 use color_eyre::eyre::Result;
-use colored::Colorize;
+use crossterm::style::{style, Stylize};
+use futures::StreamExt;
+use indicatif::{ProgressBar, ProgressStyle};
 use tokio::time::Instant;
 use crate::models;
 use crate::config::types::ApplicationConfig;
@@ -11,51 +14,43 @@ use crate::helpers;
 use crate::helpers::db::RocksDB;
 
 #[derive(Parser, Debug)]
-pub(super) struct SyncCommand {
+pub(super) struct DLSiteCommand {
     #[command(subcommand)]
-    pub(super) subcommand: SyncSubCommand,
+    pub(super) subcommand: DLSiteSubCommand,
 }
 
 #[derive(Parser, Debug)]
-pub(super) enum SyncSubCommand {
+pub(super) enum DLSiteSubCommand {
-    DLSite(SyncDLSiteCommand)
+    #[command(name = "sync")]
+    Sync(DLSiteSyncCommand)
 }
 
 #[derive(Parser, Debug)]
-pub(super) struct SyncDLSiteCommand;
+pub(super) struct DLSiteSyncCommand {
-
+    #[clap(long, short, action)]
-impl Subcommand for SyncCommand {
+    missing: bool,
-    fn augment_subcommands(cmd: Command) -> Command {
+    #[clap(long, short, action)]
-        cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite")))
+    genre: bool,
-            .subcommand_required(true)
 }
 
-    fn augment_subcommands_for_update(cmd: Command) -> Command {
+impl DLSiteSubCommand {
-        cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite")))
+    pub async fn handle(&self) -> Result<()> {
-            .subcommand_required(true)
-    }
-
-    fn has_subcommand(name: &str) -> bool {
-        matches!(name, "dlsite")
-    }
-}
-
-impl SyncSubCommand {
-    pub async fn handle(&self) -> color_eyre::Result<()> {
         match self {
-            Self::DLSite(cmd) => cmd.handle().await,
+            Self::Sync(cmd) => cmd.handle().await,
         }
     }
 }
 
-impl SyncDLSiteCommand {
+impl DLSiteSyncCommand {
-    pub async fn handle(&self) -> color_eyre::Result<()> {
+    pub async fn handle(&self) -> Result<()> {
         let now = Instant::now();
         let app_conf = ApplicationConfig::get_config()?;
         let mut db = RocksDB::new(DB_OPTIONS.clone(), DB_CF_OPTIONS.clone())?;
+        if self.genre {
             Self::sync_genres(&app_conf).await?;
-        Self::sync_works(&app_conf, &mut db).await?;
+        }
-        println!("{} Done in {:.2?}", "Syncing".green(), now.elapsed());
+        self.sync_works(&app_conf, &mut db).await?;
+        println!("{} Done in {:.2?}", style("Syncing").green(), now.elapsed());
         Ok(())
     }
 
@@ -63,31 +58,68 @@ impl SyncDLSiteCommand {
         Ok(())
     }
 
-    async fn sync_works(app_conf: &ApplicationConfig, db: &mut RocksDB) -> Result<()> {
+    async fn sync_works(&self, app_conf: &ApplicationConfig, db: &mut RocksDB) -> Result<()> {
         let crawler = DLSiteCrawler::new();
-        let mut rj_nums: Vec<String> = Vec::new();
+        let existing_works = db.get_all_values::<models::DLSiteManiax>()?;
-        let config_paths = app_conf.path_config.dlsite_paths.iter()
+
-            .map(|path| Path::new(path).to_path_buf())
+        let work_list = self.get_work_list(&app_conf, existing_works).await?;
-            .collect::<Vec<_>>();
+        let rj_nums = work_list.clone().into_keys().collect::<Vec<_>>();
-        let dir_paths = helpers::get_all_folders(&config_paths).await?;
+        let mut maniaxes: Vec<models::DLSiteManiax> = Vec::new();
-        for dir_path in dir_paths.iter() {
+        let mut game_infos = crawler.get_game_infos(rj_nums).await?;
-            if !dir_path.is_dir() {
+
-                println!("{dir_path:?} is not a directory");
+        let progress = ProgressBar::new(game_infos.len() as u64)
-                continue;
+            .with_style(ProgressStyle::default_bar());
+        while let Some(info) = game_infos.next().await {
+            let mut value: models::DLSiteManiax = info?.into();
+            let maniax_folder = work_list.get(&value.rj_num).unwrap().to_owned();
+            value.folder_path = maniax_folder;
+            maniaxes.push(value);
+            progress.inc(1);
         }
-            let dir_name = dir_path
-                .file_name().unwrap()
-                .to_str().unwrap();
-            if !dlsite::is_valid_rj_number(dir_name) {
-                println!("{} {}", dir_path.to_str().unwrap().blue(), "is not a valid rj number, please add it manually".red());
-                continue;
-            }
-            rj_nums.push(dir_name.to_string());
-        }
-        let maniaxes: Vec<models::DLSiteManiax> = crawler.get_game_infos(rj_nums).await?.into_iter()
-            .map(|x| x.into())
-            .collect::<Vec<_>>();
         db.set_values(&maniaxes)?;
+
         Ok(())
     }
+
+    async fn get_work_list(&self, app_conf: &ApplicationConfig, existing_works: Vec<models::DLSiteManiax>) -> Result<HashMap<String, PathBuf>> {
+        let existing_nums = existing_works.iter()
+            .map(|x| x.rj_num.clone())
+            .collect::<Vec<_>>();
+        let existing_folders = existing_works.iter()
+            .map(|x| x.folder_path.to_str().unwrap().to_string())
+            .collect::<Vec<_>>();
+        let mut works_list: HashMap<String, PathBuf> = HashMap::new();
+        let config_paths = app_conf.path_config.dlsite_paths.iter()
+            .map(|path| Path::new(path))
+            .collect::<Vec<_>>();
+        let dir_paths = helpers::get_all_folders(config_paths).await?;
+        for dir_path in dir_paths {
+            if !dir_path.is_dir() {
+                println!(
+                    "{} {}",
+                    style(dir_path.to_str().unwrap()).blue(),
+                    style("is not a directory").red()
+                );
+                continue;
+            }
+            let dir_path_str = dir_path.to_str().unwrap().to_string();
+            let dir_name = dir_path
+                .file_name().unwrap()
+                .to_str().unwrap()
+                .to_string();
+            if !dlsite::is_valid_rj_number(&dir_name) && !existing_folders.contains(&dir_path_str) {
+                println!(
+                    "{} {}",
+                    style(dir_path.to_str().unwrap()).blue(),
+                    style("is not a valid rj number, please add it manually").red()
+                );
+                continue;
+            }
+            if self.missing && existing_nums.contains(&dir_name) {
+                continue;
+            }
+            works_list.insert(dir_name, dir_path);
+        }
+        Ok(works_list)
+    }
 }

src/config/mod.rs

@@ -2,10 +2,17 @@ use crate::config::types::{ApplicationConfig, BasicConfig, PathConfig};
 use crate::constants::{APP_CONIFG_FILE_PATH, APP_DB_DATA_DIR};
 use color_eyre::Result;
 use std::path::PathBuf;
+use language_tags::LanguageTag;
+use ratatui::widgets::ListState;
 use serde_json;
 
 pub mod types;
 
+pub(crate) struct GameList<T> {
+    games: Vec<T>,
+    state: ListState,
+}
+
 impl ApplicationConfig {
     pub fn get_config() -> Result<Self> {
         if APP_CONIFG_FILE_PATH.exists() {
@@ -22,10 +29,12 @@ impl ApplicationConfig {
     }
 
     fn new() -> Self {
+        let default_locale = sys_locale::get_locale().unwrap_or(String::from("ja-JP"));
         let conf = Self {
             basic_config: BasicConfig {
                 db_path: APP_DB_DATA_DIR.to_str().unwrap().to_string(),
                 tick_rate: 250,
+                locale: LanguageTag::parse(&default_locale).unwrap(),
             },
             path_config: PathConfig {
                 dlsite_paths: vec![],

src/config/types.rs

@@ -1,3 +1,4 @@
+use language_tags::LanguageTag;
 use serde::{Deserialize, Serialize};
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -9,6 +10,7 @@ pub struct ApplicationConfig {
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub(crate) struct BasicConfig {
     pub db_path: String,
+    pub locale: LanguageTag,
     pub tick_rate: u64,
 }
 

src/crawler/dlsite.rs

@@ -1,16 +1,14 @@
+
 use std::collections::{HashMap, HashSet};
 use std::path::PathBuf;
 use color_eyre::eyre::eyre;
 use color_eyre::owo_colors::OwoColorize;
-use reqwest::Url;
+use reqwest::{StatusCode, Url};
 use color_eyre::{Report, Result};
-use colored::Colorize;
 use futures::stream::FuturesUnordered;
-use futures::StreamExt;
 use lazy_static::lazy_static;
 use scraper::{Html, Selector};
 use serde::{Deserialize, Serialize};
-use tokio::time::Instant;
 use crate::constants::{APP_DATA_DIR};
 use crate::crawler::Crawler;
 
@@ -40,6 +38,13 @@ pub(crate) struct DLSiteManiax {
     pub(crate) genre_ids: Vec<u16>,
     #[serde(skip)]
     pub(crate) rj_num: String,
+    #[serde(skip)]
+    pub(crate) folder_path: PathBuf,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub(crate) struct DLSiteGenreCategory {
+
 }
 
 impl DLSiteCrawler {
@@ -49,7 +54,8 @@ impl DLSiteCrawler {
         }
     }
 
-    pub async fn get_game_infos(&self, rj_nums: Vec<String>) -> Result<Vec<DLSiteManiax>> {
+    pub async fn get_game_infos(&self, rj_nums: Vec<String>) -> Result<FuturesUnordered<impl Future<Output=Result<DLSiteManiax, Report>>>>
+    {
         let invalid_nums = rj_nums.iter()
             .filter(|&n| !is_valid_rj_number(n))
             .map(|n| n.to_string())
@@ -61,17 +67,23 @@ impl DLSiteCrawler {
         }
 
         let query = &format!("product_id={}", rj_nums.join(","));
-        let (maniax_result, _) = self.crawler
+        let (value, _) = self.crawler
-            .get_json::<HashMap<String, DLSiteManiax>>(DLSITE_PRODUCT_API_ENDPOINT, Some(query))
+            .get_json::<serde_json::Value>(DLSITE_PRODUCT_API_ENDPOINT, Some(query))
             .await?;
+        // try to catch '[]' empty result from the api
+        let value_downcast_result: Result<HashMap<String, DLSiteManiax>, _> = serde_json::from_value(value);
+        let maniax_result = value_downcast_result.unwrap_or(HashMap::new());
 
         Self::verify_all_works_exists(&maniax_result, rj_nums);
 
-        let mut tasks = FuturesUnordered::new();
+        let tasks = FuturesUnordered::new();
         for (rj_num, mut info) in maniax_result {
             tasks.push(async {
                 let html_path = format!("{DLSITE_MANIAX_PATH}{rj_num}");
-                let (_, html_result) = tokio::join!(self.save_main_image(&info, &rj_num), self.crawler.get_html(&html_path));
+                let (_, html_result) = tokio::join!(
+                    self.save_main_image(&info, &rj_num),
+                    self.crawler.get_html(&html_path)
+                );
                 let (html, _) = html_result?;
                 let genres = self.get_genres(&html).await?;
                 info.genre_ids = genres;
@@ -79,12 +91,7 @@ impl DLSiteCrawler {
                 Ok::<DLSiteManiax, Report>(info)
             })
         }
-        let mut maniax_infos = Vec::new();
+        Ok(tasks)
-        while let Some(result) = tasks.next().await {
-            maniax_infos.push(result?);
-        }
-
-        Ok(maniax_infos)
     }
 
     fn verify_all_works_exists(maniax_result: &HashMap<String, DLSiteManiax>, rj_nums: Vec<String>) {
@@ -123,8 +130,9 @@ impl DLSiteCrawler {
         );
         let result = html.select(&selector).next().unwrap();
         let genre_rows = result.child_elements().collect::<Vec<_>>();
-        let genre_len = genre_rows.iter().count();
+        let genre_rows_len = genre_rows.iter().count();
-        let genre_row = genre_rows.iter().skip(genre_len - 2).next().unwrap();
+        // get second last row for genre
+        let genre_row = genre_rows.iter().skip(genre_rows_len - 2).next().unwrap();
         let data = genre_row
             .child_elements().skip(1).next().unwrap()
             .child_elements().next().unwrap();

src/helpers/db.rs

@@ -1,9 +1,11 @@
-use crate::constants::{APP_DB_DATA_DIR, DB_COLUMNS};
+use std::path::Path;
-use rocksdb::{ColumnFamilyDescriptor, IteratorMode, OptimisticTransactionDB, Options, ReadOptions};
+use crate::constants::{DB_COLUMNS};
+use rocksdb::{ColumnFamilyDescriptor, IteratorMode, OptimisticTransactionDB, Options};
 use serde::{Serialize};
 use serde::de::DeserializeOwned;
 use crate::models::{RocksColumn, RocksReference, RocksReferences};
 use color_eyre::Result;
+use crate::config::types::ApplicationConfig;
 
 pub struct RocksDB {
     db: OptimisticTransactionDB,
@@ -11,12 +13,13 @@ pub struct RocksDB {
 
 impl RocksDB {
     pub fn new(db_opts: Options, cf_opts: Options) -> Result<Self> {
+        let app_conf = ApplicationConfig::get_config()?;
         let cfs = DB_COLUMNS.iter()
             .map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), cf_opts.clone()))
             .collect::<Vec<_>>();
         let db = OptimisticTransactionDB::open_cf_descriptors(
             &db_opts,
-            APP_DB_DATA_DIR.as_path(),
+            Path::new(&app_conf.basic_config.db_path),
             cfs
         )?;
         let rocks = Self {

src/helpers/mod.rs

@@ -1,14 +1,16 @@
 pub mod db;
 
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use color_eyre::eyre::eyre;
 use color_eyre::owo_colors::OwoColorize;
 use tokio::fs;
-use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR, APP_DB_DATA_DIR};
+use crate::config::types::ApplicationConfig;
+use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR};
 use crate::crawler::DLSITE_IMG_FOLDER;
 
 
 pub async fn initialize_folders() -> color_eyre::Result<()> {
+    let app_conf = ApplicationConfig::get_config()?;
     if !APP_CONFIG_DIR.exists() {
         fs::create_dir_all(APP_CONFIG_DIR.as_path()).await?;
     }
@@ -18,16 +20,17 @@ pub async fn initialize_folders() -> color_eyre::Result<()> {
     if !DLSITE_IMG_FOLDER.exists() {
         fs::create_dir_all(DLSITE_IMG_FOLDER.as_path()).await?;
     }
-    if !APP_DB_DATA_DIR.exists() {
+    let db_path = Path::new(&app_conf.basic_config.db_path);
-        fs::create_dir_all(APP_DB_DATA_DIR.as_path()).await?;
+    if !db_path.exists() {
+        fs::create_dir_all(db_path).await?;
     }
     Ok(())
 }
 
-pub async fn get_all_folders(paths: &Vec<PathBuf>) -> color_eyre::Result<Vec<PathBuf>> {
+pub async fn get_all_folders(paths: Vec<&Path>) -> color_eyre::Result<Vec<PathBuf>> {
     let mut folders: Vec<PathBuf> = Vec::new();
     for path in paths {
-        let path = path.as_path();
+        let path = path.to_path_buf();
         if !path.exists() {
             return Err(eyre!("{:?} {}", path.blue(), "does not exist".red()));
         }

src/lib.rs

@@ -0,0 +1,4 @@
+extern crate jemallocator;
+
+#[global_allocator]
+static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;

src/models/game.rs

@@ -1,19 +1,19 @@
-use ratatui::widgets::ListState;
+use std::path::PathBuf;
+use color_eyre::{eyre, Report};
+use language_tags::LanguageTag;
 use serde::{Deserialize, Serialize};
+use crate::config::types::ApplicationConfig;
 use crate::models::{RocksColumn, RocksReference, RocksReferences};
 
-pub(crate) struct GameList<T> {
-    games: Vec<T>,
-    state: ListState,
-}
-
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub(crate) struct DLSiteManiax {
     #[serde(skip)]
     pub rj_num: String,
     pub genre_ids: Vec<u16>,
-    pub name: String,
+    pub name: Vec<DLSiteTranslation>,
-    pub sells_count: u32
+    pub sells_count: u32,
+    pub folder_path: PathBuf,
+    pub version: Option<String>
 }
 
 impl From<crate::crawler::dlsite::DLSiteManiax> for DLSiteManiax {
@@ -21,8 +21,10 @@ impl From<crate::crawler::dlsite::DLSiteManiax> for DLSiteManiax {
         Self {
             rj_num: value.rj_num,
             genre_ids: value.genre_ids,
-            name: value.title,
+            name: vec![],
-            sells_count: value.sells_count
+            sells_count: value.sells_count,
+            folder_path: value.folder_path,
+            version: None
         }
     }
 }
@@ -105,3 +107,21 @@ impl RocksColumn for DLSiteCategory {
 pub(crate) enum DLSiteTranslation {
     EN(String), JP(String)
 }
+
+impl TryFrom<&str> for DLSiteTranslation {
+    type Error = Report;
+    fn try_from(value: &str) -> color_eyre::Result<Self> {
+        let app_conf = ApplicationConfig::get_config()?;
+        let locale = app_conf.basic_config.locale;
+
+        let en_locale = LanguageTag::parse("en-US")?;
+        if locale.matches(&en_locale) {
+            return Ok(DLSiteTranslation::EN(value.to_string()));
+        }
+        let jp_locale = LanguageTag::parse("ja-JP")?;
+        if locale.matches(&jp_locale) {
+            return Ok(DLSiteTranslation::JP(value.to_string()));
+        }
+        Err(eyre::eyre!("Invalid Locale: {:?}; Support {:?}", locale, [en_locale, jp_locale]))
+    }
+}