From 2538049f4a560c2ce558c2c67aba1e84e8a202c2 Mon Sep 17 00:00:00 2001 From: fromost Date: Thu, 30 Oct 2025 17:34:39 +0800 Subject: [PATCH] Add syncing for genres and works --- .config/config.toml | 0 .gitignore | 0 Cargo.toml | 8 ++ LICENSE | 0 README.md | 0 src/app.rs | 0 src/cli/folder.rs | 0 src/cli/mod.rs | 0 src/cli/sync.rs | 110 +++++++++++++++++++++++----- src/config/mod.rs | 35 +++++---- src/config/types.rs | 0 src/constants.rs | 23 +++++- src/crawler/dlsite.rs | 114 +++++++++++++++++++++++------ src/crawler/mod.rs | 10 ++- src/event.rs | 0 src/helpers/db.rs | 7 +- src/helpers/mod.rs | 9 ++- src/lib.rs | 0 src/main.rs | 0 src/models/game.rs | 101 ++++++++++++++++++------- src/models/mod.rs | 28 ++++++- src/widgets/components/mod.rs | 0 src/widgets/components/textarea.rs | 0 src/widgets/mod.rs | 0 src/widgets/popups/folder.rs | 0 src/widgets/popups/mod.rs | 0 src/widgets/views/main_view.rs | 0 src/widgets/views/mod.rs | 0 28 files changed, 350 insertions(+), 95 deletions(-) mode change 100644 => 100755 .config/config.toml mode change 100644 => 100755 .gitignore mode change 100644 => 100755 Cargo.toml mode change 100644 => 100755 LICENSE mode change 100644 => 100755 README.md mode change 100644 => 100755 src/app.rs mode change 100644 => 100755 src/cli/folder.rs mode change 100644 => 100755 src/cli/mod.rs mode change 100644 => 100755 src/cli/sync.rs mode change 100644 => 100755 src/config/mod.rs mode change 100644 => 100755 src/config/types.rs mode change 100644 => 100755 src/constants.rs mode change 100644 => 100755 src/crawler/dlsite.rs mode change 100644 => 100755 src/crawler/mod.rs mode change 100644 => 100755 src/event.rs mode change 100644 => 100755 src/helpers/db.rs mode change 100644 => 100755 src/helpers/mod.rs mode change 100644 => 100755 src/lib.rs mode change 100644 => 100755 src/main.rs mode change 100644 => 100755 src/models/game.rs mode change 100644 => 100755 src/models/mod.rs mode change 100644 => 100755 src/widgets/components/mod.rs mode change 100644 => 100755 src/widgets/components/textarea.rs mode change 100644 => 100755 src/widgets/mod.rs mode change 100644 => 100755 src/widgets/popups/folder.rs mode change 100644 => 100755 src/widgets/popups/mod.rs mode change 100644 => 100755 src/widgets/views/main_view.rs mode change 100644 => 100755 src/widgets/views/mod.rs diff --git a/.config/config.toml b/.config/config.toml old mode 100644 new mode 100755 diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/Cargo.toml b/Cargo.toml old mode 100644 new mode 100755 index ef68a05..eb2c50c --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,12 @@ edition = "2024" [profile.dev] debug = true incremental = true +lto = "off" +opt-level = 0 + +[profile.release] lto = "fat" +opt-level = 3 [dependencies] color-eyre = "0.6.3" @@ -25,6 +30,9 @@ log = "0.4.28" num_cpus = "1.17.0" sys-locale = "0.3.2" jemallocator = "0.5.4" +reqwest_cookie_store = { version = "0.9.0", features = ["serde"] } +itertools = "0.14.0" +dashmap = { version = "6.1.0", features = ["serde"] } [dependencies.language-tags] version = "0.3.2" diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/src/app.rs b/src/app.rs old mode 100644 new mode 100755 diff --git a/src/cli/folder.rs b/src/cli/folder.rs old mode 100644 new mode 100755 diff --git a/src/cli/mod.rs b/src/cli/mod.rs old mode 100644 new mode 100755 diff --git a/src/cli/sync.rs b/src/cli/sync.rs old mode 100644 new mode 100755 index 167eebf..115791c --- a/src/cli/sync.rs +++ b/src/cli/sync.rs @@ -1,12 +1,14 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; -use clap::{Args, Command, Parser, Subcommand}; -use color_eyre::eyre::Result; +use clap::{Parser}; +use color_eyre::eyre::{Result}; use crossterm::style::{style, Stylize}; use futures::StreamExt; use indicatif::{ProgressBar, ProgressStyle}; +use itertools::Itertools; use tokio::time::Instant; use crate::models; +use crate::models::{DLSiteCategory, DLSiteGenre, DLSiteManiax, DLSiteTranslation}; use crate::config::types::ApplicationConfig; use crate::constants::{DB_CF_OPTIONS, DB_OPTIONS}; use crate::crawler::{dlsite, DLSiteCrawler}; @@ -29,8 +31,10 @@ pub(super) enum DLSiteSubCommand { pub(super) struct DLSiteSyncCommand { #[clap(long, short, action)] missing: bool, - #[clap(long, short, action)] - genre: bool, + #[clap(long = "genre", default_value = "false")] + do_sync_genre: bool, + #[clap(long = "work", default_value = "true")] + do_sync_work: bool } impl DLSiteSubCommand { @@ -46,42 +50,108 @@ impl DLSiteSyncCommand { let now = Instant::now(); let app_conf = ApplicationConfig::get_config()?; let mut db = RocksDB::new(DB_OPTIONS.clone(), DB_CF_OPTIONS.clone())?; - if self.genre { - Self::sync_genres(&app_conf).await?; + let crawler = DLSiteCrawler::new()?; + if self.do_sync_genre { + let genre_now = Instant::now(); + Self::sync_genres(&mut db, &app_conf, &crawler).await?; + println!( + "{} {} Done in {:.2?}", + style("Genres").cyan(), + style("Syncing").green(), + genre_now.elapsed() + ); + } + if self.do_sync_work { + let work_now = Instant::now(); + self.sync_works(&app_conf, &mut db, &crawler).await?; + println!( + "{} {} Done in {:.2?}", + style("Works").cyan(), + style("Syncing").green(), + work_now.elapsed() + ); } - self.sync_works(&app_conf, &mut db).await?; println!("{} Done in {:.2?}", style("Syncing").green(), now.elapsed()); Ok(()) } - async fn sync_genres(app_conf: &ApplicationConfig) -> Result<()> { + async fn sync_genres(db: &mut RocksDB, app_conf: &ApplicationConfig, crawler: &DLSiteCrawler) -> Result<()> { + let requested_categories = crawler.get_all_genres(&app_conf.basic_config.locale).await?; + let categories: Vec = requested_categories.iter() + .map(|g| g.clone().try_into()) + .filter_map(Result::ok) + .collect(); + db.set_values(&categories)?; + let genres = requested_categories.into_iter() + .flat_map(|v| v.values) + .collect_vec(); + let existing_genres = db.get_all_values::()?; + let mut modified_genres: Vec = Vec::new(); + for genre in genres { + let id = genre.value.parse::()?; + let existing_genre = + existing_genres.iter().find(|v| v.id == id); + if let Some(existing_genre) = existing_genre { + let name = DLSiteTranslation::try_from(genre.name)?; + if existing_genre.name.contains(&name) { + modified_genres.push(existing_genre.clone()); + continue; + } + let mut modified_genre = existing_genre.clone(); + modified_genre.name.push(name); + modified_genres.push(modified_genre); + } + else { + modified_genres.push(DLSiteGenre { + id, name: vec![DLSiteTranslation::try_from(genre.name)?] + }); + } + } + db.set_values(&modified_genres)?; + Ok(()) } - async fn sync_works(&self, app_conf: &ApplicationConfig, db: &mut RocksDB) -> Result<()> { - let crawler = DLSiteCrawler::new(); + async fn sync_works(&self, app_conf: &ApplicationConfig, db: &mut RocksDB, crawler: &DLSiteCrawler) -> Result<()> { let existing_works = db.get_all_values::()?; - let work_list = self.get_work_list(&app_conf, existing_works).await?; + let work_list = self.get_work_list(&app_conf, &existing_works).await?; let rj_nums = work_list.clone().into_keys().collect::>(); - let mut maniaxes: Vec = Vec::new(); - let mut game_infos = crawler.get_game_infos(rj_nums).await?; + + let mut game_infos = crawler.get_game_infos(rj_nums, &app_conf.basic_config.locale).await?; + let existing_game_infos = db.get_all_values::()?; + let mut modified_maniaxes: Vec = Vec::new(); let progress = ProgressBar::new(game_infos.len() as u64) .with_style(ProgressStyle::default_bar()); while let Some(info) = game_infos.next().await { - let mut value: models::DLSiteManiax = info?.into(); - let maniax_folder = work_list.get(&value.rj_num).unwrap().to_owned(); - value.folder_path = maniax_folder; - maniaxes.push(value); + let maniax = info?; + let existing_maniax = existing_game_infos.iter() + .find(|v| v.rj_num == maniax.rj_num); + if let Some(existing_maniax) = existing_maniax { + let name = DLSiteTranslation::try_from(maniax.title)?; + if existing_maniax.name.contains(&name) { + modified_maniaxes.push(existing_maniax.clone()); + continue; + } + let mut modified_maniax = existing_maniax.clone(); + modified_maniax.name.push(name); + modified_maniaxes.push(modified_maniax); + } + else { + let mut value: DLSiteManiax = maniax.into(); + let maniax_folder = work_list.get(&value.rj_num).unwrap().to_owned(); + value.folder_path = maniax_folder; + modified_maniaxes.push(value); + } progress.inc(1); } - db.set_values(&maniaxes)?; + db.set_values(&modified_maniaxes)?; Ok(()) } - async fn get_work_list(&self, app_conf: &ApplicationConfig, existing_works: Vec) -> Result> { + async fn get_work_list(&self, app_conf: &ApplicationConfig, existing_works: &[models::DLSiteManiax]) -> Result> { let existing_nums = existing_works.iter() .map(|x| x.rj_num.clone()) .collect::>(); @@ -92,7 +162,7 @@ impl DLSiteSyncCommand { let config_paths = app_conf.path_config.dlsite_paths.iter() .map(|path| Path::new(path)) .collect::>(); - let dir_paths = helpers::get_all_folders(config_paths).await?; + let dir_paths = helpers::get_all_folders(&config_paths).await?; for dir_path in dir_paths { if !dir_path.is_dir() { println!( diff --git a/src/config/mod.rs b/src/config/mod.rs old mode 100644 new mode 100755 index ef05e53..dd2f4bd --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1,5 +1,5 @@ use crate::config::types::{ApplicationConfig, BasicConfig, PathConfig}; -use crate::constants::{APP_CONIFG_FILE_PATH, APP_DB_DATA_DIR}; +use crate::constants::{APP_CONIFG_FILE_PATH, APP_DB_DATA_DIR, CACHE_MAP}; use color_eyre::Result; use std::path::PathBuf; use language_tags::LanguageTag; @@ -8,6 +8,8 @@ use serde_json; pub mod types; +const CONFIG_KEY: &str = "app_conf"; + pub(crate) struct GameList { games: Vec, state: ListState, @@ -15,45 +17,46 @@ pub(crate) struct GameList { impl ApplicationConfig { pub fn get_config() -> Result { - if APP_CONIFG_FILE_PATH.exists() { + if CACHE_MAP.contains_key(CONFIG_KEY) { + Ok(serde_json::from_value(CACHE_MAP.get(CONFIG_KEY).unwrap().clone())?) + } else if APP_CONIFG_FILE_PATH.exists() { ApplicationConfig::from_file(&APP_CONIFG_FILE_PATH) - } else { - Ok(ApplicationConfig::new()) + } else { + ApplicationConfig::new() } } fn from_file(path: &PathBuf) -> Result { let reader = std::fs::File::open(path)?; - let result = serde_json::from_reader(reader)?; - Ok(result) + let result: serde_json::Value = serde_json::from_reader(reader)?; + CACHE_MAP.insert(CONFIG_KEY.to_string(), result.clone()); + Ok(serde_json::from_value(result)?) } - fn new() -> Self { + fn new() -> Result { let default_locale = sys_locale::get_locale().unwrap_or(String::from("ja-JP")); let conf = Self { basic_config: BasicConfig { db_path: APP_DB_DATA_DIR.to_str().unwrap().to_string(), tick_rate: 250, - locale: LanguageTag::parse(&default_locale).unwrap(), + locale: LanguageTag::parse(&default_locale)?, }, path_config: PathConfig { dlsite_paths: vec![], }, }; - conf.clone() - .write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf()) - .unwrap(); - conf + conf.clone().write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf())?; + Ok(conf) } fn write_to_file(self, path: &PathBuf) -> Result<()> { let writer = std::fs::File::create(path)?; - let result = serde_json::to_writer_pretty(writer, &self)?; - Ok(result) + serde_json::to_writer_pretty(writer, &self)?; + Ok(()) } - pub fn save(&self) -> Result<()> { - self.clone().write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf()) + pub fn save(self) -> Result<()> { + self.write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf()) } } diff --git a/src/config/types.rs b/src/config/types.rs old mode 100644 new mode 100755 diff --git a/src/constants.rs b/src/constants.rs old mode 100644 new mode 100755 index 9b5e8bd..0fd4fec --- a/src/constants.rs +++ b/src/constants.rs @@ -1,7 +1,11 @@ +use std::hash::RandomState; use directories::BaseDirs; use lazy_static::lazy_static; use std::path::PathBuf; -use crate::models::{DLSiteManiax, RocksColumn}; +use std::sync::Arc; +use dashmap::DashMap; +use language_tags::LanguageTag; +use crate::models::{DLSiteCategory, DLSiteGenre, DLSiteManiax, RocksColumn}; const APP_DIR_NAME: &str = "sus_manager"; lazy_static! { @@ -18,7 +22,22 @@ lazy_static! { } lazy_static! { - pub static ref DB_COLUMNS: Vec = vec![DLSiteManiax::get_column_name().to_string()]; + pub static ref DB_COLUMNS: [String; 3] = [ + DLSiteManiax::get_column_name().to_string(), + DLSiteGenre::get_column_name().to_string(), + DLSiteCategory::get_column_name().to_string(), + ]; +} + +lazy_static! { + pub static ref EN_LOCALE: LanguageTag = LanguageTag::parse("en").unwrap(); + pub static ref JP_LOCALE: LanguageTag = LanguageTag::parse("ja").unwrap(); + pub static ref SUPPORTED_LOCALES: [LanguageTag; 2] = [JP_LOCALE.clone(), EN_LOCALE.clone()]; +} + +lazy_static! { + pub static ref CACHE_MAP: Arc> = + Arc::new(DashMap::with_hasher(RandomState::default())); } fn get_db_options() -> rocksdb::Options { diff --git a/src/crawler/dlsite.rs b/src/crawler/dlsite.rs old mode 100644 new mode 100755 index 6e60a1f..08fe389 --- a/src/crawler/dlsite.rs +++ b/src/crawler/dlsite.rs @@ -1,29 +1,36 @@ use std::collections::{HashMap, HashSet}; -use std::path::PathBuf; +use std::path::{PathBuf}; use color_eyre::eyre::eyre; use color_eyre::owo_colors::OwoColorize; -use reqwest::{StatusCode, Url}; +use reqwest::{Url}; use color_eyre::{Report, Result}; use futures::stream::FuturesUnordered; +use itertools::Itertools; +use language_tags::LanguageTag; use lazy_static::lazy_static; -use scraper::{Html, Selector}; +use scraper::{Element, Html, Selector}; use serde::{Deserialize, Serialize}; -use crate::constants::{APP_DATA_DIR}; +use serde_json::Value; +use crate::config::types::ApplicationConfig; +use crate::constants::{APP_DATA_DIR, JP_LOCALE}; use crate::crawler::Crawler; +use crate::helpers::matches_primary_language; +use crate::models; +use crate::models::{DLSiteTranslation, PrimaryLanguage}; //TODO: override locale with user one const DLSITE_URL: &str = "https://www.dlsite.com/"; const DLSITE_PRODUCT_API_ENDPOINT: &str = "/maniax/product/info/ajax"; -const DLSITE_FILTER_OPTIONS_ENDPOINT: &str = "/maniax/fs/=/api_access/1/locale/ja_JP"; +const DLSITE_FS_ENDPOINT: &str = "/maniax/fs/=/api_access/1/"; const DLSITE_MANIAX_PATH: &str = "/maniax/work/=/product_id/"; lazy_static! { pub static ref DLSITE_IMG_FOLDER: PathBuf = APP_DATA_DIR.clone().join("dlsite").join("img"); } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct DLSiteCrawler { - crawler: Crawler, + crawler: Crawler } #[derive(Deserialize, Serialize, Debug, Clone)] @@ -43,18 +50,38 @@ pub(crate) struct DLSiteManiax { } #[derive(Deserialize, Serialize, Debug, Clone)] -pub(crate) struct DLSiteGenreCategory { +struct DLSiteFilter { + pub(crate) genre_all: Value +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub(crate) struct DLSiteGenreCategory { + pub(crate) category_name: String, + pub(crate) values: Vec, + #[serde(skip)] + pub(crate) id: u8 +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub(crate) struct DLSiteGenre { + pub(crate) value: String, + pub(crate) name: String } impl DLSiteCrawler { - pub fn new() -> Self { - Self { - crawler: Crawler::new("DLSite", Url::parse(DLSITE_URL).unwrap()) - } + pub fn new() -> Result { + let url = Url::parse(DLSITE_URL)?; + let crawler = Self { + crawler: Crawler::new( + "DLSite", + url + ) + }; + Ok(crawler) } - pub async fn get_game_infos(&self, rj_nums: Vec) -> Result>>> + pub async fn get_game_infos(&self, rj_nums: Vec, locale: &LanguageTag) -> Result>>> { let invalid_nums = rj_nums.iter() .filter(|&n| !is_valid_rj_number(n)) @@ -65,10 +92,14 @@ impl DLSiteCrawler { eyre!("Invalid numbers: {}", invalid_nums.join(", ")) ); } - - let query = &format!("product_id={}", rj_nums.join(",")); + let primary_language: PrimaryLanguage = locale.try_into()?; + let locale_str = match primary_language { + PrimaryLanguage::EN => "en_US", + PrimaryLanguage::JP => "ja_JP", + }; + let query = &format!("product_id={}&locale={}", rj_nums.join(","), locale_str); let (value, _) = self.crawler - .get_json::(DLSITE_PRODUCT_API_ENDPOINT, Some(query)) + .get_json::(DLSITE_PRODUCT_API_ENDPOINT, Some(query)) .await?; // try to catch '[]' empty result from the api let value_downcast_result: Result, _> = serde_json::from_value(value); @@ -78,14 +109,15 @@ impl DLSiteCrawler { let tasks = FuturesUnordered::new(); for (rj_num, mut info) in maniax_result { - tasks.push(async { + tasks.push(async move { let html_path = format!("{DLSITE_MANIAX_PATH}{rj_num}"); + let query = format!("locale={locale_str}"); let (_, html_result) = tokio::join!( self.save_main_image(&info, &rj_num), - self.crawler.get_html(&html_path) + self.crawler.get_html(&html_path, Some(&query)) ); let (html, _) = html_result?; - let genres = self.get_genres(&html).await?; + let genres = self.get_work_genres(&html, locale.try_into()?).await?; info.genre_ids = genres; info.rj_num = rj_num; Ok::(info) @@ -122,17 +154,23 @@ impl DLSiteCrawler { Ok(()) } - async fn get_genres(&self, html: &Html) -> Result> { + async fn get_work_genres(&self, html: &Html, primary_language: PrimaryLanguage) -> Result> { let selector = Result::unwrap( Selector::parse( "#work_outline > tbody:nth-child(1)" ) ); + let genre_str = match primary_language { + PrimaryLanguage::EN => "Genre", + PrimaryLanguage::JP => "ジャンル" + }; + let result = html.select(&selector).next().unwrap(); let genre_rows = result.child_elements().collect::>(); - let genre_rows_len = genre_rows.iter().count(); - // get second last row for genre - let genre_row = genre_rows.iter().skip(genre_rows_len - 2).next().unwrap(); + let t = genre_rows.iter().filter_map(|v| v.first_element_child().unwrap().text().next()).collect_vec(); + let genre_row = genre_rows.iter() + .find(|v| v.first_element_child().unwrap().text().next().unwrap() == genre_str) + .unwrap(); let data = genre_row .child_elements().skip(1).next().unwrap() .child_elements().next().unwrap(); @@ -151,6 +189,36 @@ impl DLSiteCrawler { } } +impl DLSiteCrawler { + pub async fn get_all_genres(&self, locale: &LanguageTag) -> Result> { + let primary_language: PrimaryLanguage = locale.try_into()?; + let locale_str = match primary_language { + PrimaryLanguage::EN => "en_US", + PrimaryLanguage::JP => "ja_JP", + }; + let query = format!("locale={}", locale_str); + + let (json, _) = self.crawler.get_json::(DLSITE_FS_ENDPOINT, Some(&query)).await?; + let values = + if matches_primary_language(&locale, &JP_LOCALE) { + serde_json::from_value::>(json.genre_all)? + } else { + // IDK why they are using different object type bruh + serde_json::from_value::>(json.genre_all)? + .into_iter().map(|(_, v)| v) + .collect_vec() + }; + + let mut categories = Vec::new(); + for (i, value) in values.into_iter().enumerate() { + let mut category = value.clone(); + category.id = i as u8; + categories.push(category); + } + Ok(categories) + } +} + pub fn is_valid_rj_number(rj_num: &str) -> bool { let len = rj_num.len(); if len != 8 && len != 10 { diff --git a/src/crawler/mod.rs b/src/crawler/mod.rs old mode 100644 new mode 100755 index 9e3465e..49b1b64 --- a/src/crawler/mod.rs +++ b/src/crawler/mod.rs @@ -1,5 +1,6 @@ pub mod dlsite; +use std::sync::Arc; pub use dlsite::*; use color_eyre::eyre::eyre; use crate::constants::APP_CACHE_PATH; @@ -10,12 +11,12 @@ use robotstxt::DefaultMatcher; use scraper::Html; use serde::de::DeserializeOwned; -#[derive(Clone)] +#[derive(Clone, Debug)] struct Crawler { id: String, pub(crate) base_url: Url, client: Client, - robots_txt: Option, + robots_txt: Option } impl Crawler { @@ -67,10 +68,11 @@ impl Crawler { } } - pub async fn get_html(&self, path: &str) -> Result<(Html, StatusCode)> { + pub async fn get_html(&self, path: &str, query: Option<&str>) -> Result<(Html, StatusCode)> { let mut url = self.base_url.clone(); self.check_access(&url).await?; url.set_path(path); + url.set_query(query); let res = self.client.get(url).send().await?; let status = res.status(); let html_text = &res.text().await?; @@ -106,5 +108,5 @@ impl Crawler { let status = res.status(); let bytes = res.bytes().await?; Ok((bytes.to_vec(), status)) - } + } } diff --git a/src/event.rs b/src/event.rs old mode 100644 new mode 100755 diff --git a/src/helpers/db.rs b/src/helpers/db.rs old mode 100644 new mode 100755 index f067c2f..8282645 --- a/src/helpers/db.rs +++ b/src/helpers/db.rs @@ -36,7 +36,9 @@ impl RocksDB { if query_res.is_none() { return Ok(None); } - Ok(Some(serde_json::from_slice(&query_res.unwrap())?)) + let mut value: TColumn = serde_json::from_slice(&query_res.unwrap())?; + value.set_id(id.clone()); + Ok(Some(value)) } pub fn set_value(&self, value: &TColumn) -> Result<()> @@ -56,7 +58,8 @@ impl RocksDB { for id in ids { let query_res = transaction.get_cf(&cf, serde_json::to_string(id)?)?; if let Some(res) = query_res { - let value = serde_json::from_slice(&res)?; + let mut value: TColumn = serde_json::from_slice(&res)?; + value.set_id(id.clone()); values.push(value); } } diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs old mode 100644 new mode 100755 index dc3f760..907132b --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -3,9 +3,10 @@ pub mod db; use std::path::{Path, PathBuf}; use color_eyre::eyre::eyre; use color_eyre::owo_colors::OwoColorize; +use language_tags::LanguageTag; use tokio::fs; use crate::config::types::ApplicationConfig; -use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR}; +use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR, SUPPORTED_LOCALES}; use crate::crawler::DLSITE_IMG_FOLDER; @@ -27,7 +28,7 @@ pub async fn initialize_folders() -> color_eyre::Result<()> { Ok(()) } -pub async fn get_all_folders(paths: Vec<&Path>) -> color_eyre::Result> { +pub async fn get_all_folders(paths: &[&Path]) -> color_eyre::Result> { let mut folders: Vec = Vec::new(); for path in paths { let path = path.to_path_buf(); @@ -41,4 +42,8 @@ pub async fn get_all_folders(paths: Vec<&Path>) -> color_eyre::Result bool { + left.primary_language() == right.primary_language() } \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs old mode 100644 new mode 100755 diff --git a/src/main.rs b/src/main.rs old mode 100644 new mode 100755 diff --git a/src/models/game.rs b/src/models/game.rs old mode 100644 new mode 100755 index dd0d738..9ecdd4a --- a/src/models/game.rs +++ b/src/models/game.rs @@ -1,10 +1,13 @@ use std::path::PathBuf; use color_eyre::{eyre, Report}; -use language_tags::LanguageTag; use serde::{Deserialize, Serialize}; use crate::config::types::ApplicationConfig; -use crate::models::{RocksColumn, RocksReference, RocksReferences}; +use crate::constants::{EN_LOCALE, JP_LOCALE}; +use crate::crawler::DLSiteGenreCategory; +use crate::helpers::matches_primary_language; +use crate::models::{RocksColumn, RocksReferences}; +//region Maniax #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct DLSiteManiax { #[serde(skip)] @@ -18,10 +21,11 @@ pub(crate) struct DLSiteManiax { impl From for DLSiteManiax { fn from(value: crate::crawler::DLSiteManiax) -> Self { + let title = DLSiteTranslation::try_from(value.title.as_str()).unwrap(); Self { rj_num: value.rj_num, genre_ids: value.genre_ids, - name: vec![], + name: vec![title], sells_count: value.sells_count, folder_path: value.folder_path, version: None @@ -50,13 +54,14 @@ impl RocksReferences for DLSiteManiax { self.genre_ids.clone() } } +//endregion +//region Genre #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct DLSiteGenre { +pub struct DLSiteGenre { #[serde(skip)] pub id: u16, - pub category_id: u16, - pub translations: Vec + pub name: Vec } impl RocksColumn for DLSiteGenre { @@ -74,22 +79,45 @@ impl RocksColumn for DLSiteGenre { String::from("dl_genres") } } +//endregion -impl RocksReference for DLSiteGenre { - fn get_reference_id(&self) -> ::Id { - self.category_id.clone() - } -} - +//region Category #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct DLSiteCategory { #[serde(skip)] - pub id: u16, - pub translations: Vec + pub id: String, + pub genre_ids: Vec, + pub name: DLSiteTranslation +} + +impl TryFrom for DLSiteCategory { + type Error = Report; + + fn try_from(value: DLSiteGenreCategory) -> Result { + let category = Self { + id: format!( + "{}/{}", + value.id, + ApplicationConfig::get_config()?.basic_config.locale.primary_language() + ), + genre_ids: value.values.iter() + .map(|v| v.value.parse::()) + .filter_map(Result::ok) + .collect(), + name: DLSiteTranslation::try_from(value.category_name.as_str())?, + }; + Ok(category) + } +} + +impl RocksReferences for DLSiteCategory { + fn get_reference_ids(&self) -> Vec<::Id> { + self.genre_ids.clone() + } } impl RocksColumn for DLSiteCategory { - type Id = u16; + type Id = String; fn get_id(&self) -> Self::Id { self.id.clone() } @@ -99,11 +127,13 @@ impl RocksColumn for DLSiteCategory { } fn get_column_name() -> String { - String::from("dl_translations") + String::from("dl_categories") } } +//endregion -#[derive(Clone, Debug, Serialize, Deserialize)] +//region Translation +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub(crate) enum DLSiteTranslation { EN(String), JP(String) } @@ -111,17 +141,38 @@ pub(crate) enum DLSiteTranslation { impl TryFrom<&str> for DLSiteTranslation { type Error = Report; fn try_from(value: &str) -> color_eyre::Result { + Self::try_from(value.to_string()) + } +} + +impl TryFrom for DLSiteTranslation { + type Error = Report; + fn try_from(value: String) -> color_eyre::Result { let app_conf = ApplicationConfig::get_config()?; let locale = app_conf.basic_config.locale; - let en_locale = LanguageTag::parse("en-US")?; - if locale.matches(&en_locale) { - return Ok(DLSiteTranslation::EN(value.to_string())); + if matches_primary_language(&locale, &EN_LOCALE) { + return Ok(DLSiteTranslation::EN(value)); } - let jp_locale = LanguageTag::parse("ja-JP")?; - if locale.matches(&jp_locale) { - return Ok(DLSiteTranslation::JP(value.to_string())); + if matches_primary_language(&locale, &JP_LOCALE) { + return Ok(DLSiteTranslation::JP(value)); } - Err(eyre::eyre!("Invalid Locale: {:?}; Support {:?}", locale, [en_locale, jp_locale])) + Err(eyre::eyre!( + "Invalid Locale: {:?}; Support {:?}", + locale, + [EN_LOCALE.to_string(), JP_LOCALE.to_string()]) + ) } -} \ No newline at end of file +} + +impl TryInto for DLSiteTranslation { + type Error = Report; + + fn try_into(self) -> Result { + match self { + DLSiteTranslation::EN(val) => Ok(val), + DLSiteTranslation::JP(val) => Ok(val), + } + } +} +//endregion \ No newline at end of file diff --git a/src/models/mod.rs b/src/models/mod.rs old mode 100644 new mode 100755 index 3480e02..a04f582 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -1,11 +1,16 @@ mod game; +use color_eyre::eyre::eyre; +use color_eyre::Report; +use language_tags::LanguageTag; use serde::de::DeserializeOwned; use serde::Serialize; pub(crate) use game::*; +use crate::constants::{EN_LOCALE, JP_LOCALE}; +use crate::helpers::matches_primary_language; pub trait RocksColumn { - type Id: Serialize + DeserializeOwned; + type Id: Serialize + DeserializeOwned + Clone; fn get_id(&self) -> Self::Id; fn set_id(&mut self, id: Self::Id); fn get_column_name() -> String; @@ -17,4 +22,25 @@ pub trait RocksReference where T: RocksColumn { pub trait RocksReferences where T: RocksColumn { fn get_reference_ids(&self) -> Vec; +} + +#[derive(Debug, Clone)] +pub enum PrimaryLanguage { + EN, JP +} + +impl TryFrom<&LanguageTag> for PrimaryLanguage { + type Error = Report; + + fn try_from(value: &LanguageTag) -> Result { + if matches_primary_language(&value, &EN_LOCALE) { + Ok(PrimaryLanguage::EN) + } + else if matches_primary_language(&value, &JP_LOCALE) { + Ok(PrimaryLanguage::JP) + } + else { + Err(eyre!("No matching primary language found for {}", value)) + } + } } \ No newline at end of file diff --git a/src/widgets/components/mod.rs b/src/widgets/components/mod.rs old mode 100644 new mode 100755 diff --git a/src/widgets/components/textarea.rs b/src/widgets/components/textarea.rs old mode 100644 new mode 100755 diff --git a/src/widgets/mod.rs b/src/widgets/mod.rs old mode 100644 new mode 100755 diff --git a/src/widgets/popups/folder.rs b/src/widgets/popups/folder.rs old mode 100644 new mode 100755 diff --git a/src/widgets/popups/mod.rs b/src/widgets/popups/mod.rs old mode 100644 new mode 100755 diff --git a/src/widgets/views/main_view.rs b/src/widgets/views/main_view.rs old mode 100644 new mode 100755 diff --git a/src/widgets/views/mod.rs b/src/widgets/views/mod.rs old mode 100644 new mode 100755