Add syncing for genres and works

Cargo.toml

@@ -9,7 +9,12 @@ edition = "2024"
 [profile.dev]
 debug = true
 incremental = true
+lto = "off"
+opt-level = 0
+
+[profile.release]
 lto = "fat"
+opt-level = 3
 
 [dependencies]
 color-eyre = "0.6.3"
@@ -25,6 +30,9 @@ log = "0.4.28"
 num_cpus = "1.17.0"
 sys-locale = "0.3.2"
 jemallocator = "0.5.4"
+reqwest_cookie_store = { version = "0.9.0", features = ["serde"] }
+itertools = "0.14.0"
+dashmap = { version = "6.1.0", features = ["serde"] }
 
 [dependencies.language-tags]
 version = "0.3.2"

src/cli/sync.rs

@@ -1,12 +1,14 @@
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
-use clap::{Args, Command, Parser, Subcommand};
+use clap::{Parser};
-use color_eyre::eyre::Result;
+use color_eyre::eyre::{Result};
 use crossterm::style::{style, Stylize};
 use futures::StreamExt;
 use indicatif::{ProgressBar, ProgressStyle};
+use itertools::Itertools;
 use tokio::time::Instant;
 use crate::models;
+use crate::models::{DLSiteCategory, DLSiteGenre, DLSiteManiax, DLSiteTranslation};
 use crate::config::types::ApplicationConfig;
 use crate::constants::{DB_CF_OPTIONS, DB_OPTIONS};
 use crate::crawler::{dlsite, DLSiteCrawler};
@@ -29,8 +31,10 @@ pub(super) enum DLSiteSubCommand {
 pub(super) struct DLSiteSyncCommand {
     #[clap(long, short, action)]
     missing: bool,
-    #[clap(long, short, action)]
+    #[clap(long = "genre", default_value = "false")]
-    genre: bool,
+    do_sync_genre: bool,
+    #[clap(long = "work", default_value = "true")]
+    do_sync_work: bool
 }
 
 impl DLSiteSubCommand {
@@ -46,42 +50,108 @@ impl DLSiteSyncCommand {
         let now = Instant::now();
         let app_conf = ApplicationConfig::get_config()?;
         let mut db = RocksDB::new(DB_OPTIONS.clone(), DB_CF_OPTIONS.clone())?;
-        if self.genre {
+        let crawler = DLSiteCrawler::new()?;
-            Self::sync_genres(&app_conf).await?;
+        if self.do_sync_genre {
+            let genre_now = Instant::now();
+            Self::sync_genres(&mut db, &app_conf, &crawler).await?;
+            println!(
+                "{} {} Done in {:.2?}",
+                style("Genres").cyan(),
+                style("Syncing").green(),
+                genre_now.elapsed()
+            );
+        }
+        if self.do_sync_work {
+            let work_now = Instant::now();
+            self.sync_works(&app_conf, &mut db, &crawler).await?;
+            println!(
+                "{} {} Done in {:.2?}",
+                style("Works").cyan(),
+                style("Syncing").green(),
+                work_now.elapsed()
+            );
         }
-        self.sync_works(&app_conf, &mut db).await?;
         println!("{} Done in {:.2?}", style("Syncing").green(), now.elapsed());
         Ok(())
     }
 
-    async fn sync_genres(app_conf: &ApplicationConfig) -> Result<()> {
+    async fn sync_genres(db: &mut RocksDB, app_conf: &ApplicationConfig, crawler: &DLSiteCrawler) -> Result<()> {
+        let requested_categories = crawler.get_all_genres(&app_conf.basic_config.locale).await?;
+        let categories: Vec<DLSiteCategory> = requested_categories.iter()
+            .map(|g| g.clone().try_into())
+            .filter_map(Result::ok)
+            .collect();
+        db.set_values(&categories)?;
+        let genres = requested_categories.into_iter()
+            .flat_map(|v| v.values)
+            .collect_vec();
+        let existing_genres = db.get_all_values::<DLSiteGenre>()?;
+        let mut modified_genres: Vec<DLSiteGenre> = Vec::new();
+        for genre in genres {
+            let id = genre.value.parse::<u16>()?;
+            let existing_genre =
+                existing_genres.iter().find(|v| v.id == id);
+            if let Some(existing_genre) = existing_genre {
+                let name = DLSiteTranslation::try_from(genre.name)?;
+                if existing_genre.name.contains(&name) {
+                    modified_genres.push(existing_genre.clone());
+                    continue;
+                }
+                let mut modified_genre = existing_genre.clone();
+                modified_genre.name.push(name);
+                modified_genres.push(modified_genre);
+            }
+            else {
+                modified_genres.push(DLSiteGenre {
+                    id, name: vec![DLSiteTranslation::try_from(genre.name)?]
+                });
+            }
+        }
+        db.set_values(&modified_genres)?;
+
         Ok(())
     }
 
-    async fn sync_works(&self, app_conf: &ApplicationConfig, db: &mut RocksDB) -> Result<()> {
+    async fn sync_works(&self, app_conf: &ApplicationConfig, db: &mut RocksDB, crawler: &DLSiteCrawler) -> Result<()> {
-        let crawler = DLSiteCrawler::new();
         let existing_works = db.get_all_values::<models::DLSiteManiax>()?;
 
-        let work_list = self.get_work_list(&app_conf, existing_works).await?;
+        let work_list = self.get_work_list(&app_conf, &existing_works).await?;
         let rj_nums = work_list.clone().into_keys().collect::<Vec<_>>();
-        let mut maniaxes: Vec<models::DLSiteManiax> = Vec::new();
+
-        let mut game_infos = crawler.get_game_infos(rj_nums).await?;
+        let mut game_infos = crawler.get_game_infos(rj_nums, &app_conf.basic_config.locale).await?;
+        let existing_game_infos = db.get_all_values::<DLSiteManiax>()?;
+        let mut modified_maniaxes: Vec<DLSiteManiax> = Vec::new();
 
         let progress = ProgressBar::new(game_infos.len() as u64)
             .with_style(ProgressStyle::default_bar());
         while let Some(info) = game_infos.next().await {
-            let mut value: models::DLSiteManiax = info?.into();
+            let maniax = info?;
-            let maniax_folder = work_list.get(&value.rj_num).unwrap().to_owned();
+            let existing_maniax = existing_game_infos.iter()
-            value.folder_path = maniax_folder;
+                .find(|v| v.rj_num == maniax.rj_num);
-            maniaxes.push(value);
+            if let Some(existing_maniax) = existing_maniax {
+                let name = DLSiteTranslation::try_from(maniax.title)?;
+                if existing_maniax.name.contains(&name) {
+                    modified_maniaxes.push(existing_maniax.clone());
+                    continue;
+                }
+                let mut modified_maniax = existing_maniax.clone();
+                modified_maniax.name.push(name);
+                modified_maniaxes.push(modified_maniax);
+            }
+            else {
+                let mut value: DLSiteManiax = maniax.into();
+                let maniax_folder = work_list.get(&value.rj_num).unwrap().to_owned();
+                value.folder_path = maniax_folder;
+                modified_maniaxes.push(value);
+            }
             progress.inc(1);
         }
-        db.set_values(&maniaxes)?;
+        db.set_values(&modified_maniaxes)?;
 
         Ok(())
     }
 
-    async fn get_work_list(&self, app_conf: &ApplicationConfig, existing_works: Vec<models::DLSiteManiax>) -> Result<HashMap<String, PathBuf>> {
+    async fn get_work_list(&self, app_conf: &ApplicationConfig, existing_works: &[models::DLSiteManiax]) -> Result<HashMap<String, PathBuf>> {
         let existing_nums = existing_works.iter()
             .map(|x| x.rj_num.clone())
             .collect::<Vec<_>>();
@@ -92,7 +162,7 @@ impl DLSiteSyncCommand {
         let config_paths = app_conf.path_config.dlsite_paths.iter()
             .map(|path| Path::new(path))
             .collect::<Vec<_>>();
-        let dir_paths = helpers::get_all_folders(config_paths).await?;
+        let dir_paths = helpers::get_all_folders(&config_paths).await?;
         for dir_path in dir_paths {
             if !dir_path.is_dir() {
                 println!(

src/config/mod.rs

@@ -1,5 +1,5 @@
 use crate::config::types::{ApplicationConfig, BasicConfig, PathConfig};
-use crate::constants::{APP_CONIFG_FILE_PATH, APP_DB_DATA_DIR};
+use crate::constants::{APP_CONIFG_FILE_PATH, APP_DB_DATA_DIR, CACHE_MAP};
 use color_eyre::Result;
 use std::path::PathBuf;
 use language_tags::LanguageTag;
@@ -8,6 +8,8 @@ use serde_json;
 
 pub mod types;
 
+const CONFIG_KEY: &str = "app_conf";
+
 pub(crate) struct GameList<T> {
     games: Vec<T>,
     state: ListState,
@@ -15,45 +17,46 @@ pub(crate) struct GameList<T> {
 
 impl ApplicationConfig {
     pub fn get_config() -> Result<Self> {
-        if APP_CONIFG_FILE_PATH.exists() {
+        if CACHE_MAP.contains_key(CONFIG_KEY) {
+            Ok(serde_json::from_value(CACHE_MAP.get(CONFIG_KEY).unwrap().clone())?)
+        } else if APP_CONIFG_FILE_PATH.exists() {
             ApplicationConfig::from_file(&APP_CONIFG_FILE_PATH)
-        } else {
+        } else  {
-            Ok(ApplicationConfig::new())
+            ApplicationConfig::new()
         }
     }
 
     fn from_file(path: &PathBuf) -> Result<Self> {
         let reader = std::fs::File::open(path)?;
-        let result = serde_json::from_reader(reader)?;
+        let result: serde_json::Value = serde_json::from_reader(reader)?;
-        Ok(result)
+        CACHE_MAP.insert(CONFIG_KEY.to_string(), result.clone());
+        Ok(serde_json::from_value(result)?)
     }
 
-    fn new() -> Self {
+    fn new() -> Result<Self> {
         let default_locale = sys_locale::get_locale().unwrap_or(String::from("ja-JP"));
         let conf = Self {
             basic_config: BasicConfig {
                 db_path: APP_DB_DATA_DIR.to_str().unwrap().to_string(),
                 tick_rate: 250,
-                locale: LanguageTag::parse(&default_locale).unwrap(),
+                locale: LanguageTag::parse(&default_locale)?,
             },
             path_config: PathConfig {
                 dlsite_paths: vec![],
             },
         };
-        conf.clone()
+        conf.clone().write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf())?;
-            .write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf())
+        Ok(conf)
-            .unwrap();
-        conf
     }
 
     fn write_to_file(self, path: &PathBuf) -> Result<()> {
         let writer = std::fs::File::create(path)?;
-        let result = serde_json::to_writer_pretty(writer, &self)?;
+        serde_json::to_writer_pretty(writer, &self)?;
-        Ok(result)
+        Ok(())
 
     }
 
-    pub fn save(&self) -> Result<()> {
+    pub fn save(self) -> Result<()> {
-        self.clone().write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf())
+        self.write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf())
     }
 }

src/constants.rs

@@ -1,7 +1,11 @@
+use std::hash::RandomState;
 use directories::BaseDirs;
 use lazy_static::lazy_static;
 use std::path::PathBuf;
-use crate::models::{DLSiteManiax, RocksColumn};
+use std::sync::Arc;
+use dashmap::DashMap;
+use language_tags::LanguageTag;
+use crate::models::{DLSiteCategory, DLSiteGenre, DLSiteManiax, RocksColumn};
 
 const APP_DIR_NAME: &str = "sus_manager";
 lazy_static! {
@@ -18,7 +22,22 @@ lazy_static! {
 }
 
 lazy_static! {
-    pub static ref DB_COLUMNS: Vec<String> = vec![DLSiteManiax::get_column_name().to_string()];
+    pub static ref DB_COLUMNS: [String; 3] = [
+        DLSiteManiax::get_column_name().to_string(),
+        DLSiteGenre::get_column_name().to_string(),
+        DLSiteCategory::get_column_name().to_string(),
+    ];
+}
+
+lazy_static! {
+    pub static ref EN_LOCALE: LanguageTag = LanguageTag::parse("en").unwrap();
+    pub static ref JP_LOCALE: LanguageTag = LanguageTag::parse("ja").unwrap();
+    pub static ref SUPPORTED_LOCALES: [LanguageTag; 2] = [JP_LOCALE.clone(), EN_LOCALE.clone()];
+}
+
+lazy_static! {
+    pub static ref CACHE_MAP: Arc<DashMap<String, serde_json::Value>> =
+        Arc::new(DashMap::with_hasher(RandomState::default()));
 }
 
 fn get_db_options() -> rocksdb::Options {

src/crawler/dlsite.rs

@@ -1,29 +1,36 @@
 
 use std::collections::{HashMap, HashSet};
-use std::path::PathBuf;
+use std::path::{PathBuf};
 use color_eyre::eyre::eyre;
 use color_eyre::owo_colors::OwoColorize;
-use reqwest::{StatusCode, Url};
+use reqwest::{Url};
 use color_eyre::{Report, Result};
 use futures::stream::FuturesUnordered;
+use itertools::Itertools;
+use language_tags::LanguageTag;
 use lazy_static::lazy_static;
-use scraper::{Html, Selector};
+use scraper::{Element, Html, Selector};
 use serde::{Deserialize, Serialize};
-use crate::constants::{APP_DATA_DIR};
+use serde_json::Value;
+use crate::config::types::ApplicationConfig;
+use crate::constants::{APP_DATA_DIR, JP_LOCALE};
 use crate::crawler::Crawler;
+use crate::helpers::matches_primary_language;
+use crate::models;
+use crate::models::{DLSiteTranslation, PrimaryLanguage};
 
 //TODO: override locale with user one
 const DLSITE_URL: &str = "https://www.dlsite.com/";
 const DLSITE_PRODUCT_API_ENDPOINT: &str = "/maniax/product/info/ajax";
-const DLSITE_FILTER_OPTIONS_ENDPOINT: &str = "/maniax/fs/=/api_access/1/locale/ja_JP";
+const DLSITE_FS_ENDPOINT: &str = "/maniax/fs/=/api_access/1/";
 const DLSITE_MANIAX_PATH: &str = "/maniax/work/=/product_id/";
 lazy_static! {
     pub static ref DLSITE_IMG_FOLDER: PathBuf = APP_DATA_DIR.clone().join("dlsite").join("img");
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct DLSiteCrawler {
-    crawler: Crawler,
+    crawler: Crawler
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone)]
@@ -43,18 +50,38 @@ pub(crate) struct DLSiteManiax {
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone)]
-pub(crate) struct DLSiteGenreCategory {
+struct DLSiteFilter {
+    pub(crate) genre_all: Value
+}
 
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub(crate) struct DLSiteGenreCategory {
+    pub(crate) category_name: String,
+    pub(crate) values: Vec<DLSiteGenre>,
+    #[serde(skip)]
+    pub(crate) id: u8
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub(crate) struct DLSiteGenre {
+    pub(crate) value: String,
+    pub(crate) name: String
 }
 
 impl DLSiteCrawler {
-    pub fn new() -> Self {
+    pub fn new() -> Result<Self> {
-        Self {
+        let url = Url::parse(DLSITE_URL)?;
-            crawler: Crawler::new("DLSite", Url::parse(DLSITE_URL).unwrap())
+        let crawler = Self {
-        }
+            crawler: Crawler::new(
+                "DLSite",
+                url
+            )
+        };
+        Ok(crawler)
     }
 
-    pub async fn get_game_infos(&self, rj_nums: Vec<String>) -> Result<FuturesUnordered<impl Future<Output=Result<DLSiteManiax, Report>>>>
+    pub async fn get_game_infos(&self, rj_nums: Vec<String>, locale: &LanguageTag) -> Result<FuturesUnordered<impl Future<Output=Result<DLSiteManiax, Report>>>>
     {
         let invalid_nums = rj_nums.iter()
             .filter(|&n| !is_valid_rj_number(n))
@@ -65,10 +92,14 @@ impl DLSiteCrawler {
                 eyre!("Invalid numbers: {}", invalid_nums.join(", "))
             );
         }
-
+        let primary_language: PrimaryLanguage = locale.try_into()?;
-        let query = &format!("product_id={}", rj_nums.join(","));
+        let locale_str = match primary_language {
+            PrimaryLanguage::EN => "en_US",
+            PrimaryLanguage::JP => "ja_JP",
+        };
+        let query = &format!("product_id={}&locale={}", rj_nums.join(","), locale_str);
         let (value, _) = self.crawler
-            .get_json::<serde_json::Value>(DLSITE_PRODUCT_API_ENDPOINT, Some(query))
+            .get_json::<Value>(DLSITE_PRODUCT_API_ENDPOINT, Some(query))
             .await?;
         // try to catch '[]' empty result from the api
         let value_downcast_result: Result<HashMap<String, DLSiteManiax>, _> = serde_json::from_value(value);
@@ -78,14 +109,15 @@ impl DLSiteCrawler {
 
         let tasks = FuturesUnordered::new();
         for (rj_num, mut info) in maniax_result {
-            tasks.push(async {
+            tasks.push(async move {
                 let html_path = format!("{DLSITE_MANIAX_PATH}{rj_num}");
+                let query = format!("locale={locale_str}");
                 let (_, html_result) = tokio::join!(
                     self.save_main_image(&info, &rj_num),
-                    self.crawler.get_html(&html_path)
+                    self.crawler.get_html(&html_path, Some(&query))
                 );
                 let (html, _) = html_result?;
-                let genres = self.get_genres(&html).await?;
+                let genres = self.get_work_genres(&html, locale.try_into()?).await?;
                 info.genre_ids = genres;
                 info.rj_num = rj_num;
                 Ok::<DLSiteManiax, Report>(info)
@@ -122,17 +154,23 @@ impl DLSiteCrawler {
         Ok(())
     }
 
-    async fn get_genres(&self, html: &Html) -> Result<Vec<u16>> {
+    async fn get_work_genres(&self, html: &Html, primary_language: PrimaryLanguage) -> Result<Vec<u16>> {
         let selector = Result::unwrap(
             Selector::parse(
                 "#work_outline > tbody:nth-child(1)"
             )
         );
+        let genre_str = match primary_language {
+            PrimaryLanguage::EN => "Genre",
+            PrimaryLanguage::JP => "ジャンル"
+        };
+
         let result = html.select(&selector).next().unwrap();
         let genre_rows = result.child_elements().collect::<Vec<_>>();
-        let genre_rows_len = genre_rows.iter().count();
+        let t = genre_rows.iter().filter_map(|v| v.first_element_child().unwrap().text().next()).collect_vec();
-        // get second last row for genre
+        let genre_row = genre_rows.iter()
-        let genre_row = genre_rows.iter().skip(genre_rows_len - 2).next().unwrap();
+            .find(|v| v.first_element_child().unwrap().text().next().unwrap() == genre_str)
+            .unwrap();
         let data = genre_row
             .child_elements().skip(1).next().unwrap()
             .child_elements().next().unwrap();
@@ -151,6 +189,36 @@ impl DLSiteCrawler {
     }
 }
 
+impl DLSiteCrawler {
+    pub async fn get_all_genres(&self, locale: &LanguageTag) -> Result<Vec<DLSiteGenreCategory>> {
+        let primary_language: PrimaryLanguage = locale.try_into()?;
+        let locale_str = match primary_language {
+            PrimaryLanguage::EN => "en_US",
+            PrimaryLanguage::JP => "ja_JP",
+        };
+        let query = format!("locale={}", locale_str);
+
+        let (json, _) = self.crawler.get_json::<DLSiteFilter>(DLSITE_FS_ENDPOINT, Some(&query)).await?;
+        let values =
+            if matches_primary_language(&locale, &JP_LOCALE) {
+                serde_json::from_value::<Vec<DLSiteGenreCategory>>(json.genre_all)?
+            } else {
+                // IDK why they are using different object type bruh
+                serde_json::from_value::<HashMap<u16, DLSiteGenreCategory>>(json.genre_all)?
+                    .into_iter().map(|(_, v)| v)
+                    .collect_vec()
+            };
+
+        let mut categories = Vec::new();
+        for (i, value) in values.into_iter().enumerate() {
+            let mut category = value.clone();
+            category.id = i as u8;
+            categories.push(category);
+        }
+        Ok(categories)
+    }
+}
+
 pub fn is_valid_rj_number(rj_num: &str) -> bool {
     let len = rj_num.len();
     if len != 8 && len != 10 {

src/crawler/mod.rs

@@ -1,5 +1,6 @@
 pub mod dlsite;
 
+use std::sync::Arc;
 pub use dlsite::*;
 use color_eyre::eyre::eyre;
 use crate::constants::APP_CACHE_PATH;
@@ -10,12 +11,12 @@ use robotstxt::DefaultMatcher;
 use scraper::Html;
 use serde::de::DeserializeOwned;
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 struct Crawler {
     id: String,
     pub(crate) base_url: Url,
     client: Client,
-    robots_txt: Option<String>,
+    robots_txt: Option<String>
 }
 
 impl Crawler {
@@ -67,10 +68,11 @@ impl Crawler {
         }
     }
 
-    pub async fn get_html(&self, path: &str) -> Result<(Html, StatusCode)> {
+    pub async fn get_html(&self, path: &str, query: Option<&str>) -> Result<(Html, StatusCode)> {
         let mut url = self.base_url.clone();
         self.check_access(&url).await?;
         url.set_path(path);
+        url.set_query(query);
         let res = self.client.get(url).send().await?;
         let status = res.status();
         let html_text = &res.text().await?;
@@ -106,5 +108,5 @@ impl Crawler {
         let status = res.status();
         let bytes = res.bytes().await?;
         Ok((bytes.to_vec(), status))
-    }
+   }
 }

src/helpers/db.rs

@@ -36,7 +36,9 @@ impl RocksDB {
         if query_res.is_none() {
             return Ok(None);
         }
-        Ok(Some(serde_json::from_slice(&query_res.unwrap())?))
+        let mut value: TColumn = serde_json::from_slice(&query_res.unwrap())?;
+        value.set_id(id.clone());
+        Ok(Some(value))
     }
 
     pub fn set_value<TColumn>(&self, value: &TColumn) -> Result<()>
@@ -56,7 +58,8 @@ impl RocksDB {
         for id in ids {
             let query_res = transaction.get_cf(&cf, serde_json::to_string(id)?)?;
             if let Some(res) = query_res {
-                let value = serde_json::from_slice(&res)?;
+                let mut value: TColumn = serde_json::from_slice(&res)?;
+                value.set_id(id.clone());
                 values.push(value);
             }
         }

src/helpers/mod.rs

@@ -3,9 +3,10 @@ pub mod db;
 use std::path::{Path, PathBuf};
 use color_eyre::eyre::eyre;
 use color_eyre::owo_colors::OwoColorize;
+use language_tags::LanguageTag;
 use tokio::fs;
 use crate::config::types::ApplicationConfig;
-use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR};
+use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR, SUPPORTED_LOCALES};
 use crate::crawler::DLSITE_IMG_FOLDER;
 
 
@@ -27,7 +28,7 @@ pub async fn initialize_folders() -> color_eyre::Result<()> {
     Ok(())
 }
 
-pub async fn get_all_folders(paths: Vec<&Path>) -> color_eyre::Result<Vec<PathBuf>> {
+pub async fn get_all_folders(paths: &[&Path]) -> color_eyre::Result<Vec<PathBuf>> {
     let mut folders: Vec<PathBuf> = Vec::new();
     for path in paths {
         let path = path.to_path_buf();
@@ -42,3 +43,7 @@ pub async fn get_all_folders(paths: Vec<&Path>) -> color_eyre::Result<Vec<PathBu
     }
     Ok(folders)
 }
+
+pub fn matches_primary_language(left: &LanguageTag, right: &LanguageTag) -> bool {
+    left.primary_language() == right.primary_language()
+}

src/models/game.rs

@@ -1,10 +1,13 @@
 use std::path::PathBuf;
 use color_eyre::{eyre, Report};
-use language_tags::LanguageTag;
 use serde::{Deserialize, Serialize};
 use crate::config::types::ApplicationConfig;
-use crate::models::{RocksColumn, RocksReference, RocksReferences};
+use crate::constants::{EN_LOCALE, JP_LOCALE};
+use crate::crawler::DLSiteGenreCategory;
+use crate::helpers::matches_primary_language;
+use crate::models::{RocksColumn, RocksReferences};
 
+//region Maniax
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub(crate) struct DLSiteManiax {
     #[serde(skip)]
@@ -18,10 +21,11 @@ pub(crate) struct DLSiteManiax {
 
 impl From<crate::crawler::dlsite::DLSiteManiax> for DLSiteManiax {
     fn from(value: crate::crawler::DLSiteManiax) -> Self {
+        let title = DLSiteTranslation::try_from(value.title.as_str()).unwrap();
         Self {
             rj_num: value.rj_num,
             genre_ids: value.genre_ids,
-            name: vec![],
+            name: vec![title],
             sells_count: value.sells_count,
             folder_path: value.folder_path,
             version: None
@@ -50,13 +54,14 @@ impl RocksReferences<DLSiteGenre> for DLSiteManiax {
         self.genre_ids.clone()
     }
 }
+//endregion
 
+//region Genre
 #[derive(Clone, Debug, Serialize, Deserialize)]
-pub(crate) struct DLSiteGenre {
+pub struct DLSiteGenre {
     #[serde(skip)]
     pub id: u16,
-    pub category_id: u16,
+    pub name: Vec<DLSiteTranslation>
-    pub translations: Vec<DLSiteTranslation>
 }
 
 impl RocksColumn for DLSiteGenre {
@@ -74,22 +79,45 @@ impl RocksColumn for DLSiteGenre {
         String::from("dl_genres")
     }
 }
+//endregion
 
-impl RocksReference<DLSiteCategory> for DLSiteGenre {
+//region Category
-    fn get_reference_id(&self) -> <DLSiteCategory as RocksColumn>::Id {
-        self.category_id.clone()
-    }
-}
-
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub(crate) struct DLSiteCategory {
     #[serde(skip)]
-    pub id: u16,
+    pub id: String,
-    pub translations: Vec<DLSiteTranslation>
+    pub genre_ids: Vec<u16>,
+    pub name: DLSiteTranslation
+}
+
+impl TryFrom<DLSiteGenreCategory> for DLSiteCategory {
+    type Error = Report;
+
+    fn try_from(value: DLSiteGenreCategory) -> Result<Self, Self::Error> {
+        let category = Self {
+            id: format!(
+                "{}/{}",
+                value.id,
+                ApplicationConfig::get_config()?.basic_config.locale.primary_language()
+            ),
+            genre_ids: value.values.iter()
+                .map(|v| v.value.parse::<u16>())
+                .filter_map(Result::ok)
+                .collect(),
+            name: DLSiteTranslation::try_from(value.category_name.as_str())?,
+        };
+        Ok(category)
+    }
+}
+
+impl RocksReferences<DLSiteGenre> for DLSiteCategory {
+    fn get_reference_ids(&self) -> Vec<<DLSiteGenre as RocksColumn>::Id> {
+        self.genre_ids.clone()
+    }
 }
 
 impl RocksColumn for DLSiteCategory {
-    type Id = u16;
+    type Id = String;
     fn get_id(&self) -> Self::Id {
         self.id.clone()
     }
@@ -99,11 +127,13 @@ impl RocksColumn for DLSiteCategory {
     }
 
     fn get_column_name() -> String {
-        String::from("dl_translations")
+        String::from("dl_categories")
     }
 }
+//endregion
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+//region Translation
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 pub(crate) enum DLSiteTranslation {
     EN(String), JP(String)
 }
@@ -111,17 +141,38 @@ pub(crate) enum DLSiteTranslation {
 impl TryFrom<&str> for DLSiteTranslation {
     type Error = Report;
     fn try_from(value: &str) -> color_eyre::Result<Self> {
+        Self::try_from(value.to_string())
+    }
+}
+
+impl TryFrom<String> for DLSiteTranslation {
+    type Error = Report;
+    fn try_from(value: String) -> color_eyre::Result<Self> {
         let app_conf = ApplicationConfig::get_config()?;
         let locale = app_conf.basic_config.locale;
 
-        let en_locale = LanguageTag::parse("en-US")?;
+        if matches_primary_language(&locale, &EN_LOCALE) {
-        if locale.matches(&en_locale) {
+            return Ok(DLSiteTranslation::EN(value));
-            return Ok(DLSiteTranslation::EN(value.to_string()));
         }
-        let jp_locale = LanguageTag::parse("ja-JP")?;
+        if matches_primary_language(&locale, &JP_LOCALE) {
-        if locale.matches(&jp_locale) {
+            return Ok(DLSiteTranslation::JP(value));
-            return Ok(DLSiteTranslation::JP(value.to_string()));
         }
-        Err(eyre::eyre!("Invalid Locale: {:?}; Support {:?}", locale, [en_locale, jp_locale]))
+        Err(eyre::eyre!(
+            "Invalid Locale: {:?}; Support {:?}",
+            locale,
+            [EN_LOCALE.to_string(), JP_LOCALE.to_string()])
+        )
     }
 }
+
+impl TryInto<String> for DLSiteTranslation {
+    type Error = Report;
+
+    fn try_into(self) -> Result<String, Self::Error> {
+        match self {
+            DLSiteTranslation::EN(val) => Ok(val),
+            DLSiteTranslation::JP(val) => Ok(val),
+        }
+    }
+}
+//endregion

src/models/mod.rs

@@ -1,11 +1,16 @@
 mod game;
 
+use color_eyre::eyre::eyre;
+use color_eyre::Report;
+use language_tags::LanguageTag;
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 pub(crate) use game::*;
+use crate::constants::{EN_LOCALE, JP_LOCALE};
+use crate::helpers::matches_primary_language;
 
 pub trait RocksColumn {
-    type Id: Serialize + DeserializeOwned;
+    type Id: Serialize + DeserializeOwned + Clone;
     fn get_id(&self) -> Self::Id;
     fn set_id(&mut self, id: Self::Id);
     fn get_column_name() -> String;
@@ -18,3 +23,24 @@ pub trait RocksReference<T> where T: RocksColumn {
 pub trait RocksReferences<T> where T: RocksColumn {
     fn get_reference_ids(&self) -> Vec<T::Id>;
 }
+
+#[derive(Debug, Clone)]
+pub enum PrimaryLanguage {
+    EN, JP
+}
+
+impl TryFrom<&LanguageTag> for PrimaryLanguage {
+    type Error = Report;
+
+    fn try_from(value: &LanguageTag) -> Result<Self, Self::Error> {
+        if matches_primary_language(&value, &EN_LOCALE) {
+            Ok(PrimaryLanguage::EN)
+        }
+        else if matches_primary_language(&value, &JP_LOCALE) {
+            Ok(PrimaryLanguage::JP)
+        }
+        else {
+            Err(eyre!("No matching primary language found for {}", value))
+        }
+    }
+}