fromost/sus-manager
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Add syncing for genres and works

Browse Source
This commit is contained in:
fromost
2025-10-30 17:34:39 +08:00
parent 3f5dee13f5
commit 2538049f4a
28 changed files with 350 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
src/cli/sync.rs Normal file → Executable file
View File

@@ -1,12 +1,14 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use clap::{Args, Command, Parser, Subcommand};
use color_eyre::eyre::Result;
use clap::{Parser};
use color_eyre::eyre::{Result};
use crossterm::style::{style, Stylize};
use futures::StreamExt;
use indicatif::{ProgressBar, ProgressStyle};
use itertools::Itertools;
use tokio::time::Instant;
use crate::models;
use crate::models::{DLSiteCategory, DLSiteGenre, DLSiteManiax, DLSiteTranslation};
use crate::config::types::ApplicationConfig;
use crate::constants::{DB_CF_OPTIONS, DB_OPTIONS};
use crate::crawler::{dlsite, DLSiteCrawler};
@@ -29,8 +31,10 @@ pub(super) enum DLSiteSubCommand {
pub(super) struct DLSiteSyncCommand {
#[clap(long, short, action)]
missing: bool,
#[clap(long, short, action)]
genre: bool,
#[clap(long = "genre", default_value = "false")]
do_sync_genre: bool,
#[clap(long = "work", default_value = "true")]
do_sync_work: bool
}
impl DLSiteSubCommand {
@@ -46,42 +50,108 @@ impl DLSiteSyncCommand {
let now = Instant::now();
let app_conf = ApplicationConfig::get_config()?;
let mut db = RocksDB::new(DB_OPTIONS.clone(), DB_CF_OPTIONS.clone())?;
if self.genre {
Self::sync_genres(&app_conf).await?;
let crawler = DLSiteCrawler::new()?;
if self.do_sync_genre {
let genre_now = Instant::now();
Self::sync_genres(&mut db, &app_conf, &crawler).await?;
println!(
"{} {} Done in {:.2?}",
style("Genres").cyan(),
style("Syncing").green(),
genre_now.elapsed()
);
}
if self.do_sync_work {
let work_now = Instant::now();
self.sync_works(&app_conf, &mut db, &crawler).await?;
println!(
"{} {} Done in {:.2?}",
style("Works").cyan(),
style("Syncing").green(),
work_now.elapsed()
);
}
self.sync_works(&app_conf, &mut db).await?;
println!("{} Done in {:.2?}", style("Syncing").green(), now.elapsed());
Ok(())
}
async fn sync_genres(app_conf: &ApplicationConfig) -> Result<()> {
async fn sync_genres(db: &mut RocksDB, app_conf: &ApplicationConfig, crawler: &DLSiteCrawler) -> Result<()> {
let requested_categories = crawler.get_all_genres(&app_conf.basic_config.locale).await?;
let categories: Vec<DLSiteCategory> = requested_categories.iter()
.map(|g| g.clone().try_into())
.filter_map(Result::ok)
.collect();
db.set_values(&categories)?;
let genres = requested_categories.into_iter()
.flat_map(|v| v.values)
.collect_vec();
let existing_genres = db.get_all_values::<DLSiteGenre>()?;
let mut modified_genres: Vec<DLSiteGenre> = Vec::new();
for genre in genres {
let id = genre.value.parse::<u16>()?;
let existing_genre =
existing_genres.iter().find(|v| v.id == id);
if let Some(existing_genre) = existing_genre {
let name = DLSiteTranslation::try_from(genre.name)?;
if existing_genre.name.contains(&name) {
modified_genres.push(existing_genre.clone());
continue;
}
let mut modified_genre = existing_genre.clone();
modified_genre.name.push(name);
modified_genres.push(modified_genre);
}
else {
modified_genres.push(DLSiteGenre {
id, name: vec![DLSiteTranslation::try_from(genre.name)?]
});
}
}
db.set_values(&modified_genres)?;
Ok(())
}
async fn sync_works(&self, app_conf: &ApplicationConfig, db: &mut RocksDB) -> Result<()> {
let crawler = DLSiteCrawler::new();
async fn sync_works(&self, app_conf: &ApplicationConfig, db: &mut RocksDB, crawler: &DLSiteCrawler) -> Result<()> {
let existing_works = db.get_all_values::<models::DLSiteManiax>()?;
let work_list = self.get_work_list(&app_conf, existing_works).await?;
let work_list = self.get_work_list(&app_conf, &existing_works).await?;
let rj_nums = work_list.clone().into_keys().collect::<Vec<_>>();
let mut maniaxes: Vec<models::DLSiteManiax> = Vec::new();
let mut game_infos = crawler.get_game_infos(rj_nums).await?;
let mut game_infos = crawler.get_game_infos(rj_nums, &app_conf.basic_config.locale).await?;
let existing_game_infos = db.get_all_values::<DLSiteManiax>()?;
let mut modified_maniaxes: Vec<DLSiteManiax> = Vec::new();
let progress = ProgressBar::new(game_infos.len() as u64)
.with_style(ProgressStyle::default_bar());
while let Some(info) = game_infos.next().await {
let mut value: models::DLSiteManiax = info?.into();
let maniax_folder = work_list.get(&value.rj_num).unwrap().to_owned();
value.folder_path = maniax_folder;
maniaxes.push(value);
let maniax = info?;
let existing_maniax = existing_game_infos.iter()
.find(|v| v.rj_num == maniax.rj_num);
if let Some(existing_maniax) = existing_maniax {
let name = DLSiteTranslation::try_from(maniax.title)?;
if existing_maniax.name.contains(&name) {
modified_maniaxes.push(existing_maniax.clone());
continue;
}
let mut modified_maniax = existing_maniax.clone();
modified_maniax.name.push(name);
modified_maniaxes.push(modified_maniax);
}
else {
let mut value: DLSiteManiax = maniax.into();
let maniax_folder = work_list.get(&value.rj_num).unwrap().to_owned();
value.folder_path = maniax_folder;
modified_maniaxes.push(value);
}
progress.inc(1);
}
db.set_values(&maniaxes)?;
db.set_values(&modified_maniaxes)?;
Ok(())
}
async fn get_work_list(&self, app_conf: &ApplicationConfig, existing_works: Vec<models::DLSiteManiax>) -> Result<HashMap<String, PathBuf>> {
async fn get_work_list(&self, app_conf: &ApplicationConfig, existing_works: &[models::DLSiteManiax]) -> Result<HashMap<String, PathBuf>> {
let existing_nums = existing_works.iter()
.map(|x| x.rj_num.clone())
.collect::<Vec<_>>();
@@ -92,7 +162,7 @@ impl DLSiteSyncCommand {
let config_paths = app_conf.path_config.dlsite_paths.iter()
.map(|path| Path::new(path))
.collect::<Vec<_>>();
let dir_paths = helpers::get_all_folders(config_paths).await?;
let dir_paths = helpers::get_all_folders(&config_paths).await?;
for dir_path in dir_paths {
if !dir_path.is_dir() {
println!(