fromost/sus-manager
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor config

Browse Source
This commit is contained in:
fromost
2025-10-27 20:37:42 +08:00
parent ab7f6fe206
commit 3f5dee13f5
11 changed files with 188 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.config/config.toml Normal file
View File

@@ -0,0 +1,3 @@
[env]
TERM = "xterm-256color"
RUST_BACKTRACE = 0

13
Cargo.toml
View File

@@ -21,9 +21,18 @@ scraper = "0.24.0"
rat-cursor = "1.2.1"
serde_json = "1.0.145"
image = "0.25.8"
colored = "3.0.0"
log = "0.4.28"
num_cpus = "1.17.0"
sys-locale = "0.3.2"
jemallocator = "0.5.4"
[dependencies.language-tags]
version = "0.3.2"
features = ["serde"]
[dependencies.indicatif]
version = "0.18.1"
features = ["futures", "tokio"]
[dependencies.rocksdb]
version = "0.24.0"
@@ -52,7 +61,7 @@ features = ["derive", "cargo"]
[dependencies.reqwest]
version = "0.12.23"
features = ["blocking", "json"]
features = ["blocking", "json", "rustls-tls"]
[dependencies.tokio]
version = "1.47.1"

27
src/cli/mod.rs
View File

@@ -6,12 +6,13 @@ use clap::{command, Args, Command, Parser, Subcommand};
use color_eyre::Result;
use ratatui::crossterm;
use crate::cli::folder::FolderCommand;
use crate::cli::sync::SyncCommand;
use crate::cli::sync::DLSiteCommand;
#[derive(Parser, Debug)]
enum CliSubCommand {
Folder(FolderCommand),
Sync(SyncCommand),
#[command(name = "dlsite")]
DLSite(DLSiteCommand),
}
#[derive(Parser, Debug)]
@@ -21,26 +22,6 @@ pub(crate) struct Cli {
subcommand: Option<CliSubCommand>,
}
impl Subcommand for Cli {
fn augment_subcommands(cmd: Command) -> Command {
cmd.subcommand(FolderCommand::augment_args(Command::new("folder")))
.subcommand_required(true)
.subcommand(SyncCommand::augment_args(Command::new("sync")))
.subcommand_required(true)
}
fn augment_subcommands_for_update(cmd: Command) -> Command {
cmd.subcommand(FolderCommand::augment_args(Command::new("folder")))
.subcommand_required(true)
.subcommand(SyncCommand::augment_args(Command::new("sync")))
.subcommand_required(true)
}
fn has_subcommand(name: &str) -> bool {
matches!(name, "folder" | "sync")
}
}
impl Cli {
pub async fn run(&self) -> Result<()> {
helpers::initialize_folders().await?;
@@ -70,7 +51,7 @@ impl CliSubCommand {
pub async fn handle(&self) -> Result<()> {
match self {
CliSubCommand::Folder(cmd) => cmd.subcommand.handle().await,
CliSubCommand::Sync(cmd) => cmd.subcommand.handle().await,
CliSubCommand::DLSite(cmd) => cmd.subcommand.handle().await,
}
}
}

134
src/cli/sync.rs
View File

@@ -1,7 +1,10 @@
use std::path::Path;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use clap::{Args, Command, Parser, Subcommand};
use color_eyre::eyre::Result;
use colored::Colorize;
use crossterm::style::{style, Stylize};
use futures::StreamExt;
use indicatif::{ProgressBar, ProgressStyle};
use tokio::time::Instant;
use crate::models;
use crate::config::types::ApplicationConfig;
@@ -11,51 +14,43 @@ use crate::helpers;
use crate::helpers::db::RocksDB;
#[derive(Parser, Debug)]
pub(super) struct SyncCommand {
pub(super) struct DLSiteCommand {
#[command(subcommand)]
pub(super) subcommand: SyncSubCommand,
pub(super) subcommand: DLSiteSubCommand,
}
#[derive(Parser, Debug)]
pub(super) enum SyncSubCommand {
DLSite(SyncDLSiteCommand)
pub(super) enum DLSiteSubCommand {
#[command(name = "sync")]
Sync(DLSiteSyncCommand)
}
#[derive(Parser, Debug)]
pub(super) struct SyncDLSiteCommand;
impl Subcommand for SyncCommand {
fn augment_subcommands(cmd: Command) -> Command {
cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite")))
.subcommand_required(true)
pub(super) struct DLSiteSyncCommand {
#[clap(long, short, action)]
missing: bool,
#[clap(long, short, action)]
genre: bool,
}
fn augment_subcommands_for_update(cmd: Command) -> Command {
cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite")))
.subcommand_required(true)
}
fn has_subcommand(name: &str) -> bool {
matches!(name, "dlsite")
}
}
impl SyncSubCommand {
pub async fn handle(&self) -> color_eyre::Result<()> {
impl DLSiteSubCommand {
pub async fn handle(&self) -> Result<()> {
match self {
Self::DLSite(cmd) => cmd.handle().await,
Self::Sync(cmd) => cmd.handle().await,
}
}
}
impl SyncDLSiteCommand {
pub async fn handle(&self) -> color_eyre::Result<()> {
impl DLSiteSyncCommand {
pub async fn handle(&self) -> Result<()> {
let now = Instant::now();
let app_conf = ApplicationConfig::get_config()?;
let mut db = RocksDB::new(DB_OPTIONS.clone(), DB_CF_OPTIONS.clone())?;
if self.genre {
Self::sync_genres(&app_conf).await?;
Self::sync_works(&app_conf, &mut db).await?;
println!("{} Done in {:.2?}", "Syncing".green(), now.elapsed());
}
self.sync_works(&app_conf, &mut db).await?;
println!("{} Done in {:.2?}", style("Syncing").green(), now.elapsed());
Ok(())
}
@@ -63,31 +58,68 @@ impl SyncDLSiteCommand {
Ok(())
}
async fn sync_works(app_conf: &ApplicationConfig, db: &mut RocksDB) -> Result<()> {
async fn sync_works(&self, app_conf: &ApplicationConfig, db: &mut RocksDB) -> Result<()> {
let crawler = DLSiteCrawler::new();
let mut rj_nums: Vec<String> = Vec::new();
let config_paths = app_conf.path_config.dlsite_paths.iter()
.map(|path| Path::new(path).to_path_buf())
.collect::<Vec<_>>();
let dir_paths = helpers::get_all_folders(&config_paths).await?;
for dir_path in dir_paths.iter() {
if !dir_path.is_dir() {
println!("{dir_path:?} is not a directory");
continue;
let existing_works = db.get_all_values::<models::DLSiteManiax>()?;
let work_list = self.get_work_list(&app_conf, existing_works).await?;
let rj_nums = work_list.clone().into_keys().collect::<Vec<_>>();
let mut maniaxes: Vec<models::DLSiteManiax> = Vec::new();
let mut game_infos = crawler.get_game_infos(rj_nums).await?;
let progress = ProgressBar::new(game_infos.len() as u64)
.with_style(ProgressStyle::default_bar());
while let Some(info) = game_infos.next().await {
let mut value: models::DLSiteManiax = info?.into();
let maniax_folder = work_list.get(&value.rj_num).unwrap().to_owned();
value.folder_path = maniax_folder;
maniaxes.push(value);
progress.inc(1);
}
let dir_name = dir_path
.file_name().unwrap()
.to_str().unwrap();
if !dlsite::is_valid_rj_number(dir_name) {
println!("{} {}", dir_path.to_str().unwrap().blue(), "is not a valid rj number, please add it manually".red());
continue;
}
rj_nums.push(dir_name.to_string());
}
let maniaxes: Vec<models::DLSiteManiax> = crawler.get_game_infos(rj_nums).await?.into_iter()
.map(|x| x.into())
.collect::<Vec<_>>();
db.set_values(&maniaxes)?;
Ok(())
}
async fn get_work_list(&self, app_conf: &ApplicationConfig, existing_works: Vec<models::DLSiteManiax>) -> Result<HashMap<String, PathBuf>> {
let existing_nums = existing_works.iter()
.map(|x| x.rj_num.clone())
.collect::<Vec<_>>();
let existing_folders = existing_works.iter()
.map(|x| x.folder_path.to_str().unwrap().to_string())
.collect::<Vec<_>>();
let mut works_list: HashMap<String, PathBuf> = HashMap::new();
let config_paths = app_conf.path_config.dlsite_paths.iter()
.map(|path| Path::new(path))
.collect::<Vec<_>>();
let dir_paths = helpers::get_all_folders(config_paths).await?;
for dir_path in dir_paths {
if !dir_path.is_dir() {
println!(
"{} {}",
style(dir_path.to_str().unwrap()).blue(),
style("is not a directory").red()
);
continue;
}
let dir_path_str = dir_path.to_str().unwrap().to_string();
let dir_name = dir_path
.file_name().unwrap()
.to_str().unwrap()
.to_string();
if !dlsite::is_valid_rj_number(&dir_name) && !existing_folders.contains(&dir_path_str) {
println!(
"{} {}",
style(dir_path.to_str().unwrap()).blue(),
style("is not a valid rj number, please add it manually").red()
);
continue;
}
if self.missing && existing_nums.contains(&dir_name) {
continue;
}
works_list.insert(dir_name, dir_path);
}
Ok(works_list)
}
}

9
src/config/mod.rs
View File

@@ -2,10 +2,17 @@ use crate::config::types::{ApplicationConfig, BasicConfig, PathConfig};
use crate::constants::{APP_CONIFG_FILE_PATH, APP_DB_DATA_DIR};
use color_eyre::Result;
use std::path::PathBuf;
use language_tags::LanguageTag;
use ratatui::widgets::ListState;
use serde_json;
pub mod types;
pub(crate) struct GameList<T> {
games: Vec<T>,
state: ListState,
}
impl ApplicationConfig {
pub fn get_config() -> Result<Self> {
if APP_CONIFG_FILE_PATH.exists() {
@@ -22,10 +29,12 @@ impl ApplicationConfig {
}
fn new() -> Self {
let default_locale = sys_locale::get_locale().unwrap_or(String::from("ja-JP"));
let conf = Self {
basic_config: BasicConfig {
db_path: APP_DB_DATA_DIR.to_str().unwrap().to_string(),
tick_rate: 250,
locale: LanguageTag::parse(&default_locale).unwrap(),
},
path_config: PathConfig {
dlsite_paths: vec![],

2
src/config/types.rs
View File

@@ -1,3 +1,4 @@
use language_tags::LanguageTag;
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize)]
@@ -9,6 +10,7 @@ pub struct ApplicationConfig {
#[derive(Clone, Debug, Serialize, Deserialize)]
pub(crate) struct BasicConfig {
pub db_path: String,
pub locale: LanguageTag,
pub tick_rate: u64,
}

42
src/crawler/dlsite.rs
View File

@@ -1,16 +1,14 @@
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use color_eyre::eyre::eyre;
use color_eyre::owo_colors::OwoColorize;
use reqwest::Url;
use reqwest::{StatusCode, Url};
use color_eyre::{Report, Result};
use colored::Colorize;
use futures::stream::FuturesUnordered;
use futures::StreamExt;
use lazy_static::lazy_static;
use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use tokio::time::Instant;
use crate::constants::{APP_DATA_DIR};
use crate::crawler::Crawler;
@@ -40,6 +38,13 @@ pub(crate) struct DLSiteManiax {
pub(crate) genre_ids: Vec<u16>,
#[serde(skip)]
pub(crate) rj_num: String,
#[serde(skip)]
pub(crate) folder_path: PathBuf,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
pub(crate) struct DLSiteGenreCategory {
}
impl DLSiteCrawler {
@@ -49,7 +54,8 @@ impl DLSiteCrawler {
}
}
pub async fn get_game_infos(&self, rj_nums: Vec<String>) -> Result<Vec<DLSiteManiax>> {
pub async fn get_game_infos(&self, rj_nums: Vec<String>) -> Result<FuturesUnordered<impl Future<Output=Result<DLSiteManiax, Report>>>>
{
let invalid_nums = rj_nums.iter()
.filter(|&n| !is_valid_rj_number(n))
.map(|n| n.to_string())
@@ -61,17 +67,23 @@ impl DLSiteCrawler {
}
let query = &format!("product_id={}", rj_nums.join(","));
let (maniax_result, _) = self.crawler
.get_json::<HashMap<String, DLSiteManiax>>(DLSITE_PRODUCT_API_ENDPOINT, Some(query))
let (value, _) = self.crawler
.get_json::<serde_json::Value>(DLSITE_PRODUCT_API_ENDPOINT, Some(query))
.await?;
// try to catch '[]' empty result from the api
let value_downcast_result: Result<HashMap<String, DLSiteManiax>, _> = serde_json::from_value(value);
let maniax_result = value_downcast_result.unwrap_or(HashMap::new());
Self::verify_all_works_exists(&maniax_result, rj_nums);
let mut tasks = FuturesUnordered::new();
let tasks = FuturesUnordered::new();
for (rj_num, mut info) in maniax_result {
tasks.push(async {
let html_path = format!("{DLSITE_MANIAX_PATH}{rj_num}");
let (_, html_result) = tokio::join!(self.save_main_image(&info, &rj_num), self.crawler.get_html(&html_path));
let (_, html_result) = tokio::join!(
self.save_main_image(&info, &rj_num),
self.crawler.get_html(&html_path)
);
let (html, _) = html_result?;
let genres = self.get_genres(&html).await?;
info.genre_ids = genres;
@@ -79,12 +91,7 @@ impl DLSiteCrawler {
Ok::<DLSiteManiax, Report>(info)
})
}
let mut maniax_infos = Vec::new();
while let Some(result) = tasks.next().await {
maniax_infos.push(result?);
}
Ok(maniax_infos)
Ok(tasks)
}
fn verify_all_works_exists(maniax_result: &HashMap<String, DLSiteManiax>, rj_nums: Vec<String>) {
@@ -123,8 +130,9 @@ impl DLSiteCrawler {
);
let result = html.select(&selector).next().unwrap();
let genre_rows = result.child_elements().collect::<Vec<_>>();
let genre_len = genre_rows.iter().count();
let genre_row = genre_rows.iter().skip(genre_len - 2).next().unwrap();
let genre_rows_len = genre_rows.iter().count();
// get second last row for genre
let genre_row = genre_rows.iter().skip(genre_rows_len - 2).next().unwrap();
let data = genre_row
.child_elements().skip(1).next().unwrap()
.child_elements().next().unwrap();

9
src/helpers/db.rs
View File

@@ -1,9 +1,11 @@
use crate::constants::{APP_DB_DATA_DIR, DB_COLUMNS};
use rocksdb::{ColumnFamilyDescriptor, IteratorMode, OptimisticTransactionDB, Options, ReadOptions};
use std::path::Path;
use crate::constants::{DB_COLUMNS};
use rocksdb::{ColumnFamilyDescriptor, IteratorMode, OptimisticTransactionDB, Options};
use serde::{Serialize};
use serde::de::DeserializeOwned;
use crate::models::{RocksColumn, RocksReference, RocksReferences};
use color_eyre::Result;
use crate::config::types::ApplicationConfig;
pub struct RocksDB {
db: OptimisticTransactionDB,
@@ -11,12 +13,13 @@ pub struct RocksDB {
impl RocksDB {
pub fn new(db_opts: Options, cf_opts: Options) -> Result<Self> {
let app_conf = ApplicationConfig::get_config()?;
let cfs = DB_COLUMNS.iter()
.map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), cf_opts.clone()))
.collect::<Vec<_>>();
let db = OptimisticTransactionDB::open_cf_descriptors(
&db_opts,
APP_DB_DATA_DIR.as_path(),
Path::new(&app_conf.basic_config.db_path),
cfs
)?;
let rocks = Self {

15
src/helpers/mod.rs
View File

@@ -1,14 +1,16 @@
pub mod db;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use color_eyre::eyre::eyre;
use color_eyre::owo_colors::OwoColorize;
use tokio::fs;
use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR, APP_DB_DATA_DIR};
use crate::config::types::ApplicationConfig;
use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR};
use crate::crawler::DLSITE_IMG_FOLDER;
pub async fn initialize_folders() -> color_eyre::Result<()> {
let app_conf = ApplicationConfig::get_config()?;
if !APP_CONFIG_DIR.exists() {
fs::create_dir_all(APP_CONFIG_DIR.as_path()).await?;
}
@@ -18,16 +20,17 @@ pub async fn initialize_folders() -> color_eyre::Result<()> {
if !DLSITE_IMG_FOLDER.exists() {
fs::create_dir_all(DLSITE_IMG_FOLDER.as_path()).await?;
}
if !APP_DB_DATA_DIR.exists() {
fs::create_dir_all(APP_DB_DATA_DIR.as_path()).await?;
let db_path = Path::new(&app_conf.basic_config.db_path);
if !db_path.exists() {
fs::create_dir_all(db_path).await?;
}
Ok(())
}
pub async fn get_all_folders(paths: &Vec<PathBuf>) -> color_eyre::Result<Vec<PathBuf>> {
pub async fn get_all_folders(paths: Vec<&Path>) -> color_eyre::Result<Vec<PathBuf>> {
let mut folders: Vec<PathBuf> = Vec::new();
for path in paths {
let path = path.as_path();
let path = path.to_path_buf();
if !path.exists() {
return Err(eyre!("{:?} {}", path.blue(), "does not exist".red()));
}

4
src/lib.rs Normal file
View File

@@ -0,0 +1,4 @@
extern crate jemallocator;
#[global_allocator]
static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;

40
src/models/game.rs
View File

@@ -1,19 +1,19 @@
use ratatui::widgets::ListState;
use std::path::PathBuf;
use color_eyre::{eyre, Report};
use language_tags::LanguageTag;
use serde::{Deserialize, Serialize};
use crate::config::types::ApplicationConfig;
use crate::models::{RocksColumn, RocksReference, RocksReferences};
pub(crate) struct GameList<T> {
games: Vec<T>,
state: ListState,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub(crate) struct DLSiteManiax {
#[serde(skip)]
pub rj_num: String,
pub genre_ids: Vec<u16>,
pub name: String,
pub sells_count: u32
pub name: Vec<DLSiteTranslation>,
pub sells_count: u32,
pub folder_path: PathBuf,
pub version: Option<String>
}
impl From<crate::crawler::dlsite::DLSiteManiax> for DLSiteManiax {
@@ -21,8 +21,10 @@ impl From<crate::crawler::dlsite::DLSiteManiax> for DLSiteManiax {
Self {
rj_num: value.rj_num,
genre_ids: value.genre_ids,
name: value.title,
sells_count: value.sells_count
name: vec![],
sells_count: value.sells_count,
folder_path: value.folder_path,
version: None
}
}
}
@@ -105,3 +107,21 @@ impl RocksColumn for DLSiteCategory {
pub(crate) enum DLSiteTranslation {
EN(String), JP(String)
}
impl TryFrom<&str> for DLSiteTranslation {
type Error = Report;
fn try_from(value: &str) -> color_eyre::Result<Self> {
let app_conf = ApplicationConfig::get_config()?;
let locale = app_conf.basic_config.locale;
let en_locale = LanguageTag::parse("en-US")?;
if locale.matches(&en_locale) {
return Ok(DLSiteTranslation::EN(value.to_string()));
}
let jp_locale = LanguageTag::parse("ja-JP")?;
if locale.matches(&jp_locale) {
return Ok(DLSiteTranslation::JP(value.to_string()));
}
Err(eyre::eyre!("Invalid Locale: {:?}; Support {:?}", locale, [en_locale, jp_locale]))
}
}