fromost/sus-manager
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove diesel

Browse Source
This commit is contained in:
fromost
2025-10-19 23:25:56 +08:00
parent eacf897f8c
commit 5c466d37e9
13 changed files with 318 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/app.rs
View File

@@ -6,7 +6,6 @@ use crate::widgets::views::View;
use color_eyre::Result;
use crossterm::event::Event as CrosstermEvent;
use crossterm::event::{Event, KeyEvent};
use diesel::{Connection, SqliteConnection};
use rat_cursor::HasScreenCursor;
use ratatui::{DefaultTerminal, Frame};
use std::any::Any;
@@ -16,7 +15,6 @@ use crate::crawler::DLSITE_IMG_FOLDER;
pub(crate) struct App {
events: EventHandler,
db_connection: SqliteConnection,
state: AppState,
}
@@ -27,24 +25,16 @@ struct AppState {
impl App {
pub async fn create() -> Result<Self> {
let config = ApplicationConfig::get_config()?;
let db_conn = Self::establish_db_connection(&config);
let state = AppState {
view: Some(Box::new(MainView::new())),
};
let app = Self {
events: EventHandler::new(Duration::from_millis(config.basic_config.tick_rate)),
db_connection: db_conn,
state,
};
Ok(app)
}
fn establish_db_connection(application_config: &ApplicationConfig) -> SqliteConnection {
let database_url = application_config.clone().basic_config.db_path;
SqliteConnection::establish(&database_url)
.unwrap_or_else(|_| panic!("Error connecting to {}", database_url))
}
pub async fn run(mut self, terminal: &mut DefaultTerminal) -> Result<()> {
loop {
terminal.draw(|frame| self.draw(frame))?;

43
src/cli.rs
View File

@@ -3,9 +3,11 @@ use crate::config::types::ApplicationConfig;
use clap::{command, Args, Command, Parser, Subcommand};
use color_eyre::Result;
use ratatui::crossterm;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use color_eyre::eyre::eyre;
use colored::Colorize;
use crate::crawler::DLSiteCrawler;
use crate::crawler::dlsite;
// region Folder Command
#[derive(Parser, Debug)]
@@ -159,8 +161,45 @@ impl SyncSubCommand {
impl SyncDLSiteCommand {
pub async fn handle(&self) -> Result<()> {
let app_conf = ApplicationConfig::get_config()?;
Self::sync_genres(&app_conf).await?;
Self::sync_works(&app_conf).await?;
Ok(())
}
async fn sync_genres(app_conf: &ApplicationConfig) -> Result<()> {
Ok(())
}
async fn sync_works(app_conf: &ApplicationConfig) -> Result<()> {
let crawler = DLSiteCrawler::new();
crawler.get_game_info("RJ163319").await?;
let mut rj_nums: Vec<String> = Vec::new();
for path_str in app_conf.path_config.dlsite_paths.iter() {
let path = Path::new(path_str);
if !path.exists() {
return Err(eyre!("{} {}", path_str.blue(), "does not exist".red()));
}
let dir_paths = path.read_dir()?
.filter_map(Result::ok)
.map(|e| e.path())
.collect::<Vec<_>>();
for dir_path in dir_paths.iter() {
if !dir_path.is_dir() {
println!("{dir_path:?} is not a directory");
continue;
}
let dir_name = dir_path
.file_name().unwrap()
.to_str().unwrap();
if !dlsite::is_valid_rj_number(dir_name) {
println!("{} {}", dir_path.to_str().unwrap().blue(), "is not a valid rj number, please add it manually".red());
continue;
}
rj_nums.push(dir_name.to_string());
}
}
let maniaxes = crawler.get_game_infos(rj_nums).await?;
//TODO: save into db/probably change to use jsonb
Ok(())
}
}

106
src/crawler/dlsite.rs
View File

@@ -1,4 +1,4 @@
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use color_eyre::eyre::eyre;
use reqwest::Url;
@@ -6,11 +6,13 @@ use color_eyre::Result;
use lazy_static::lazy_static;
use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use crate::constants::APP_DATA_DIR;
use crate::constants::{APP_DATA_DIR};
use crate::crawler::Crawler;
//TODO: override locale with user one
const DLSITE_URL: &str = "https://www.dlsite.com/";
const DLSITE_API_ENDPOINT: &str = "/maniax/product/info/ajax";
const DLSITE_PRODUCT_API_ENDPOINT: &str = "/maniax/product/info/ajax";
const DLSITE_FILTER_OPTIONS_ENDPOINT: &str = "/maniax/fs/=/api_access/1/locale/ja_JP";
const DLSITE_MANIAX_PATH: &str = "/maniax/work/=/product_id/";
lazy_static! {
pub static ref DLSITE_IMG_FOLDER: PathBuf = APP_DATA_DIR.clone().join("dlsite").join("img");
@@ -23,7 +25,8 @@ pub struct DLSiteCrawler {
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct DLSiteManiax {
pub work_name: String,
#[serde(rename = "work_name")]
pub title: String,
#[serde(rename = "work_image")]
work_image_url: String,
#[serde(rename = "dl_count")]
@@ -39,51 +42,56 @@ impl DLSiteCrawler {
}
}
fn is_valid_number(rj_num: &str) -> bool {
let len = rj_num.len();
if len != 8 && len != 10 {
return false;
pub async fn get_game_infos(&self, rj_nums: Vec<String>) -> Result<Vec<DLSiteManiax>> {
let invalid_nums = rj_nums.iter()
.filter(|&n| !is_valid_rj_number(n))
.map(|n| n.to_string())
.collect::<Vec<String>>();
if !invalid_nums.is_empty() {
return Err(
eyre!("Invalid numbers: {}", invalid_nums.join(", "))
);
}
if !rj_num.starts_with("RJ") {
return false;
let query = &format!("product_id={}", rj_nums.join(","));
let (maniax_result, _) = self.crawler
.get_json::<HashMap<String, DLSiteManiax>>(DLSITE_PRODUCT_API_ENDPOINT, Some(query))
.await?;
let keys = maniax_result.keys()
.map(|k| k.to_string())
.collect::<Vec<String>>();
let keys_hash: HashSet<String> = HashSet::from_iter(keys);
let nums_hash: HashSet<String> = HashSet::from_iter(rj_nums);
let nums_diff = nums_hash.difference(&keys_hash)
.map(|n| n.to_string())
.collect::<Vec<String>>();
if !nums_diff.is_empty() {
return Err(eyre!("Restricted/Removed Works: {}", nums_diff.join(", ")));
}
if !rj_num.chars().skip(2).all(|c| c.is_numeric()) {
return false;
let mut maniax_infos = Vec::new();
for (rj_num, mut info) in maniax_result {
self.save_main_image(&info, &rj_num).await?;
let html_path = format!("{DLSITE_MANIAX_PATH}{rj_num}");
let (html, _) = self.crawler.get_html(&html_path).await?;
let genres = self.get_genres(&html)?;
info.genre_ids = genres;
maniax_infos.push(info);
}
true
}
pub async fn get_game_info(&self, rj_num: &str) -> Result<DLSiteManiax> {
if !Self::is_valid_number(rj_num) {
return Err(eyre!("Invalid number: {rj_num}"));
}
let mut api_url = self.crawler.base_url.clone();
api_url.set_path(DLSITE_API_ENDPOINT);
api_url.set_query(Some(&format!("product_id={rj_num}")));
let api_res = self.crawler.client.get(api_url).send().await?;
let maniax_result = match api_res.json::<HashMap<String, DLSiteManiax>>().await {
Ok(maniax_result) => maniax_result,
Err(_) => return Err(eyre!("Maniax {rj_num} is restricted/removed")),
};
let mut maniax_info = maniax_result.iter().next().unwrap().1.clone();
self.save_main_image(&maniax_info, rj_num).await?;
let html_path = format!("{DLSITE_MANIAX_PATH}{rj_num}");
let (html, _) = self.crawler.get_html(&html_path).await?;
let genres = self.get_genres(&html)?;
maniax_info.genre_ids = genres;
Ok(maniax_info)
Ok(maniax_infos)
}
async fn save_main_image(&self, info: &DLSiteManiax, rj_num: &str) -> Result<()> {
let img_file_name = format!("{rj_num}.jpg");
let img_save_path = DLSITE_IMG_FOLDER.clone().join(img_file_name);
if img_save_path.exists() {
return Ok(());
}
let url_string = format!("https:{}", info.work_image_url);
let url = Url::parse(&url_string)?;
let img_res = self.crawler.client.get(url).send().await?;
let img_bytes = img_res.bytes().await?;
let img = image::load_from_memory(&img_bytes)?;
img.save(DLSITE_IMG_FOLDER.clone().join(format!("{rj_num}.jpg")).as_path())?;
let (img, _) = self.crawler.get_img(&url).await?;
img.save(img_save_path)?;
Ok(())
}
@@ -96,7 +104,7 @@ impl DLSiteCrawler {
let result = html.select(&selector).next().unwrap();
let genre_row = result.child_elements()
.filter(|e|
e.child_elements().any(|e| e.inner_html() == "ジャンル")
e.child_elements().any(|e| e.inner_html() == "ジャンル") // TODO: will not work with english
).next().unwrap();
let data = genre_row
.child_elements().skip(1).next().unwrap()
@@ -114,4 +122,18 @@ impl DLSiteCrawler {
.collect::<Vec<_>>();
Ok(genre_ids)
}
}
pub fn is_valid_rj_number(rj_num: &str) -> bool {
let len = rj_num.len();
if len != 8 && len != 10 {
return false;
}
if !rj_num.starts_with("RJ") {
return false;
}
if !rj_num.chars().skip(2).all(|c| c.is_numeric()) {
return false;
}
true
}

39
src/crawler/mod.rs
View File

@@ -1,18 +1,20 @@
mod dlsite;
pub mod dlsite;
pub use dlsite::*;
use color_eyre::eyre::eyre;
use crate::constants::APP_CACHE_PATH;
use color_eyre::Result;
use image::DynamicImage;
use reqwest::{Client, StatusCode, Url};
use robotstxt::DefaultMatcher;
use scraper::Html;
use serde::de::DeserializeOwned;
#[derive(Clone)]
struct Crawler {
id: String,
pub(crate) base_url: Url,
pub(crate) client: Client,
client: Client,
robots_txt: Option<String>,
}
@@ -32,7 +34,7 @@ impl Crawler {
let is_access_allowed = matcher.one_agent_allowed_by_robots(
&self.get_robots_txt().await?,
"reqwest",
self.base_url.as_str(),
url.as_str(),
);
if !is_access_allowed {
return Err(eyre!("Crawler cannot access site {}", self.base_url.as_str()));
@@ -74,4 +76,35 @@ impl Crawler {
let html_text = &res.text().await?;
Ok((Html::parse_document(html_text), status))
}
pub async fn get_json<T>(&self, path: &str, query: Option<&str>) -> Result<(T, StatusCode)>
where T : DeserializeOwned {
let mut url = self.base_url.clone();
url.set_path(path);
url.set_query(query);
self.check_access(&url).await?;
let res = self.client.get(url).send().await?;
let status = res.status();
let json = res.json().await?;
Ok((json, status))
}
pub async fn get_img(&self, url: &Url) -> Result<(DynamicImage, StatusCode)> {
self.check_access(url).await?;
let res = self.client.get(url.clone()).send().await?;
let status = res.status();
let bytes = res.bytes().await?;
let img = image::load_from_memory(&bytes)?;
Ok((img, status))
}
pub async fn get_bytes(&self, path: &str) -> Result<(Vec<u8>, StatusCode)> {
let mut url = self.base_url.clone();
url.set_path(path);
self.check_access(&url).await?;
let res = self.client.get(url).send().await?;
let status = res.status();
let bytes = res.bytes().await?;
Ok((bytes.to_vec(), status))
}
}

1
src/main.rs
View File

@@ -5,7 +5,6 @@ mod constants;
mod crawler;
mod event;
mod helpers;
mod schema;
mod models;
mod widgets;

8
src/models/game.rs
View File

@@ -1,14 +1,6 @@
use diesel::{Queryable, Selectable};
use ratatui::widgets::ListState;
pub(crate) struct GameList<T> {
games: Vec<T>,
state: ListState,
}
#[derive(Queryable, Selectable)]
#[diesel(table_name = crate::schema::dl_games)]
#[diesel(check_for_backend(diesel::sqlite::Sqlite))]
pub(crate) struct DLSiteGame {
serial_number: String,
}

3
src/models/mod.rs
View File

@@ -1 +1,2 @@
pub mod game;
mod game;
pub use game::*;

7
src/schema.rs
View File

@@ -1,7 +0,0 @@
// @generated automatically by Diesel CLI.
diesel::table! {
dl_games (serial_number) {
serial_number -> Text,
}
}