Add rocksdb
This commit is contained in:
5
Cargo.lock
generated
5
Cargo.lock
generated
@@ -2740,6 +2740,7 @@ dependencies = [
|
||||
"futures",
|
||||
"image",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"rat-cursor",
|
||||
"ratatui",
|
||||
"reqwest",
|
||||
@@ -2754,9 +2755,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.106"
|
||||
version = "2.0.108"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6"
|
||||
checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
@@ -17,7 +17,11 @@ rat-cursor = "1.2.1"
|
||||
serde_json = "1.0.145"
|
||||
image = "0.25.8"
|
||||
colored = "3.0.0"
|
||||
rocksdb = "0.24.0"
|
||||
log = "0.4.28"
|
||||
|
||||
[dependencies.rocksdb]
|
||||
version = "0.24.0"
|
||||
features = ["multi-threaded-cf"]
|
||||
|
||||
[dependencies.serde]
|
||||
version = "1.0.228"
|
||||
|
||||
16
src/app.rs
16
src/app.rs
@@ -1,5 +1,4 @@
|
||||
use crate::config::types::ApplicationConfig;
|
||||
use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR};
|
||||
use crate::event::{AppEvent, EventHandler};
|
||||
use crate::widgets::views::MainView;
|
||||
use crate::widgets::views::View;
|
||||
@@ -10,8 +9,6 @@ use rat_cursor::HasScreenCursor;
|
||||
use ratatui::{DefaultTerminal, Frame};
|
||||
use std::any::Any;
|
||||
use std::time::Duration;
|
||||
use tokio::fs;
|
||||
use crate::crawler::DLSITE_IMG_FOLDER;
|
||||
|
||||
pub(crate) struct App {
|
||||
events: EventHandler,
|
||||
@@ -93,16 +90,3 @@ impl App {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn initialize_folders() -> Result<()> {
|
||||
if !APP_CONFIG_DIR.exists() {
|
||||
fs::create_dir_all(APP_CONFIG_DIR.as_path()).await?;
|
||||
}
|
||||
if !APP_DATA_DIR.exists() {
|
||||
fs::create_dir_all(APP_DATA_DIR.as_path()).await?;
|
||||
}
|
||||
if !DLSITE_IMG_FOLDER.exists() {
|
||||
fs::create_dir_all(DLSITE_IMG_FOLDER.as_path()).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
223
src/cli.rs
223
src/cli.rs
@@ -1,223 +0,0 @@
|
||||
use crate::app;
|
||||
use crate::config::types::ApplicationConfig;
|
||||
use clap::{command, Args, Command, Parser, Subcommand};
|
||||
use color_eyre::Result;
|
||||
use ratatui::crossterm;
|
||||
use std::path::{Path, PathBuf};
|
||||
use color_eyre::eyre::eyre;
|
||||
use colored::Colorize;
|
||||
use crate::crawler::DLSiteCrawler;
|
||||
use crate::crawler::dlsite;
|
||||
|
||||
// region Folder Command
|
||||
#[derive(Parser, Debug)]
|
||||
struct FolderAddCommand {
|
||||
path: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
enum FolderSubCommand {
|
||||
Add(FolderAddCommand),
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
struct FolderCommand {
|
||||
#[command(subcommand)]
|
||||
subcommand: FolderSubCommand,
|
||||
}
|
||||
// endregion
|
||||
|
||||
// region Sync
|
||||
#[derive(Parser, Debug)]
|
||||
struct SyncCommand {
|
||||
#[command(subcommand)]
|
||||
subcommand: SyncSubCommand,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
enum SyncSubCommand {
|
||||
DLSite(SyncDLSiteCommand)
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
struct SyncDLSiteCommand;
|
||||
|
||||
// endregion
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
enum CliSubCommand {
|
||||
Folder(FolderCommand),
|
||||
Sync(SyncCommand),
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about)]
|
||||
pub(crate) struct Cli {
|
||||
#[command(subcommand)]
|
||||
subcommand: Option<CliSubCommand>,
|
||||
}
|
||||
|
||||
impl Subcommand for Cli {
|
||||
fn augment_subcommands(cmd: Command) -> Command {
|
||||
cmd.subcommand(FolderCommand::augment_args(Command::new("folder")))
|
||||
.subcommand_required(true)
|
||||
.subcommand(SyncCommand::augment_args(Command::new("sync")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn augment_subcommands_for_update(cmd: Command) -> Command {
|
||||
cmd.subcommand(FolderCommand::augment_args(Command::new("folder")))
|
||||
.subcommand_required(true)
|
||||
.subcommand(SyncCommand::augment_args(Command::new("sync")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn has_subcommand(name: &str) -> bool {
|
||||
matches!(name, "folder" | "sync")
|
||||
}
|
||||
}
|
||||
|
||||
impl Subcommand for FolderCommand {
|
||||
fn augment_subcommands(cmd: Command) -> Command {
|
||||
cmd.subcommand(FolderAddCommand::augment_args(Command::new("add")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn augment_subcommands_for_update(cmd: Command) -> Command {
|
||||
cmd.subcommand(FolderAddCommand::augment_args(Command::new("add")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn has_subcommand(name: &str) -> bool {
|
||||
matches!(name, "add")
|
||||
}
|
||||
}
|
||||
|
||||
impl Subcommand for SyncCommand {
|
||||
fn augment_subcommands(cmd: Command) -> Command {
|
||||
cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn augment_subcommands_for_update(cmd: Command) -> Command {
|
||||
cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn has_subcommand(name: &str) -> bool {
|
||||
matches!(name, "dlsite")
|
||||
}
|
||||
}
|
||||
|
||||
impl Cli {
|
||||
pub async fn run(&self) -> Result<()> {
|
||||
app::initialize_folders().await?;
|
||||
if self.subcommand.is_none() {
|
||||
return self.start_tui().await;
|
||||
}
|
||||
if let Some(sub_command) = &self.subcommand {
|
||||
return sub_command.handle().await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn start_tui(&self) -> Result<()> {
|
||||
crossterm::terminal::enable_raw_mode()?;
|
||||
|
||||
let mut terminal = ratatui::init();
|
||||
let app = app::App::create().await?;
|
||||
let result = app.run(&mut terminal).await;
|
||||
ratatui::restore();
|
||||
|
||||
crossterm::terminal::disable_raw_mode()?;
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl CliSubCommand {
|
||||
pub async fn handle(&self) -> Result<()> {
|
||||
match self {
|
||||
CliSubCommand::Folder(cmd) => cmd.subcommand.handle().await,
|
||||
CliSubCommand::Sync(cmd) => cmd.subcommand.handle().await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FolderSubCommand {
|
||||
pub async fn handle(&self) -> Result<()> {
|
||||
match self {
|
||||
FolderSubCommand::Add(cmd) => cmd.handle().await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SyncSubCommand {
|
||||
pub async fn handle(&self) -> Result<()> {
|
||||
match self {
|
||||
Self::DLSite(cmd) => cmd.handle().await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SyncDLSiteCommand {
|
||||
pub async fn handle(&self) -> Result<()> {
|
||||
let app_conf = ApplicationConfig::get_config()?;
|
||||
Self::sync_genres(&app_conf).await?;
|
||||
Self::sync_works(&app_conf).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn sync_genres(app_conf: &ApplicationConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn sync_works(app_conf: &ApplicationConfig) -> Result<()> {
|
||||
let crawler = DLSiteCrawler::new();
|
||||
let mut rj_nums: Vec<String> = Vec::new();
|
||||
for path_str in app_conf.path_config.dlsite_paths.iter() {
|
||||
let path = Path::new(path_str);
|
||||
if !path.exists() {
|
||||
return Err(eyre!("{} {}", path_str.blue(), "does not exist".red()));
|
||||
}
|
||||
let dir_paths = path.read_dir()?
|
||||
.filter_map(Result::ok)
|
||||
.map(|e| e.path())
|
||||
.collect::<Vec<_>>();
|
||||
for dir_path in dir_paths.iter() {
|
||||
if !dir_path.is_dir() {
|
||||
println!("{dir_path:?} is not a directory");
|
||||
continue;
|
||||
}
|
||||
let dir_name = dir_path
|
||||
.file_name().unwrap()
|
||||
.to_str().unwrap();
|
||||
if !dlsite::is_valid_rj_number(dir_name) {
|
||||
println!("{} {}", dir_path.to_str().unwrap().blue(), "is not a valid rj number, please add it manually".red());
|
||||
continue;
|
||||
}
|
||||
rj_nums.push(dir_name.to_string());
|
||||
}
|
||||
}
|
||||
let maniaxes = crawler.get_game_infos(rj_nums).await?;
|
||||
//TODO: save into db/probably change to use jsonb
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl FolderAddCommand {
|
||||
pub async fn handle(&self) -> Result<()> {
|
||||
let mut config = ApplicationConfig::get_config()?;
|
||||
let path = PathBuf::from(&self.path);
|
||||
let abs_path = path.canonicalize()?;
|
||||
if !abs_path.is_dir() {
|
||||
return Err(eyre!("{:?} is not a directory", abs_path));
|
||||
}
|
||||
config
|
||||
.path_config
|
||||
.dlsite_paths
|
||||
.push(abs_path.to_str().unwrap().to_string());
|
||||
config.save()?;
|
||||
println!("Added {:?} to path config", abs_path);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
65
src/cli/folder.rs
Normal file
65
src/cli/folder.rs
Normal file
@@ -0,0 +1,65 @@
|
||||
use std::path::PathBuf;
|
||||
use clap::{Args, Command, Parser, Subcommand};
|
||||
use color_eyre::eyre::eyre;
|
||||
use crate::config::types::ApplicationConfig;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub(super) struct FolderAddCommand {
|
||||
path: String,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub(super) enum FolderSubCommand {
|
||||
Add(FolderAddCommand),
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub(super) struct FolderCommand {
|
||||
#[command(subcommand)]
|
||||
pub(super) subcommand: FolderSubCommand,
|
||||
}
|
||||
|
||||
impl Subcommand for FolderCommand {
|
||||
fn augment_subcommands(cmd: Command) -> Command {
|
||||
cmd.subcommand(FolderAddCommand::augment_args(Command::new("add")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn augment_subcommands_for_update(cmd: Command) -> Command {
|
||||
cmd.subcommand(FolderAddCommand::augment_args(Command::new("add")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn has_subcommand(name: &str) -> bool {
|
||||
matches!(name, "add")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
impl FolderSubCommand {
|
||||
pub async fn handle(&self) -> color_eyre::Result<()> {
|
||||
match self {
|
||||
FolderSubCommand::Add(cmd) => cmd.handle().await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
impl FolderAddCommand {
|
||||
pub async fn handle(&self) -> color_eyre::Result<()> {
|
||||
let mut config = ApplicationConfig::get_config()?;
|
||||
let path = PathBuf::from(&self.path);
|
||||
let abs_path = path.canonicalize()?;
|
||||
if !abs_path.is_dir() {
|
||||
return Err(eyre!("{:?} is not a directory", abs_path));
|
||||
}
|
||||
config
|
||||
.path_config
|
||||
.dlsite_paths
|
||||
.push(abs_path.to_str().unwrap().to_string());
|
||||
config.save()?;
|
||||
println!("Added {:?} to path config", abs_path);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
76
src/cli/mod.rs
Normal file
76
src/cli/mod.rs
Normal file
@@ -0,0 +1,76 @@
|
||||
mod folder;
|
||||
mod sync;
|
||||
|
||||
use crate::{app, helpers};
|
||||
use clap::{command, Args, Command, Parser, Subcommand};
|
||||
use color_eyre::Result;
|
||||
use ratatui::crossterm;
|
||||
use crate::cli::folder::FolderCommand;
|
||||
use crate::cli::sync::SyncCommand;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
enum CliSubCommand {
|
||||
Folder(FolderCommand),
|
||||
Sync(SyncCommand),
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(version, about)]
|
||||
pub(crate) struct Cli {
|
||||
#[command(subcommand)]
|
||||
subcommand: Option<CliSubCommand>,
|
||||
}
|
||||
|
||||
impl Subcommand for Cli {
|
||||
fn augment_subcommands(cmd: Command) -> Command {
|
||||
cmd.subcommand(FolderCommand::augment_args(Command::new("folder")))
|
||||
.subcommand_required(true)
|
||||
.subcommand(SyncCommand::augment_args(Command::new("sync")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn augment_subcommands_for_update(cmd: Command) -> Command {
|
||||
cmd.subcommand(FolderCommand::augment_args(Command::new("folder")))
|
||||
.subcommand_required(true)
|
||||
.subcommand(SyncCommand::augment_args(Command::new("sync")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn has_subcommand(name: &str) -> bool {
|
||||
matches!(name, "folder" | "sync")
|
||||
}
|
||||
}
|
||||
|
||||
impl Cli {
|
||||
pub async fn run(&self) -> Result<()> {
|
||||
helpers::initialize_folders().await?;
|
||||
if self.subcommand.is_none() {
|
||||
return self.start_tui().await;
|
||||
}
|
||||
if let Some(sub_command) = &self.subcommand {
|
||||
return sub_command.handle().await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn start_tui(&self) -> Result<()> {
|
||||
crossterm::terminal::enable_raw_mode()?;
|
||||
|
||||
let mut terminal = ratatui::init();
|
||||
let app = app::App::create().await?;
|
||||
let result = app.run(&mut terminal).await;
|
||||
ratatui::restore();
|
||||
|
||||
crossterm::terminal::disable_raw_mode()?;
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl CliSubCommand {
|
||||
pub async fn handle(&self) -> Result<()> {
|
||||
match self {
|
||||
CliSubCommand::Folder(cmd) => cmd.subcommand.handle().await,
|
||||
CliSubCommand::Sync(cmd) => cmd.subcommand.handle().await,
|
||||
}
|
||||
}
|
||||
}
|
||||
94
src/cli/sync.rs
Normal file
94
src/cli/sync.rs
Normal file
@@ -0,0 +1,94 @@
|
||||
use std::path::Path;
|
||||
use clap::{Args, Command, Parser, Subcommand};
|
||||
use color_eyre::eyre::eyre;
|
||||
use color_eyre::eyre::Result;
|
||||
use colored::Colorize;
|
||||
use crate::config::types::ApplicationConfig;
|
||||
use crate::constants::{DB_CF_OPTIONS, DB_OPTIONS};
|
||||
use crate::crawler::{dlsite, DLSiteCrawler};
|
||||
use crate::helpers::db::RocksDB;
|
||||
use crate::models::DLSiteManiax;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub(super) struct SyncCommand {
|
||||
#[command(subcommand)]
|
||||
pub(super) subcommand: SyncSubCommand,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub(super) enum SyncSubCommand {
|
||||
DLSite(SyncDLSiteCommand)
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub(super) struct SyncDLSiteCommand;
|
||||
|
||||
impl Subcommand for SyncCommand {
|
||||
fn augment_subcommands(cmd: Command) -> Command {
|
||||
cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn augment_subcommands_for_update(cmd: Command) -> Command {
|
||||
cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite")))
|
||||
.subcommand_required(true)
|
||||
}
|
||||
|
||||
fn has_subcommand(name: &str) -> bool {
|
||||
matches!(name, "dlsite")
|
||||
}
|
||||
}
|
||||
|
||||
impl SyncSubCommand {
|
||||
pub async fn handle(&self) -> color_eyre::Result<()> {
|
||||
match self {
|
||||
Self::DLSite(cmd) => cmd.handle().await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SyncDLSiteCommand {
|
||||
pub async fn handle(&self) -> color_eyre::Result<()> {
|
||||
let app_conf = ApplicationConfig::get_config()?;
|
||||
let db = RocksDB::new(DB_OPTIONS.clone(), DB_CF_OPTIONS.clone())?;
|
||||
Self::sync_genres(&app_conf).await?;
|
||||
Self::sync_works(&app_conf, &db).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn sync_genres(app_conf: &ApplicationConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn sync_works(app_conf: &ApplicationConfig, db: &RocksDB) -> Result<()> {
|
||||
let crawler = DLSiteCrawler::new();
|
||||
let mut rj_nums: Vec<String> = Vec::new();
|
||||
for path_str in app_conf.path_config.dlsite_paths.iter() {
|
||||
let path = Path::new(path_str);
|
||||
if !path.exists() {
|
||||
return Err(eyre!("{} {}", path_str.blue(), "does not exist".red()));
|
||||
}
|
||||
let dir_paths = path.read_dir()?
|
||||
.filter_map(Result::ok)
|
||||
.map(|e| e.path())
|
||||
.collect::<Vec<_>>();
|
||||
for dir_path in dir_paths.iter() {
|
||||
if !dir_path.is_dir() {
|
||||
println!("{dir_path:?} is not a directory");
|
||||
continue;
|
||||
}
|
||||
let dir_name = dir_path
|
||||
.file_name().unwrap()
|
||||
.to_str().unwrap();
|
||||
if !dlsite::is_valid_rj_number(dir_name) {
|
||||
println!("{} {}", dir_path.to_str().unwrap().blue(), "is not a valid rj number, please add it manually".red());
|
||||
continue;
|
||||
}
|
||||
rj_nums.push(dir_name.to_string());
|
||||
}
|
||||
}
|
||||
let maniaxes = crawler.get_game_infos(rj_nums).await?;
|
||||
db.set_values(&maniaxes)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::config::types::{ApplicationConfig, BasicConfig, PathConfig};
|
||||
use crate::constants::{APP_CONIFG_FILE_PATH, APP_DATA_DIR};
|
||||
use crate::constants::{APP_CONIFG_FILE_PATH, APP_DB_DATA_DIR};
|
||||
use color_eyre::Result;
|
||||
use std::path::PathBuf;
|
||||
use serde_json;
|
||||
@@ -24,12 +24,7 @@ impl ApplicationConfig {
|
||||
fn new() -> Self {
|
||||
let conf = Self {
|
||||
basic_config: BasicConfig {
|
||||
db_path: APP_DATA_DIR
|
||||
.clone()
|
||||
.join("games.db")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
db_path: APP_DB_DATA_DIR.to_str().unwrap().to_string(),
|
||||
tick_rate: 250,
|
||||
},
|
||||
path_config: PathConfig {
|
||||
|
||||
@@ -14,5 +14,5 @@ pub(crate) struct BasicConfig {
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct PathConfig {
|
||||
pub dlsite_paths: Vec<String>,
|
||||
pub dlsite_paths: Vec<String>
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use directories::BaseDirs;
|
||||
use lazy_static::lazy_static;
|
||||
use std::path::PathBuf;
|
||||
use crate::config::types::ApplicationConfig;
|
||||
use crate::models::{DLSiteManiax, RocksColumn};
|
||||
|
||||
const APP_DIR_NAME: &str = "sus_manager";
|
||||
lazy_static! {
|
||||
@@ -11,4 +11,27 @@ lazy_static! {
|
||||
pub static ref APP_DATA_DIR: PathBuf = BASE_DIRS.data_dir().to_path_buf().join(APP_DIR_NAME);
|
||||
pub static ref APP_CACHE_PATH: PathBuf = BASE_DIRS.cache_dir().to_path_buf().join(APP_DIR_NAME);
|
||||
pub static ref APP_CONIFG_FILE_PATH: PathBuf = APP_CONFIG_DIR.clone().join("config.json");
|
||||
pub static ref APP_DB_DATA_DIR: PathBuf = APP_DATA_DIR.clone().join("db");
|
||||
|
||||
pub static ref DB_OPTIONS: rocksdb::Options = get_db_options();
|
||||
pub static ref DB_CF_OPTIONS: rocksdb::Options = get_cf_options();
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
pub static ref DB_COLUMNS: Vec<String> = vec![DLSiteManiax::get_column_name().to_string()];
|
||||
}
|
||||
|
||||
fn get_db_options() -> rocksdb::Options {
|
||||
let mut opts = rocksdb::Options::default();
|
||||
|
||||
opts.create_missing_column_families(true);
|
||||
opts.create_if_missing(true);
|
||||
|
||||
opts
|
||||
}
|
||||
|
||||
fn get_cf_options() -> rocksdb::Options {
|
||||
let opts = rocksdb::Options::default();
|
||||
|
||||
opts
|
||||
}
|
||||
@@ -3,11 +3,12 @@ use std::path::PathBuf;
|
||||
use color_eyre::eyre::eyre;
|
||||
use reqwest::Url;
|
||||
use color_eyre::Result;
|
||||
use colored::Colorize;
|
||||
use lazy_static::lazy_static;
|
||||
use scraper::{Html, Selector};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use crate::constants::{APP_DATA_DIR};
|
||||
use crate::crawler::Crawler;
|
||||
use crate::models::DLSiteManiax;
|
||||
|
||||
//TODO: override locale with user one
|
||||
const DLSITE_URL: &str = "https://www.dlsite.com/";
|
||||
@@ -23,18 +24,6 @@ pub struct DLSiteCrawler {
|
||||
crawler: Crawler,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct DLSiteManiax {
|
||||
#[serde(rename = "work_name")]
|
||||
pub title: String,
|
||||
#[serde(rename = "work_image")]
|
||||
work_image_url: String,
|
||||
#[serde(rename = "dl_count")]
|
||||
pub sells_count: u32,
|
||||
#[serde(skip)]
|
||||
pub genre_ids: Vec<u16>
|
||||
}
|
||||
|
||||
impl DLSiteCrawler {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
@@ -65,7 +54,7 @@ impl DLSiteCrawler {
|
||||
.map(|n| n.to_string())
|
||||
.collect::<Vec<String>>();
|
||||
if !nums_diff.is_empty() {
|
||||
return Err(eyre!("Restricted/Removed Works: {}", nums_diff.join(", ")));
|
||||
println!("Restricted/Removed Works: {}", nums_diff.join(", ").red());
|
||||
}
|
||||
|
||||
let mut maniax_infos = Vec::new();
|
||||
@@ -76,6 +65,7 @@ impl DLSiteCrawler {
|
||||
let (html, _) = self.crawler.get_html(&html_path).await?;
|
||||
let genres = self.get_genres(&html)?;
|
||||
info.genre_ids = genres;
|
||||
info.id = rj_num;
|
||||
maniax_infos.push(info);
|
||||
}
|
||||
Ok(maniax_infos)
|
||||
|
||||
90
src/helpers/db.rs
Normal file
90
src/helpers/db.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
use crate::constants::{APP_DB_DATA_DIR, DB_COLUMNS};
|
||||
use rocksdb::{ColumnFamilyDescriptor, IteratorMode, OptimisticTransactionDB, Options};
|
||||
use serde::{Serialize};
|
||||
use serde::de::DeserializeOwned;
|
||||
use crate::models::RocksColumn;
|
||||
|
||||
pub struct RocksDB {
|
||||
db: OptimisticTransactionDB,
|
||||
}
|
||||
|
||||
impl RocksDB {
|
||||
pub fn new(db_opts: Options, cf_opts: Options) -> color_eyre::Result<Self> {
|
||||
let cfs = DB_COLUMNS.iter()
|
||||
.map(|cf| ColumnFamilyDescriptor::new(cf.to_string(), cf_opts.clone()))
|
||||
.collect::<Vec<_>>();
|
||||
let db = OptimisticTransactionDB::open_cf_descriptors(
|
||||
&db_opts,
|
||||
APP_DB_DATA_DIR.as_path(),
|
||||
cfs
|
||||
)?;
|
||||
let rocks = Self {
|
||||
db
|
||||
};
|
||||
Ok(rocks)
|
||||
}
|
||||
|
||||
pub fn get_value<TValue, TColumn>(&self, id: TColumn::Id) -> color_eyre::Result<Option<TValue>>
|
||||
where TColumn: RocksColumn, TValue: DeserializeOwned
|
||||
{
|
||||
let cf = self.db.cf_handle(TColumn::get_column_name().as_str()).unwrap();
|
||||
let query_res = self.db.get_cf(&cf, serde_json::to_string(&id)?)?;
|
||||
if query_res.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(serde_json::from_slice(&query_res.unwrap())?))
|
||||
}
|
||||
|
||||
pub fn set_value<TColumn>(&self, value: &TColumn) -> color_eyre::Result<()>
|
||||
where TColumn: RocksColumn + Serialize
|
||||
{
|
||||
let cf = self.db.cf_handle(TColumn::get_column_name().as_str()).unwrap();
|
||||
self.db.put_cf(&cf, serde_json::to_string(&value.get_id())?, serde_json::to_string(value)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_values<TColumn>(&self, ids: &[TColumn::Id]) -> color_eyre::Result<Vec<TColumn>>
|
||||
where TColumn: RocksColumn + DeserializeOwned
|
||||
{
|
||||
let transaction = self.db.transaction();
|
||||
let cf = self.db.cf_handle(TColumn::get_column_name().as_str()).unwrap();
|
||||
let mut values = Vec::new();
|
||||
for id in ids {
|
||||
let query_res = transaction.get_cf(&cf, serde_json::to_string(id)?)?;
|
||||
if let Some(res) = query_res {
|
||||
let value = serde_json::from_slice(&res)?;
|
||||
values.push(value);
|
||||
}
|
||||
}
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
pub fn get_all_values<TColumn>(&self) -> color_eyre::Result<Vec<(TColumn::Id, TColumn)>>
|
||||
where TColumn: RocksColumn + DeserializeOwned
|
||||
{
|
||||
let cf = self.db.cf_handle(TColumn::get_column_name().as_str()).unwrap();
|
||||
let values = self.db.iterator_cf(&cf, IteratorMode::Start)
|
||||
.filter_map(|res| res.ok())
|
||||
.map(|(k, v)|
|
||||
(
|
||||
serde_json::from_slice::<TColumn::Id>(&k).unwrap(),
|
||||
serde_json::from_slice::<TColumn>(&v).unwrap()
|
||||
)
|
||||
)
|
||||
.collect::<Vec<_>>();
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
|
||||
pub fn set_values<TColumn>(&self, values: &[TColumn]) -> color_eyre::Result<()>
|
||||
where TColumn: RocksColumn + Serialize
|
||||
{
|
||||
let transaction = self.db.transaction();
|
||||
let cf = self.db.cf_handle(TColumn::get_column_name().as_str()).unwrap();
|
||||
for value in values {
|
||||
transaction.put_cf(&cf, serde_json::to_string(&value.get_id())?, serde_json::to_string(value)?)?;
|
||||
}
|
||||
transaction.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
pub mod db;
|
||||
|
||||
use tokio::fs;
|
||||
use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR, APP_DB_DATA_DIR};
|
||||
use crate::crawler::DLSITE_IMG_FOLDER;
|
||||
|
||||
|
||||
pub async fn initialize_folders() -> color_eyre::Result<()> {
|
||||
if !APP_CONFIG_DIR.exists() {
|
||||
fs::create_dir_all(APP_CONFIG_DIR.as_path()).await?;
|
||||
}
|
||||
if !APP_DATA_DIR.exists() {
|
||||
fs::create_dir_all(APP_DATA_DIR.as_path()).await?;
|
||||
}
|
||||
if !DLSITE_IMG_FOLDER.exists() {
|
||||
fs::create_dir_all(DLSITE_IMG_FOLDER.as_path()).await?;
|
||||
}
|
||||
if !APP_DB_DATA_DIR.exists() {
|
||||
fs::create_dir_all(APP_DB_DATA_DIR.as_path()).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,6 +1,34 @@
|
||||
use ratatui::widgets::ListState;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use crate::models::RocksColumn;
|
||||
|
||||
pub(crate) struct GameList<T> {
|
||||
games: Vec<T>,
|
||||
state: ListState,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub(crate) struct DLSiteManiax {
|
||||
#[serde(rename = "work_name")]
|
||||
pub title: String,
|
||||
#[serde(rename = "work_image")]
|
||||
pub work_image_url: String,
|
||||
#[serde(rename = "dl_count")]
|
||||
pub sells_count: u32,
|
||||
#[serde(skip)]
|
||||
pub genre_ids: Vec<u16>,
|
||||
#[serde(skip)]
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
impl RocksColumn for DLSiteManiax {
|
||||
type Id = String;
|
||||
|
||||
fn get_id(&self) -> Self::Id {
|
||||
self.id.clone()
|
||||
}
|
||||
|
||||
fn get_column_name() -> String {
|
||||
String::from("dl_games")
|
||||
}
|
||||
}
|
||||
@@ -1,2 +1,11 @@
|
||||
mod game;
|
||||
pub use game::*;
|
||||
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::Serialize;
|
||||
pub(crate) use game::*;
|
||||
|
||||
pub trait RocksColumn {
|
||||
type Id: Serialize + DeserializeOwned;
|
||||
fn get_id(&self) -> Self::Id;
|
||||
fn get_column_name() -> String;
|
||||
}
|
||||
Reference in New Issue
Block a user