fromost/sus-manager
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Add dlsite cli add folder

Browse Source
This commit is contained in:
fromost
2025-10-10 23:57:29 +08:00
parent 473ef8452b
commit 4a0fa965c6
11 changed files with 443 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,5 +1,6 @@
target
.idea
games.db
*.env
diesel.toml
migrations/.*

216
Cargo.lock generated
View File

@@ -23,6 +23,56 @@ version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "anstream"
version = "0.6.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
"is_terminal_polyfill",
"utf8parse",
]
[[package]]
name = "anstyle"
version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
[[package]]
name = "anstyle-parse"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
dependencies = [
"windows-sys 0.60.2",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.60.2",
]
[[package]]
name = "async-stdin"
version = "0.3.1"
@@ -139,6 +189,46 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9"
[[package]]
name = "clap"
version = "4.5.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae"
dependencies = [
"clap_builder",
"clap_derive",
]
[[package]]
name = "clap_builder"
version = "4.5.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9"
dependencies = [
"anstream",
"anstyle",
"clap_lex",
"strsim",
]
[[package]]
name = "clap_derive"
version = "4.5.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "clap_lex"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
[[package]]
name = "color-eyre"
version = "0.6.5"
@@ -166,6 +256,12 @@ dependencies = [
"tracing-error",
]
[[package]]
name = "colorchoice"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
[[package]]
name = "compact_str"
version = "0.8.1"
@@ -511,12 +607,6 @@ dependencies = [
"litrs",
]
[[package]]
name = "dotenvy"
version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
[[package]]
name = "downcast-rs"
version = "2.0.2"
@@ -1150,6 +1240,12 @@ dependencies = [
"serde",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
[[package]]
name = "itertools"
version = "0.13.0"
@@ -1361,6 +1457,12 @@ version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "once_cell_polyfill"
version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
[[package]]
name = "openssl"
version = "0.10.73"
@@ -2114,11 +2216,11 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
name = "sus-manager"
version = "0.1.0"
dependencies = [
"clap",
"color-eyre",
"crossterm 0.29.0",
"diesel",
"directories",
"dotenvy",
"futures",
"lazy_static",
"libsqlite3-sys",
@@ -2531,6 +2633,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "utf8parse"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.18.1"
@@ -2750,7 +2858,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
@@ -2759,7 +2867,16 @@ version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.4",
]
[[package]]
@@ -2777,14 +2894,31 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d42b7b7f66d2a06854650af09cfdf8713e427a439c97ad65a6375318033ac4b"
dependencies = [
"windows-link 0.2.0",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
]
[[package]]
@@ -2793,48 +2927,96 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "wit-bindgen"
version = "0.46.0"

12
Cargo.toml
View File

@@ -1,7 +1,7 @@
[package]
name = "sus-manager"
version = "0.1.0"
description = "An example generated using the hello-world template"
description = "manager for SuS gamers"
authors = ["fromost"]
license = "MIT"
edition = "2024"
@@ -14,11 +14,17 @@ tokio-util = "0.7.9"
tokio-utils = "0.1.2"
directories = "6.0.0"
lazy_static = "1.5.0"
dotenvy = "0.15"
rust-ini = "0.21.3"
robotstxt = "0.3.0"
scraper = "0.24.0"
reqwest = { version = "0.12.23", features = ["blocking"] }
[dependencies.clap]
version = "4.5.48"
features = ["derive", "cargo"]
[dependencies.reqwest]
version = "0.12.23"
features = ["blocking"]
[dependencies.crossterm]
version = "0.29.0"

77
src/app.rs
View File

@@ -1,13 +1,12 @@
use std::path::PathBuf;
use crate::event::{Event, EventHandler};
use ratatui::widgets::{Block, Borders, Paragraph};
use ratatui::{DefaultTerminal};
use std::time::Duration;
use color_eyre::Result;
use crossterm::event;
use crossterm::event::KeyCode;
use crossterm::event::KeyCode::Char;
use diesel::{Connection, SqliteConnection};
use lazy_static::lazy_static;
use ratatui::buffer::Buffer;
use ratatui::layout::{Constraint, Direction, Layout, Rect};
use ratatui::prelude::{Widget};
@@ -16,11 +15,24 @@ use ratatui::text::{Line, Span, Text};
use crate::config::types::ApplicationConfig;
use crate::constants::{APP_CONFIG_DIR, APP_CONIFG_FILE_PATH, APP_DATA_DIR};
enum AppState {
Running,
Exiting,
Input
}
enum AppPopUp {
AddFolder,
None
}
pub(crate) struct App {
running: bool,
state: AppState,
popup: AppPopUp,
events: EventHandler,
db_connection: SqliteConnection,
app_config: ApplicationConfig,
current_event: Option<Event>
}
impl App {
@@ -31,10 +43,12 @@ impl App {
Self::initialize();
let db_conn = Self::establish_db_connection(app_conf.clone());
Self {
running: true,
state: AppState::Running,
popup: AppPopUp::None,
events: EventHandler::new(Duration::from_millis(app_conf.basic_config.tick_rate)),
db_connection: db_conn,
app_config: app_conf
app_config: app_conf,
current_event: None
}
}
@@ -58,16 +72,29 @@ impl App {
terminal.draw(|frame| frame.render_widget(&mut self, frame.area()))?;
let event = self.events.next().await?;
self.update(event)?;
if !self.running {
if matches!(self.state, AppState::Exiting) {
break Ok(())
}
}
}
fn update(&mut self, event: Event) -> Result<()> {
if let Event::Key(key) = event && event::KeyEventKind::is_press(&key.kind) {
self.current_event = Some(event.clone());
if let Event::Key(key) = event && matches!(self.state, AppState::Input) {
match key.code {
KeyCode::Esc => {
self.state = AppState::Running;
},
_ => {}
}
}
if let Event::Key(key) = event &&
event::KeyEventKind::is_press(&key.kind) &&
!matches!(self.state, AppState::Input) {
match key.code {
Char('q') => self.quit()?,
Char('a') => self.folder_popup(),
_ => {}
}
}
@@ -84,7 +111,7 @@ impl Widget for &mut App {
.constraints([
Constraint::Length(1),
Constraint::Min(1),
Constraint::Length(3),
Constraint::Length(1),
])
.split(area);
@@ -92,6 +119,17 @@ impl Widget for &mut App {
self.render_game_list(chunks[1], buf);
self.render_footer(chunks[2], buf);
let popup_area = Rect {
x: area.width / 4,
y: area.height / 3,
width: area.width / 2,
height: area.height / 3,
};
match self.popup {
AppPopUp::AddFolder => self.render_folder_popup(popup_area, buf),
AppPopUp::None => {},
};
}
}
@@ -117,23 +155,40 @@ impl App {
}
fn render_footer(&mut self, area: Rect, buf: &mut Buffer) {
let navigation_text = vec![
let mut navigation_text = vec![
Span::styled("(q) quit / (a) add folders", Style::default().fg(Color::Green)),
];
if matches!(self.state, AppState::Input) {
navigation_text[0] = Span::styled("Input Mode", Style::default().fg(Color::Green));
}
let line = Line::from(navigation_text);
let footer = Paragraph::new(line);
footer.render(area, buf);
}
fn render_folder_popup(&mut self, area: Rect, buf: &mut Buffer) {
todo!()
}
}
// event handlers
impl App {
fn quit(&mut self) -> Result<()> {
self.running = false;
if matches!(self.popup, AppPopUp::None) {
self.state = AppState::Exiting;
self.app_config
.clone()
.write_to_file(APP_CONIFG_FILE_PATH.to_path_buf())?;
.write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf())?;
}
else {
self.popup = AppPopUp::None;
}
Ok(())
}
fn folder_popup(&mut self) {
self.popup = AppPopUp::AddFolder;
self.state = AppState::Input;
}
}

114
src/cli.rs Normal file
View File

@@ -0,0 +1,114 @@
use std::path::PathBuf;
use clap::{command, Args, Command, Parser, Subcommand};
use color_eyre::Result;
use crate::app;
use crate::config::types::ApplicationConfig;
use crate::constants::APP_CONIFG_FILE_PATH;
#[derive(Parser, Debug)]
struct FolderAddCommand {
path: String
}
#[derive(Parser, Debug)]
enum FolderSubCommand {
Add(FolderAddCommand)
}
#[derive(Parser, Debug)]
struct FolderCommand {
#[command(subcommand)]
subcommand: FolderSubCommand
}
#[derive(Parser, Debug)]
enum CliSubCommand {
Folder(FolderCommand),
}
#[derive(Parser, Debug)]
#[command(version, about)]
pub(crate) struct Cli {
#[command(subcommand)]
subcommand: Option<CliSubCommand>
}
impl Subcommand for Cli {
fn augment_subcommands(cmd: Command) -> Command {
cmd.subcommand(FolderCommand::augment_args(Command::new("folder")))
.subcommand_required(true)
}
fn augment_subcommands_for_update(cmd: Command) -> Command {
cmd.subcommand(FolderCommand::augment_args(Command::new("folder")))
.subcommand_required(true)
}
fn has_subcommand(name: &str) -> bool {
matches!(name, "folder")
}
}
impl Subcommand for FolderCommand {
fn augment_subcommands(cmd: Command) -> Command {
cmd.subcommand(FolderAddCommand::augment_args(Command::new("add")))
.subcommand_required(true)
}
fn augment_subcommands_for_update(cmd: Command) -> Command {
cmd.subcommand(FolderAddCommand::augment_args(Command::new("add")))
.subcommand_required(true)
}
fn has_subcommand(name: &str) -> bool {
matches!(name, "add")
}
}
impl Cli {
pub async fn run(&self) -> Result<()> {
if self.subcommand.is_none() {
return self.start_tui().await
}
if let Some(sub_command) = &self.subcommand {
return sub_command.handle().await
}
Ok(())
}
async fn start_tui(&self) -> Result<()> {
let terminal = ratatui::init();
let app = app::App::new();
let result = app.run(terminal).await;
ratatui::restore();
result
}
}
impl CliSubCommand {
pub async fn handle(&self) -> Result<()> {
match self {
CliSubCommand::Folder(cmd) => {
cmd.subcommand.handle().await
}
}
}
}
impl FolderSubCommand {
pub async fn handle(&self) -> Result<()> {
match self {
FolderSubCommand::Add(cmd) => cmd.handle().await,
}
}
}
impl FolderAddCommand {
pub async fn handle(&self) -> Result<()> {
let mut config = ApplicationConfig::from_file(&APP_CONIFG_FILE_PATH.to_path_buf())?;
let path = PathBuf::from(&self.path);
let abs_path = path.canonicalize()?;
config.path_config.dlsite_paths.push(abs_path.to_str().unwrap().to_string());
config.write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf())
}
}

20
src/config/mod.rs
View File

@@ -1,7 +1,7 @@
use std::path::{PathBuf};
use ini::Ini;
use color_eyre::Result;
use crate::config::types::{ApplicationConfig, BasicConfig};
use crate::config::types::{ApplicationConfig, BasicConfig, PathConfig};
use crate::constants::{APP_CONIFG_FILE_PATH, APP_DATA_DIR};
pub mod types;
@@ -14,8 +14,14 @@ impl ApplicationConfig {
db_path: basic_conf_section.get("DBPath").unwrap().to_string(),
tick_rate: basic_conf_section.get("TickRate").unwrap().parse()?,
};
let path_conf_section = conf.section(Some("Path")).unwrap();
let path_conf = PathConfig {
dlsite_paths: path_conf_section.get("DLSite").unwrap()
.split(":").map(|s| s.to_string()).collect(),
};
Ok(Self {
basic_config: basic_conf
basic_config: basic_conf,
path_config: path_conf
})
}
@@ -24,17 +30,23 @@ impl ApplicationConfig {
basic_config: BasicConfig {
db_path: APP_DATA_DIR.clone().join("games.db").to_str().unwrap().to_string(),
tick_rate: 250
},
path_config: PathConfig {
dlsite_paths: vec![]
}
};
conf.clone().write_to_file(APP_CONIFG_FILE_PATH.to_path_buf()).unwrap();
conf.clone().write_to_file(&APP_CONIFG_FILE_PATH.to_path_buf()).unwrap();
conf
}
pub fn write_to_file(self, path: PathBuf) -> Result<()> {
pub fn write_to_file(self, path: &PathBuf) -> Result<()> {
let mut conf = Ini::new();
conf.with_section(Some("Basic"))
.set("DBPath", self.basic_config.db_path)
.set("TickRate", self.basic_config.tick_rate.to_string());
conf.with_section(Some("Path"))
.set("DLSite", self.path_config.dlsite_paths.into_iter()
.filter(|x| !x.is_empty()).collect::<Vec<String>>().join(":"));
conf.write_to_file(path)?;
Ok(())
}

6
src/config/types.rs
View File

@@ -1,6 +1,7 @@
#[derive(Clone)]
pub(crate) struct ApplicationConfig {
pub(crate) basic_config: BasicConfig,
pub(crate) path_config: PathConfig
}
#[derive(Clone)]
@@ -8,3 +9,8 @@ pub(crate) struct BasicConfig {
pub(crate) db_path: String,
pub(crate) tick_rate: u64
}
#[derive(Clone)]
pub(crate) struct PathConfig {
pub(crate) dlsite_paths: Vec<String>
}

10
src/crawler/dlsite.rs Normal file
View File

@@ -0,0 +1,10 @@
use crate::crawler::Crawler;
#[derive(Clone)]
pub(crate) struct DLSiteCrawler {
crawler: Crawler,
}
impl DLSiteCrawler {
}

16
src/crawler/mod.rs
View File

@@ -1,6 +1,9 @@
mod dlsite;
use reqwest::{Client, Url};
use robotstxt::{DefaultMatcher};
use color_eyre::Result;
use scraper::Html;
use crate::constants::{APP_CACHE_PATH};
#[derive(Clone)]
@@ -20,8 +23,10 @@ impl Crawler {
base_url,
};
let mut matcher = DefaultMatcher::default();
let access_allowed = matcher.one_agent_allowed_by_robots(&crawler.robots_txt, "reqwest", crawler.base_url.as_str());
assert_eq!(true, access_allowed);
let is_access_allowed = matcher.one_agent_allowed_by_robots(&crawler.robots_txt, "reqwest", crawler.base_url.as_str());
if !is_access_allowed {
panic!("Crawler cannot access site {}", crawler.base_url.as_str());
}
crawler
}
@@ -43,4 +48,11 @@ impl Crawler {
Ok(tokio::fs::read_to_string(&local_robots_path).await?)
}
}
pub async fn get_html(&self, path: &str) -> Result<Html> {
let mut url = self.base_url.clone();
url.set_path(path);
let html_text = &self.client.get(url).send().await?.text().await?;
Ok(Html::parse_document(html_text))
}
}

1
src/event.rs
View File

@@ -7,6 +7,7 @@ use std::time::Duration;
use tokio::sync::mpsc::UnboundedSender;
use tokio::task::JoinHandle;
#[derive(Clone)]
pub(crate) enum Event {
Error,
Tick,

16
src/main.rs
View File

@@ -6,22 +6,16 @@ mod config;
mod helpers;
mod crawler;
mod constants;
mod cli;
use clap::{command, Command, Parser};
use color_eyre::Result;
use reqwest::Url;
use tokio;
use crate::crawler::Crawler;
use crate::cli::Cli;
#[tokio::main]
async fn main() -> Result<()> {
color_eyre::install()?;
let crawler = Crawler::new("dlsite", Url::parse("https://www.dlsite.com/")?).await;
let terminal = ratatui::init();
let app = app::App::new();
let result = app.run(terminal).await;
ratatui::restore();
result
let cli = Cli::parse();
cli.run().await
}