fromost/sus-manager
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

add robots txt detection

Browse Source
This commit is contained in:
fromost
2025-10-09 15:26:36 +08:00
parent 10b89aee17
commit d660f25fb1
8 changed files with 1268 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.toml
View File

@@ -16,6 +16,9 @@ directories = "6.0.0"
lazy_static = "1.5.0"
dotenvy = "0.15"
rust-ini = "0.21.3"
robotstxt = "0.3.0"
scraper = "0.24.0"
reqwest = { version = "0.12.23", features = ["blocking"] }
[dependencies.crossterm]
version = "0.29.0"