fromost/sus-manager
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Change using Result

Browse Source
This commit is contained in:
fromost
2025-10-09 18:28:35 +08:00
parent 6f8a1eada1
commit 473ef8452b
3 changed files with 31 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/crawler/mod.rs
View File

@@ -1,7 +1,9 @@
use reqwest::{Client, ClientBuilder, Url};
use robotstxt::{DefaultMatcher, RobotsParseHandler};
use crate::constants::{APP_CACHE_PATH, APP_DATA_DIR};
use reqwest::{Client, Url};
use robotstxt::{DefaultMatcher};
use color_eyre::Result;
use crate::constants::{APP_CACHE_PATH};
#[derive(Clone)]
pub(crate) struct Crawler {
id: String,
base_url: Url,
@@ -14,7 +16,7 @@ impl Crawler {
let crawler = Self {
id: id.to_string(),
client: Client::new(),
robots_txt: Self::get_robots_txt(id, &base_url).await,
robots_txt: Self::get_robots_txt(id, &base_url).await.unwrap(),
base_url,
};
let mut matcher = DefaultMatcher::default();
@@ -23,7 +25,7 @@ impl Crawler {
crawler
}
async fn get_robots_txt(id: &str, base_url: &Url) -> String {
async fn get_robots_txt(id: &str, base_url: &Url) -> Result<String> {
let local_robots_path = APP_CACHE_PATH.clone()
.join(id).join("robots.txt");
if !local_robots_path.exists() {
@@ -31,13 +33,14 @@ impl Crawler {
robots_url.set_path("/robots.txt");
let response = reqwest::get(robots_url).await
.expect(format!("Failed to get robots.txt in `{}/robots.txt`", base_url.as_str()).as_str());
let content = response.text().await.unwrap();
tokio::fs::create_dir_all(local_robots_path.parent().unwrap()).await.unwrap();
tokio::fs::write(&local_robots_path, &content).await.unwrap();
content
let content = response.text().await?;
tokio::fs::create_dir_all(local_robots_path.parent().unwrap()).await?;
tokio::fs::write(&local_robots_path, &content).await?;
Ok(content)
}
else {
tokio::fs::read_to_string(&local_robots_path).await.unwrap()
Ok(tokio::fs::read_to_string(&local_robots_path).await?)
}
}
}