change robots.txt to cache

src/config/mod.rs

@@ -1,7 +1,7 @@
 use std::path::{PathBuf};
 use ini::Ini;
 use crate::config::types::{ApplicationConfig, BasicConfig};
-use crate::constants::{APP_CONFIG_DIR, APP_CONIFG_FILE_PATH};
+use crate::constants::{APP_CONFIG_DIR, APP_CONIFG_FILE_PATH, APP_DATA_DIR};
 
 pub mod types;
 
@@ -21,7 +21,7 @@ impl ApplicationConfig {
     pub fn new() -> Self {
         let conf = Self {
             basic_config: BasicConfig {
-                db_path: APP_CONFIG_DIR.clone().join("games.db").to_str().unwrap().to_string(),
+                db_path: APP_DATA_DIR.clone().join("games.db").to_str().unwrap().to_string(),
                 tick_rate: 250
             }
         };

src/constants.rs

@@ -9,6 +9,8 @@ lazy_static!(
         .join(APP_DIR_NAME);
     pub static ref APP_DATA_DIR: PathBuf = BASE_DIRS.data_dir().to_path_buf()
         .join(APP_DIR_NAME);
+    pub static ref APP_CACHE_PATH: PathBuf = BASE_DIRS.cache_dir().to_path_buf()
+        .join(APP_DIR_NAME);
     pub static ref APP_CONIFG_FILE_PATH: PathBuf = APP_CONFIG_DIR.clone()
         .join("config.ini");
 );

src/crawler/mod.rs

@@ -1,9 +1,6 @@
-use std::fs;
-use reqwest::{Client, Url};
-use robotstxt::DefaultMatcher;
-use robotstxt::matcher::{LongestMatchRobotsMatchStrategy, RobotsMatcher};
-use crate::constants::APP_DATA_DIR;
-use crate::crawler;
+use reqwest::{Client, ClientBuilder, Url};
+use robotstxt::{DefaultMatcher, RobotsParseHandler};
+use crate::constants::{APP_CACHE_PATH, APP_DATA_DIR};
 
 pub(crate) struct Crawler {
     id: String,
@@ -27,7 +24,8 @@ impl Crawler {
     }
 
     async fn get_robots_txt(id: &str, base_url: &Url) -> String {
-        let local_robots_path = APP_DATA_DIR.clone().join(id).join("robots.txt");
+        let local_robots_path = APP_CACHE_PATH.clone()
+            .join(id).join("robots.txt");
         if !local_robots_path.exists() {
             let mut robots_url = base_url.clone();
             robots_url.set_path("/robots.txt");