add robots txt detection

src/constants.rs

@@ -0,0 +1,12 @@
+use std::path::PathBuf;
+use directories::BaseDirs;
+use lazy_static::lazy_static;
+
+const APP_DIR_NAME: &str = "sus_manager";
+lazy_static!(
+    static ref BASE_DIRS: BaseDirs = BaseDirs::new().unwrap();
+    pub static ref APP_CONFIG_DIR: PathBuf = BASE_DIRS.config_dir().to_path_buf()
+        .join(APP_DIR_NAME);
+    pub static ref APP_DATA_DIR: PathBuf = BASE_DIRS.data_dir().to_path_buf()
+        .join(APP_DIR_NAME);
+);