add robots txt detection

src/main.rs

@@ -4,13 +4,21 @@ mod schema;
 mod types;
 mod config;
 mod helpers;
+mod crawler;
+mod constants;
 
 use color_eyre::Result;
+use reqwest::Url;
 use tokio;
+use crate::crawler::Crawler;
 
 #[tokio::main]
 async fn main() -> Result<()> {
     color_eyre::install()?;
+
+    let crawler = Crawler::new("dlsite", Url::parse("https://www.dlsite.com/")?).await;
+
+
     let terminal = ratatui::init();
     let app = app::App::new();
     let result = app.run(terminal).await;