[package]
name = "crawler"
version = "0.1.0"
edition = "2024"

[dependencies]
image = "0.25.9"
reqwest = { version = "0.12.25", features = ["json", "rustls-tls"] }
scraper = "0.25.0"
robotstxt = "0.3.0"

models = { path = "../models" }

tokio.workspace = true
serde.workspace = true
color-eyre.workspace = true
lazy_static.workspace = true
serde_json.workspace = true

futures = "0.3.31"
itertools = "0.14.0"
language-tags = "0.3.2"