From 27bee0cfde0759c161f468c4d38aef1a991893aa Mon Sep 17 00:00:00 2001 From: fromost Date: Sat, 18 Oct 2025 01:32:00 +0800 Subject: [PATCH] Add basic dlsite crawler Reformat application config --- Cargo.lock | 788 +++++++++++++++++++++++++++++---- Cargo.toml | 13 +- src/app.rs | 54 +-- src/cli.rs | 63 ++- src/config/mod.rs | 8 + src/config/types.rs | 14 +- src/constants.rs | 1 + src/crawler/dlsite.rs | 74 +++- src/crawler/mod.rs | 59 ++- src/widgets/views/main_view.rs | 14 +- 10 files changed, 922 insertions(+), 166 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d92f77b..96f827f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -74,24 +83,33 @@ dependencies = [ ] [[package]] -name = "async-stdin" -version = "0.3.1" +name = "anyhow" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1ff8b5d9b5ec29e0f49583ba71847b8c8888b67a8510133048a380903aa6822" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ - "tokio", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "async-throttle" -version = "0.3.2" +name = "arrayvec" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c99532de164435a0b91279e715bff4fa0d164643b409a67761907ffc210ee8f" -dependencies = [ - "backoff", - "dashmap", - "tokio", -] +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "atomic-waker" @@ -100,17 +118,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] -name = "backoff" -version = "0.4.0" +name = "autocfg" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "av1-grain" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3efb2ca85bc610acfa917b5aaa36f3fcbebed5b3182d7f877b02531c4b80c8" dependencies = [ - "futures-core", - "getrandom 0.2.16", - "instant", - "pin-project-lite", - "rand", - "tokio", + "anyhow", + "arrayvec", + "log", + "nom", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47c8fbc0f831f4519fe8b810b6a7a91410ec83031b8233f730a0480029f6a23f" +dependencies = [ + "arrayvec", ] [[package]] @@ -134,24 +167,54 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + [[package]] name = "bitflags" version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +[[package]] +name = "bitstream-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2" + +[[package]] +name = "built" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" + [[package]] name = "bumpalo" version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.10.1" @@ -180,9 +243,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d05d92f4b1fd76aad469d46cdd858ca761576082cd37df81416691e50199fb" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cfg-expr" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.3" @@ -256,6 +331,12 @@ dependencies = [ "tracing-error", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.4" @@ -302,10 +383,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "crossbeam-queue" -version = "0.3.12" +name = "crc32fast" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ "crossbeam-utils", ] @@ -360,6 +460,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "cssparser" version = "0.35.0" @@ -453,19 +559,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dashmap" -version = "5.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" -dependencies = [ - "cfg-if", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "deranged" version = "0.5.4" @@ -628,6 +721,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -644,6 +757,21 @@ dependencies = [ "windows-sys 0.61.1", ] +[[package]] +name = "exr" +version = "1.73.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + [[package]] name = "eyre" version = "0.6.12" @@ -660,12 +788,51 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fax" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab" +dependencies = [ + "fax_derive", +] + +[[package]] +name = "fax_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "find-msvc-tools" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0399f9d26e5191ce32c498bebd31e7a3ceabc2745f0ac54af3f335126c3f24b3" +[[package]] +name = "flate2" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -842,6 +1009,16 @@ dependencies = [ "wasi 0.14.7+wasi-0.2.4", ] +[[package]] +name = "gif" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae047235e33e2829703574b54fdec96bfbad892062d97fed2f76022287de61b" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.32.3" @@ -868,10 +1045,15 @@ dependencies = [ ] [[package]] -name = "hashbrown" -version = "0.14.5" +name = "half" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] [[package]] name = "hashbrown" @@ -1134,6 +1316,46 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "529feb3e6769d234375c4cf1ee2ce713682b8e76538cb13f9fc23e1400a591e7" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c5cedc30da3a610cac6b4ba17597bdf7152cf974e8aab3afb3d54455e371c8" + [[package]] name = "indenter" version = "0.3.4" @@ -1147,7 +1369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.15.5", + "hashbrown", ] [[package]] @@ -1170,12 +1392,14 @@ dependencies = [ ] [[package]] -name = "instant" -version = "0.1.13" +name = "interpolate_name" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ - "cfg-if", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1211,6 +1435,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -1226,6 +1459,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.81" @@ -1242,12 +1485,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + [[package]] name = "libc" version = "0.2.176" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" +[[package]] +name = "libfuzzer-sys" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5037190e1f70cbeef565bd267599242926f724d3b8a9f510fd7e0b540cfa4404" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libredox" version = "0.1.10" @@ -1264,7 +1523,6 @@ version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" dependencies = [ - "cc", "pkg-config", "vcpkg", ] @@ -1308,13 +1566,22 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "lru" version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.15.5", + "hashbrown", ] [[package]] @@ -1345,6 +1612,16 @@ dependencies = [ "syn", ] +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "memchr" version = "2.7.6" @@ -1357,6 +1634,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1364,6 +1647,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -1378,6 +1662,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "moxcms" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c588e11a3082784af229e23e8e4ecf5bcc6fbe4f69101e0421ce8d79da7f0b40" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -1401,12 +1695,78 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_threads" version = "0.1.7" @@ -1598,6 +1958,19 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "png" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97baced388464909d42d89643fe4361939af9b7ce7a31ee32a168f832a70f2a0" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "potential_utf" version = "0.1.3" @@ -1637,6 +2010,49 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "pxfm" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cbdf373972bf78df4d3b518d07003938e2c7d1fb5891e55f9cb6df57009d84" +dependencies = [ + "num-traits", +] + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.41" @@ -1700,7 +2116,7 @@ dependencies = [ "crossterm 0.28.1", "indoc", "instability", - "itertools", + "itertools 0.13.0", "lru", "paste", "strum", @@ -1710,6 +2126,76 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "rav1e" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" +dependencies = [ + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.12.1", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "once_cell", + "paste", + "profiling", + "rand", + "rand_chacha", + "simd_helpers", + "system-deps", + "thiserror 1.0.69", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.11.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5825c26fddd16ab9f515930d49028a630efec172e903483c94796cfe31893e6b" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.17" @@ -1727,7 +2213,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror", + "thiserror 2.0.17", ] [[package]] @@ -1772,6 +2258,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rgb" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" + [[package]] name = "ring" version = "0.17.14" @@ -1984,6 +2476,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2020,15 +2521,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "shutdown-async" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2799e69bde7e68bedd86c6d94bffa783219114f1f31435ddda61f4aeba348ff" -dependencies = [ - "tokio", -] - [[package]] name = "signal-hook" version = "0.3.18" @@ -2059,6 +2551,21 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "siphasher" version = "1.0.1" @@ -2095,7 +2602,7 @@ checksum = "9aead1c279716985b981b7940ef9b652d3f93d70a7296853c633b7ce8fa8088a" dependencies = [ "js-sys", "once_cell", - "thiserror", + "thiserror 2.0.17", "tokio", "wasm-bindgen", "wasm-bindgen-futures", @@ -2183,8 +2690,8 @@ dependencies = [ "diesel", "directories", "futures", + "image", "lazy_static", - "libsqlite3-sys", "rat-cursor", "ratatui", "reqwest", @@ -2193,10 +2700,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tokio-util", - "tokio-utils", "tui-input", - "uuid", ] [[package]] @@ -2251,6 +2755,25 @@ dependencies = [ "libc", ] +[[package]] +name = "system-deps" +version = "6.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + [[package]] name = "tempfile" version = "3.23.0" @@ -2275,13 +2798,33 @@ dependencies = [ "utf-8", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.17", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2304,6 +2847,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiff" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9605de7fee8d9551863fd692cce7637f548dbd9db9180fcc07ccc6d26c336f" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "time" version = "0.3.44" @@ -2412,15 +2969,37 @@ dependencies = [ ] [[package]] -name = "tokio-utils" -version = "0.1.2" +name = "toml" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de75f75f464153a50fe48b9675360e3cf2ae1d7d81f9751363bd2ee4888f5ce8" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ - "async-stdin", - "async-throttle", - "shutdown-async", - "tub", + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", ] [[package]] @@ -2515,16 +3094,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tub" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bca43faba247bc76eb1d6c1b8b561e4a1c5bdd427cc3d7a007faabea75c683a" -dependencies = [ - "crossbeam-queue", - "tokio", -] - [[package]] name = "tui-input" version = "0.14.0" @@ -2553,7 +3122,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" dependencies = [ - "itertools", + "itertools 0.13.0", "unicode-segmentation", "unicode-width 0.1.14", ] @@ -2607,13 +3176,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] -name = "uuid" -version = "1.18.1" +name = "v_frame" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" dependencies = [ - "getrandom 0.3.3", - "js-sys", + "aligned-vec", + "num-traits", "wasm-bindgen", ] @@ -2629,6 +3198,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version-compare" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b" + [[package]] name = "want" version = "0.3.1" @@ -2756,6 +3331,12 @@ dependencies = [ "string_cache_codegen", ] +[[package]] +name = "weezl" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" + [[package]] name = "winapi" version = "0.3.9" @@ -2984,6 +3565,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.46.0" @@ -3099,3 +3689,27 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index fa985f3..5dae21b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,14 +9,13 @@ edition = "2024" [dependencies] color-eyre = "0.6.3" futures = "0.3.28" -tokio-util = "0.7.9" -tokio-utils = "0.1.2" directories = "6.0.0" lazy_static = "1.5.0" robotstxt = "0.3.0" scraper = "0.24.0" rat-cursor = "1.2.1" serde_json = "1.0.145" +image = "0.25.8" [dependencies.serde] version = "1.0.228" @@ -41,7 +40,7 @@ features = ["derive", "cargo"] [dependencies.reqwest] version = "0.12.23" -features = ["blocking"] +features = ["blocking", "json"] [dependencies.tokio] version = "1.47.1" @@ -50,11 +49,3 @@ features = ["full"] [dependencies.diesel] version = "2.3.2" features = ["sqlite"] - -[dependencies.libsqlite3-sys] -version = "0.35.0" -features = ["bundled"] - -[dependencies.uuid] -version = "1.18.1" -features = ["v4"] diff --git a/src/app.rs b/src/app.rs index e646735..5bb88c5 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,5 +1,5 @@ use crate::config::types::ApplicationConfig; -use crate::constants::{APP_CONFIG_DIR, APP_CONIFG_FILE_PATH, APP_DATA_DIR}; +use crate::constants::{APP_CONFIG_DIR, APP_DATA_DIR}; use crate::event::{AppEvent, EventHandler}; use crate::widgets::views::MainView; use crate::widgets::views::View; @@ -11,11 +11,12 @@ use rat_cursor::HasScreenCursor; use ratatui::{DefaultTerminal, Frame}; use std::any::Any; use std::time::Duration; +use tokio::fs; +use crate::crawler::DLSITE_IMG_FOLDER; pub(crate) struct App { events: EventHandler, db_connection: SqliteConnection, - app_config: ApplicationConfig, state: AppState, } @@ -24,36 +25,22 @@ struct AppState { } impl App { - pub async fn create() -> Self { - let app_conf = if APP_CONIFG_FILE_PATH.exists() { - ApplicationConfig::from_file(&APP_CONIFG_FILE_PATH).unwrap() - } else { - ApplicationConfig::new() - }; - Self::initialize_folders(); - let db_conn = Self::establish_db_connection(app_conf.clone()); + pub async fn create() -> Result { + let config = ApplicationConfig::get_config()?; + let db_conn = Self::establish_db_connection(&config); let state = AppState { - view: Some(Box::new(MainView::new(&app_conf))), + view: Some(Box::new(MainView::new())), }; - Self { - events: EventHandler::new(Duration::from_millis(app_conf.basic_config.tick_rate)), + let app = Self { + events: EventHandler::new(Duration::from_millis(config.basic_config.tick_rate)), db_connection: db_conn, - app_config: app_conf, state, - } + }; + Ok(app) } - fn initialize_folders() { - if !APP_CONFIG_DIR.exists() { - std::fs::create_dir_all(APP_CONFIG_DIR.as_path()).unwrap(); - } - if !APP_DATA_DIR.exists() { - std::fs::create_dir_all(APP_DATA_DIR.as_path()).unwrap(); - } - } - - fn establish_db_connection(application_config: ApplicationConfig) -> SqliteConnection { - let database_url = application_config.basic_config.db_path; + fn establish_db_connection(application_config: &ApplicationConfig) -> SqliteConnection { + let database_url = application_config.clone().basic_config.db_path; SqliteConnection::establish(&database_url) .unwrap_or_else(|_| panic!("Error connecting to {}", database_url)) } @@ -105,7 +92,7 @@ impl App { if let Some(view) = self.state.view.as_mut() { if let Some(main_view) = view.downcast_mut::() { frame.render_stateful_widget( - MainView::new(&self.app_config), + MainView::new(), frame.area(), &mut main_view.state, ); @@ -116,3 +103,16 @@ impl App { } } } + +pub async fn initialize_folders() -> Result<()> { + if !APP_CONFIG_DIR.exists() { + fs::create_dir_all(APP_CONFIG_DIR.as_path()).await?; + } + if !APP_DATA_DIR.exists() { + fs::create_dir_all(APP_DATA_DIR.as_path()).await?; + } + if !DLSITE_IMG_FOLDER.exists() { + fs::create_dir_all(DLSITE_IMG_FOLDER.as_path()).await?; + } + Ok(()) +} diff --git a/src/cli.rs b/src/cli.rs index d6922cd..98dc1c3 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -6,7 +6,9 @@ use color_eyre::Result; use ratatui::crossterm; use std::path::PathBuf; use color_eyre::eyre::eyre; +use crate::crawler::DLSiteCrawler; +// region Folder Command #[derive(Parser, Debug)] struct FolderAddCommand { path: String, @@ -22,10 +24,29 @@ struct FolderCommand { #[command(subcommand)] subcommand: FolderSubCommand, } +// endregion + +// region Sync +#[derive(Parser, Debug)] +struct SyncCommand { + #[command(subcommand)] + subcommand: SyncSubCommand, +} + +#[derive(Parser, Debug)] +enum SyncSubCommand { + DLSite(SyncDLSiteCommand) +} + +#[derive(Parser, Debug)] +struct SyncDLSiteCommand; + +// endregion #[derive(Parser, Debug)] enum CliSubCommand { Folder(FolderCommand), + Sync(SyncCommand), } #[derive(Parser, Debug)] @@ -39,15 +60,19 @@ impl Subcommand for Cli { fn augment_subcommands(cmd: Command) -> Command { cmd.subcommand(FolderCommand::augment_args(Command::new("folder"))) .subcommand_required(true) + .subcommand(SyncCommand::augment_args(Command::new("sync"))) + .subcommand_required(true) } fn augment_subcommands_for_update(cmd: Command) -> Command { cmd.subcommand(FolderCommand::augment_args(Command::new("folder"))) .subcommand_required(true) + .subcommand(SyncCommand::augment_args(Command::new("sync"))) + .subcommand_required(true) } fn has_subcommand(name: &str) -> bool { - matches!(name, "folder") + matches!(name, "folder" | "sync") } } @@ -67,8 +92,25 @@ impl Subcommand for FolderCommand { } } +impl Subcommand for SyncCommand { + fn augment_subcommands(cmd: Command) -> Command { + cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite"))) + .subcommand_required(true) + } + + fn augment_subcommands_for_update(cmd: Command) -> Command { + cmd.subcommand(SyncDLSiteCommand::augment_args(Command::new("dlsite"))) + .subcommand_required(true) + } + + fn has_subcommand(name: &str) -> bool { + matches!(name, "dlsite") + } +} + impl Cli { pub async fn run(&self) -> Result<()> { + app::initialize_folders().await?; if self.subcommand.is_none() { return self.start_tui().await; } @@ -82,7 +124,7 @@ impl Cli { crossterm::terminal::enable_raw_mode()?; let mut terminal = ratatui::init(); - let app = app::App::create().await; + let app = app::App::create().await?; let result = app.run(&mut terminal).await; ratatui::restore(); @@ -95,6 +137,7 @@ impl CliSubCommand { pub async fn handle(&self) -> Result<()> { match self { CliSubCommand::Folder(cmd) => cmd.subcommand.handle().await, + CliSubCommand::Sync(cmd) => cmd.subcommand.handle().await, } } } @@ -107,6 +150,22 @@ impl FolderSubCommand { } } +impl SyncSubCommand { + pub async fn handle(&self) -> Result<()> { + match self { + Self::DLSite(cmd) => cmd.handle().await, + } + } +} + +impl SyncDLSiteCommand { + pub async fn handle(&self) -> Result<()> { + let crawler = DLSiteCrawler::new(); + crawler.get_game_info("RJ163319").await?; + Ok(()) + } +} + impl FolderAddCommand { pub async fn handle(&self) -> Result<()> { let mut config = ApplicationConfig::from_file(&APP_CONIFG_FILE_PATH.to_path_buf())?; diff --git a/src/config/mod.rs b/src/config/mod.rs index a3e3c5d..e15d764 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -7,6 +7,14 @@ use serde_json; pub mod types; impl ApplicationConfig { + pub fn get_config() -> Result { + if APP_CONIFG_FILE_PATH.exists() { + ApplicationConfig::from_file(&APP_CONIFG_FILE_PATH) + } else { + Ok(ApplicationConfig::new()) + } + } + pub fn from_file(path: &PathBuf) -> Result { let reader = std::fs::File::open(path)?; let result = serde_json::from_reader(reader)?; diff --git a/src/config/types.rs b/src/config/types.rs index f5cfe59..e2b7cb6 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -1,18 +1,18 @@ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct ApplicationConfig { - pub(crate) basic_config: BasicConfig, - pub(crate) path_config: PathConfig, +pub struct ApplicationConfig { + pub basic_config: BasicConfig, + pub path_config: PathConfig, } #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct BasicConfig { - pub(crate) db_path: String, - pub(crate) tick_rate: u64, + pub db_path: String, + pub tick_rate: u64, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct PathConfig { - pub(crate) dlsite_paths: Vec, +pub struct PathConfig { + pub dlsite_paths: Vec, } diff --git a/src/constants.rs b/src/constants.rs index 286bc98..b10950f 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -1,6 +1,7 @@ use directories::BaseDirs; use lazy_static::lazy_static; use std::path::PathBuf; +use crate::config::types::ApplicationConfig; const APP_DIR_NAME: &str = "sus_manager"; lazy_static! { diff --git a/src/crawler/dlsite.rs b/src/crawler/dlsite.rs index b791ca5..8adf557 100644 --- a/src/crawler/dlsite.rs +++ b/src/crawler/dlsite.rs @@ -1,8 +1,78 @@ +use std::collections::HashMap; +use std::path::PathBuf; +use color_eyre::eyre::eyre; +use reqwest::Url; +use color_eyre::Result; +use lazy_static::lazy_static; +use serde::{Deserialize, Serialize}; +use crate::constants::APP_DATA_DIR; use crate::crawler::Crawler; +const DLSITE_URL: &str = "https://www.dlsite.com/"; +const DLSITE_API_ENDPOINT: &str = "/maniax/product/info/ajax"; +lazy_static! { + pub static ref DLSITE_IMG_FOLDER: PathBuf = APP_DATA_DIR.clone().join("dlsite").join("img"); +} + #[derive(Clone)] -pub(crate) struct DLSiteCrawler { +pub struct DLSiteCrawler { crawler: Crawler, } -impl DLSiteCrawler {} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DLSiteManiax { + pub work_name: String, + #[serde(rename = "work_image")] + work_image_url: String, + #[serde(rename = "dl_count")] + pub sells_count: u32 +} + +impl DLSiteCrawler { + pub fn new() -> Self { + Self { + crawler: Crawler::new("DLSite", Url::parse(DLSITE_URL).unwrap()) + } + } + + fn is_valid_number(rj_num: &str) -> bool { + let len = rj_num.len(); + if len != 8 && len != 10 { + return false; + } + if !rj_num.starts_with("RJ") { + return false; + } + if !rj_num.chars().skip(2).all(|c| c.is_numeric()) { + return false; + } + true + } + + pub async fn get_game_info(&self, rj_num: &str) -> Result { + if !Self::is_valid_number(rj_num) { + return Err(eyre!("Invalid number: {}", rj_num)); + } + let mut api_url = self.crawler.base_url.clone(); + api_url.set_path(DLSITE_API_ENDPOINT); + api_url.set_query(Some(&format!("product_id={}", rj_num))); + let res = self.crawler.client.get(api_url).send().await?; + let maniax_result = match res.json::>().await { + Ok(maniax_result) => maniax_result, + Err(_) => return Err(eyre!("Maniax {} is restricted/removed", rj_num)), + }; + let maniax_info = maniax_result.iter().next().unwrap().1.clone(); + self.save_main_image(&maniax_info, rj_num).await?; + Ok(maniax_info) + } + + async fn save_main_image(&self, info: &DLSiteManiax, rj_num: &str) -> Result<()> { + let url_string = format!("https:{}", info.work_image_url); + let url = Url::parse(&url_string)?; + let img_res = self.crawler.client.get(url).send().await?; + let img_bytes = img_res.bytes().await?; + let img = image::load_from_memory(&img_bytes)?; + img.save(DLSITE_IMG_FOLDER.clone().join(format!("{}.jpg", rj_num)).as_path())?; + Ok(()) + } +} \ No newline at end of file diff --git a/src/crawler/mod.rs b/src/crawler/mod.rs index 24625b7..5de643f 100644 --- a/src/crawler/mod.rs +++ b/src/crawler/mod.rs @@ -1,48 +1,58 @@ mod dlsite; +pub use dlsite::*; +use color_eyre::eyre::eyre; use crate::constants::APP_CACHE_PATH; use color_eyre::Result; -use reqwest::{Client, Url}; +use reqwest::{Client, StatusCode, Url}; use robotstxt::DefaultMatcher; use scraper::Html; #[derive(Clone)] -pub(crate) struct Crawler { +struct Crawler { id: String, - base_url: Url, - client: Client, - robots_txt: String, + pub(crate) base_url: Url, + pub(crate) client: Client, + robots_txt: Option, } impl Crawler { - pub async fn new(id: &str, base_url: Url) -> Self { + pub fn new(id: &str, base_url: Url) -> Self { let crawler = Self { id: id.to_string(), client: Client::new(), - robots_txt: Self::get_robots_txt(id, &base_url).await.unwrap(), + robots_txt: None, base_url, }; - let mut matcher = DefaultMatcher::default(); - let is_access_allowed = matcher.one_agent_allowed_by_robots( - &crawler.robots_txt, - "reqwest", - crawler.base_url.as_str(), - ); - if !is_access_allowed { - panic!("Crawler cannot access site {}", crawler.base_url.as_str()); - } crawler } - async fn get_robots_txt(id: &str, base_url: &Url) -> Result { - let local_robots_path = APP_CACHE_PATH.clone().join(id).join("robots.txt"); + async fn check_access(&self, url: &Url) -> Result<()> { + let mut matcher = DefaultMatcher::default(); + let is_access_allowed = matcher.one_agent_allowed_by_robots( + &self.get_robots_txt().await?, + "reqwest", + self.base_url.as_str(), + ); + if !is_access_allowed { + return Err(eyre!("Crawler cannot access site {}", self.base_url.as_str())); + } + Ok(()) + } + + async fn get_robots_txt(&self) -> Result { + if let Some(txt) = &self.robots_txt { + return Ok(txt.clone()); + } + + let local_robots_path = APP_CACHE_PATH.clone().join(&self.id).join("robots.txt"); if !local_robots_path.exists() { - let mut robots_url = base_url.clone(); + let mut robots_url = self.base_url.clone(); robots_url.set_path("/robots.txt"); let response = reqwest::get(robots_url).await.expect( format!( "Failed to get robots.txt in `{}/robots.txt`", - base_url.as_str() + self.base_url.as_str() ) .as_str(), ); @@ -55,10 +65,13 @@ impl Crawler { } } - pub async fn get_html(&self, path: &str) -> Result { + pub async fn get_html(&self, path: &str) -> Result<(Html, StatusCode)> { let mut url = self.base_url.clone(); + self.check_access(&url).await?; url.set_path(path); - let html_text = &self.client.get(url).send().await?.text().await?; - Ok(Html::parse_document(html_text)) + let res = self.client.get(url).send().await?; + let status = res.status(); + let html_text = &res.text().await?; + Ok((Html::parse_document(html_text), status)) } } diff --git a/src/widgets/views/main_view.rs b/src/widgets/views/main_view.rs index abd5566..1f0d7dd 100644 --- a/src/widgets/views/main_view.rs +++ b/src/widgets/views/main_view.rs @@ -11,7 +11,6 @@ use ratatui::widgets::{Block, Borders, Paragraph, StatefulWidget}; use std::any::Any; pub struct MainView { - app_config: ApplicationConfig, pub state: MainViewState, } @@ -29,20 +28,19 @@ enum Status { } impl MainView { - pub fn new(app_conf: &ApplicationConfig) -> Self { + pub fn new() -> Self { Self { state: MainViewState { popup: None, status: Status::Running, - }, - app_config: app_conf.clone(), + } } } fn quit(&mut self) -> color_eyre::Result<()> { if self.state.popup.is_none() { self.state.status = Status::Exiting; - self.app_config.save()?; + ApplicationConfig::get_config()?.save()?; } Ok(()) } @@ -73,9 +71,11 @@ impl View for MainView { let Some(value) = popup.get_folder_value() && key.code.is_enter() { - self.app_config.path_config.dlsite_paths.push(value); + let mut config = ApplicationConfig::get_config()?; + config.path_config.dlsite_paths.push(value); + popup.textarea.reset_value()?; - self.app_config.save()?; + config.save()?; } if !matches!(self.state.status, Status::Popup) && matches!(key.kind, KeyEventKind::Press) { match key.code {