From 4baf4639a3cc059f6d2566832747c9c51710279e Mon Sep 17 00:00:00 2001 From: tonymushah Date: Fri, 6 Oct 2023 08:45:18 +0300 Subject: [PATCH 1/4] Added cors handling --- Cargo.lock | 19 +++++++++++++++++-- Cargo.toml | 13 +++++++++---- .../special-eureka-downloader.rs} | 0 src/lib.rs | 2 +- src/server.rs | 5 ++++- 5 files changed, 31 insertions(+), 8 deletions(-) rename src/{main.rs => bin/special-eureka-downloader.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index 6caa559..4e554a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,6 +19,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "actix-cors" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b340e9cfa5b08690aae90fb61beb44e9b06f44fe3d0f93781aaa58cfba86245e" +dependencies = [ + "actix-utils", + "actix-web", + "derive_more", + "futures-util", + "log", + "once_cell", + "smallvec", +] + [[package]] name = "actix-http" version = "3.4.0" @@ -1090,8 +1105,9 @@ dependencies = [ [[package]] name = "mangadex-desktop-api2" -version = "0.5.6" +version = "0.5.7" dependencies = [ + "actix-cors", "actix-web", "anyhow", "async-stream", @@ -1104,7 +1120,6 @@ dependencies = [ "mangadex-api", "mangadex-api-schema-rust", "mangadex-api-types-rust", - "once_cell", "reqwest", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 0f287b3..4f25118 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangadex-desktop-api2" -version = "0.5.6" +version = "0.5.7" authors = ["tonymushah "] license = "MIT OR Apache-2.0" description = "A Actix server for downloading manga, chapters, covers from Mangadex" @@ -23,13 +23,12 @@ url = { version = "2.4.1", features = ["serde"] } mangadex-api = { version = "2.3.1", default-features = false, features = ["multi-thread", "utils", "deserializable-endpoint"] } anyhow = "1.0.75" serde_json = "1.0.107" -tokio = { version = "1.32.0", features = ["full"] } +tokio = { version = "1.32.0", features = ["sync", "rt", "macros", "rt-multi-thread"] } actix-web = { version = "4" } mangadex-api-schema-rust = { version = "0.4.1", default-features = false, features = ["serialize"] } mangadex-api-types-rust = { version = "0.4", default-features = false } log = "^0.4.20" -once_cell = "1.18.0" -fern = "0.6.2" +fern = {version = "0.6.2", optional = true} futures = { version = "0.3.28" } tokio-stream = { version = "0.1"} async-stream = { version = "0.3"} @@ -37,6 +36,12 @@ bytes = { version = "1.5"} itertools = "0.11.0" serde_qs = { version = "0", features = ["actix4"]} async-trait = "0.1" +actix-cors = "0.6.4" [features] default = [] +use_fern = ["dep:fern"] + +[[bin]] +name = "special-eureka-downloader" +required-features = ["use_fern"] \ No newline at end of file diff --git a/src/main.rs b/src/bin/special-eureka-downloader.rs similarity index 100% rename from src/main.rs rename to src/bin/special-eureka-downloader.rs diff --git a/src/lib.rs b/src/lib.rs index 3c3229a..f5ff4f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ use log::{info, warn}; use server::AppState; mod r#core; -pub use r#core::{Error, ErrorType}; +pub use crate::r#core::{Error, ErrorType}; pub mod download; #[cfg(feature = "feeds")] diff --git a/src/server.rs b/src/server.rs index a069bc1..f144358 100644 --- a/src/server.rs +++ b/src/server.rs @@ -16,6 +16,7 @@ use crate::methods::put::{ download_cover, download_cover_quality, download_manga_by_id, download_manga_cover, download_manga_cover_quality, download_manga_covers, }; +use actix_cors::Cors; use actix_web::body::MessageBody; use actix_web::dev::{self, Server, ServiceFactory, ServiceRequest, ServiceResponse}; use actix_web::http::header::{self}; @@ -93,10 +94,12 @@ pub fn get_actix_app( InitError = (), > + 'static, > { + let cors = Cors::default().allow_any_origin(); App::new() .app_data(app_state) .wrap(ErrorHandlers::new().handler(StatusCode::NOT_FOUND, not_found_message)) .wrap(ErrorHandlers::new().handler(StatusCode::METHOD_NOT_ALLOWED, not_allowed_message)) + .wrap(cors) /* get Methods */ @@ -152,6 +155,6 @@ pub fn launch_async_server( Ok( HttpServer::new(move || get_actix_app(app_state_ref.clone())) .bind((address, port))? - .run(), + .run() ) } From ab7abbd8cd937b933fb9ca8a415f735ba0a8c659 Mon Sep 17 00:00:00 2001 From: tonymushah Date: Fri, 6 Oct 2023 10:59:23 +0300 Subject: [PATCH 2/4] added benchmark and such --- Cargo.lock | 299 +++++++++++++++++- Cargo.toml | 20 +- benches/aggregate.rs | 45 +++ .../special-eureka-downloader.rs} | 0 src/lib.rs | 2 +- src/server.rs | 5 +- src/utils/manga.rs | 21 +- .../mod.rs} | 4 +- src/utils/manga_aggregate/stream.rs | 186 +++++++++++ 9 files changed, 563 insertions(+), 19 deletions(-) create mode 100644 benches/aggregate.rs rename src/{main.rs => bin/special-eureka-downloader.rs} (100%) rename src/utils/{manga_aggregate.rs => manga_aggregate/mod.rs} (98%) create mode 100644 src/utils/manga_aggregate/stream.rs diff --git a/Cargo.lock b/Cargo.lock index 6caa559..7db4994 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,6 +19,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "actix-cors" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b340e9cfa5b08690aae90fb61beb44e9b06f44fe3d0f93781aaa58cfba86245e" +dependencies = [ + "actix-utils", + "actix-web", + "derive_more", + "futures-util", + "log", + "once_cell", + "smallvec", +] + [[package]] name = "actix-http" version = "3.4.0" @@ -232,6 +247,18 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + [[package]] name = "anyhow" version = "1.0.75" @@ -361,6 +388,12 @@ dependencies = [ "bytes", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.83" @@ -377,6 +410,58 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "ciborium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" + +[[package]] +name = "ciborium-ll" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" + [[package]] name = "convert_case" version = "0.4.0" @@ -445,6 +530,77 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "futures", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -791,6 +947,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.12.3" @@ -941,6 +1103,26 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.11.0" @@ -1090,21 +1272,22 @@ dependencies = [ [[package]] name = "mangadex-desktop-api2" -version = "0.5.6" +version = "0.5.7" dependencies = [ + "actix-cors", "actix-web", "anyhow", "async-stream", "async-trait", "bytes", + "criterion", "fern", "futures", - "itertools", + "itertools 0.11.0", "log", "mangadex-api", "mangadex-api-schema-rust", "mangadex-api-types-rust", - "once_cell", "reqwest", "serde", "serde_json", @@ -1128,6 +1311,15 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -1183,6 +1375,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "num-traits" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -1217,6 +1418,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "openssl" version = "0.10.57" @@ -1314,6 +1521,34 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1408,6 +1643,26 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -1573,6 +1828,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.22" @@ -1835,6 +2099,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2034,6 +2308,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2160,6 +2444,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 0f287b3..628cca7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mangadex-desktop-api2" -version = "0.5.6" +version = "0.5.7" authors = ["tonymushah "] license = "MIT OR Apache-2.0" description = "A Actix server for downloading manga, chapters, covers from Mangadex" @@ -23,13 +23,12 @@ url = { version = "2.4.1", features = ["serde"] } mangadex-api = { version = "2.3.1", default-features = false, features = ["multi-thread", "utils", "deserializable-endpoint"] } anyhow = "1.0.75" serde_json = "1.0.107" -tokio = { version = "1.32.0", features = ["full"] } +tokio = { version = "1.32.0", features = ["sync", "rt", "macros", "rt-multi-thread"] } actix-web = { version = "4" } mangadex-api-schema-rust = { version = "0.4.1", default-features = false, features = ["serialize"] } mangadex-api-types-rust = { version = "0.4", default-features = false } log = "^0.4.20" -once_cell = "1.18.0" -fern = "0.6.2" +fern = {version = "0.6.2", optional = true} futures = { version = "0.3.28" } tokio-stream = { version = "0.1"} async-stream = { version = "0.3"} @@ -37,6 +36,19 @@ bytes = { version = "1.5"} itertools = "0.11.0" serde_qs = { version = "0", features = ["actix4"]} async-trait = "0.1" +actix-cors = "0.6.4" + +[dev-dependencies] +criterion = { version = "0.5", features = ["async_tokio"]} [features] default = [] +use_fern = ["dep:fern"] + +[[bin]] +name = "special-eureka-downloader" +required-features = ["use_fern"] + +[[bench]] +name = "aggregate" +harness = false diff --git a/benches/aggregate.rs b/benches/aggregate.rs new file mode 100644 index 0000000..8c48af0 --- /dev/null +++ b/benches/aggregate.rs @@ -0,0 +1,45 @@ +use std::sync::Arc; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use mangadex_desktop_api2::{settings::files_dirs::DirsOptions, utils::manga::MangaUtils}; + +async fn aggregate(manga_id: uuid::Uuid) { + let manga_utils = MangaUtils::new(Arc::new(DirsOptions::new().unwrap()), Default::default()) + .with_id(manga_id.to_string()); + serde_json::to_string(&(manga_utils.aggregate_manga_chapters().await.unwrap())).unwrap(); +} + +async fn aggregate_stream(manga_id: uuid::Uuid) { + let manga_utils = MangaUtils::new(Arc::new(DirsOptions::new().unwrap()), Default::default()) + .with_id(manga_id.to_string()); + serde_json::to_string( + &(manga_utils + .aggregate_manga_chapters_async_friendly() + .await + .unwrap()), + ) + .unwrap(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let manga_id = uuid::Uuid::try_parse("1c8f0358-d663-4d60-8590-b5e82890a1e3").unwrap(); + + c.bench_with_input( + BenchmarkId::new("aggregate", manga_id), + &manga_id, + |b, &s| { + b.to_async(&runtime).iter_batched(|| s, aggregate, criterion::BatchSize::LargeInput); + }, + ); + c.bench_with_input( + BenchmarkId::new("aggregate_stream", manga_id), + &manga_id, + |b, &s| { + b.to_async(&runtime).iter_batched(|| s, aggregate_stream, criterion::BatchSize::LargeInput); + }, + ); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/src/main.rs b/src/bin/special-eureka-downloader.rs similarity index 100% rename from src/main.rs rename to src/bin/special-eureka-downloader.rs diff --git a/src/lib.rs b/src/lib.rs index 3c3229a..f5ff4f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ use log::{info, warn}; use server::AppState; mod r#core; -pub use r#core::{Error, ErrorType}; +pub use crate::r#core::{Error, ErrorType}; pub mod download; #[cfg(feature = "feeds")] diff --git a/src/server.rs b/src/server.rs index a069bc1..f144358 100644 --- a/src/server.rs +++ b/src/server.rs @@ -16,6 +16,7 @@ use crate::methods::put::{ download_cover, download_cover_quality, download_manga_by_id, download_manga_cover, download_manga_cover_quality, download_manga_covers, }; +use actix_cors::Cors; use actix_web::body::MessageBody; use actix_web::dev::{self, Server, ServiceFactory, ServiceRequest, ServiceResponse}; use actix_web::http::header::{self}; @@ -93,10 +94,12 @@ pub fn get_actix_app( InitError = (), > + 'static, > { + let cors = Cors::default().allow_any_origin(); App::new() .app_data(app_state) .wrap(ErrorHandlers::new().handler(StatusCode::NOT_FOUND, not_found_message)) .wrap(ErrorHandlers::new().handler(StatusCode::METHOD_NOT_ALLOWED, not_allowed_message)) + .wrap(cors) /* get Methods */ @@ -152,6 +155,6 @@ pub fn launch_async_server( Ok( HttpServer::new(move || get_actix_app(app_state_ref.clone())) .bind((address, port))? - .run(), + .run() ) } diff --git a/src/utils/manga.rs b/src/utils/manga.rs index 9b7b912..013360e 100644 --- a/src/utils/manga.rs +++ b/src/utils/manga.rs @@ -41,15 +41,10 @@ impl<'a> MangaUtils { ) -> ManagerCoreResult { let chapter_utils: ChapterUtils = From::from(self); let chapter: ApiObject = chapter_utils.with_id(chap_id).get_chapter()?; - let mut is = false; - for relas in chapter.relationships { - if relas.type_ == RelationshipType::Manga - && relas.id.hyphenated().to_string() == manga_id - { - is = true; - } - } - Ok(is) + + Ok(chapter.relationships.iter().any(|relas| { + relas.type_ == RelationshipType::Manga && relas.id.hyphenated().to_string() == manga_id + })) } pub(self) fn find_all_downloades_by_manga_id( &'a self, @@ -489,6 +484,14 @@ impl MangaUtilsWithMangaId { volumes, }) } + pub async fn aggregate_manga_chapters_async_friendly(&self) -> ManagerCoreResult{ + let data = Box::pin(self.get_all_downloaded_chapter_data().await?); + let volumes = super::manga_aggregate::stream::group_chapter_to_volume_aggregate(data).await?; + Ok(MangaAggregate { + result: ResultType::Ok, + volumes, + }) + } } impl<'a> From<&'a MangaDownload> for MangaUtils { diff --git a/src/utils/manga_aggregate.rs b/src/utils/manga_aggregate/mod.rs similarity index 98% rename from src/utils/manga_aggregate.rs rename to src/utils/manga_aggregate/mod.rs index 5f3e5df..e822483 100644 --- a/src/utils/manga_aggregate.rs +++ b/src/utils/manga_aggregate/mod.rs @@ -8,7 +8,9 @@ use mangadex_api_schema_rust::{ ApiObject, }; -type ChapterHashMap = HashMap>>; +pub mod stream; + +pub type ChapterHashMap = HashMap>>; fn group_chapter_to_chapter_hash_map(input: Vec>) -> ChapterHashMap { let mut data: ChapterHashMap = ChapterHashMap::new(); diff --git a/src/utils/manga_aggregate/stream.rs b/src/utils/manga_aggregate/stream.rs new file mode 100644 index 0000000..135fdcc --- /dev/null +++ b/src/utils/manga_aggregate/stream.rs @@ -0,0 +1,186 @@ +use std::{cmp::Ordering, collections::HashMap, io::Result, vec}; + +use futures::Stream; +use mangadex_api_schema_rust::{ + v5::{ + manga_aggregate::{ChapterAggregate, VolumeAggregate}, + ChapterAttributes, + }, + ApiObject, +}; +use tokio_stream::StreamExt; + +pub type ChapterHashMap = HashMap>>; + +fn group_chapter_to_chapter_hash_map(input: Vec>) -> ChapterHashMap { + let mut data: ChapterHashMap = ChapterHashMap::new(); + for chap in input { + let chap_ = chap.clone(); + let volume = match chap.attributes.chapter { + None => "none".to_string(), + Some(d) => d, + }; + match data.get_mut(&volume) { + None => { + data.insert(volume, vec![chap_]); + } + Some(arr) => { + arr.push(chap_); + } + } + } + data +} + +fn chap_hashmapentry_to_chapter_aggregate( + input: (String, Vec>), +) -> Result { + let id = match input.1.get(0) { + None => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "The input vector is empty", + )); + } + Some(d) => d.clone(), + }; + let others: Vec = match input.1.get(1..(input.1.len())) { + None => Vec::new(), + Some(d) => d.iter().map(|d_| d_.id).collect(), + }; + let chapter: ChapterAggregate = ChapterAggregate { + chapter: input.0, + id: id.id, + others, + count: input.1.len() as u32, + }; + Ok(chapter) +} + +fn group_chapter_to_chapter_aggregate( + input: Vec>, +) -> Result> { + let data = group_chapter_to_chapter_hash_map(input); + let mut returns: Vec = Vec::new(); + for chunk in data { + returns.push(chap_hashmapentry_to_chapter_aggregate(chunk)?); + } + returns.sort_by(|a, b| { + let a_chp = match a.chapter.parse::() { + Ok(d) => d, + Err(_) => return Ordering::Equal, + }; + let b_chp = match b.chapter.parse::() { + Ok(d) => d, + Err(_) => return Ordering::Equal, + }; + a_chp.total_cmp(&b_chp) + }); + Ok(returns) +} + +fn chapter_volume_hashmap_entry_to_volume_aggregate( + (volume, chapters): (String, Vec>), +) -> Result { + let chapters = group_chapter_to_chapter_aggregate(chapters)?; + Ok(VolumeAggregate { + volume, + count: chapters.len() as u32, + chapters, + }) +} + +pub type ChapterVolumeHashMap = HashMap>>; +/// Convert an array of chapter to an HashMap with +/// - Key : volume number +/// - Value : The chapter Attributes +async fn group_chapter_to_volume_hash_map(mut input: I) -> Result +where + I: Stream> + Unpin, +{ + let mut data: ChapterVolumeHashMap = ChapterVolumeHashMap::new(); + while let Some(chap) = input.next().await { + let chap_ = chap.clone(); + let volume = match chap.attributes.volume { + None => "none".to_string(), + Some(d) => d, + }; + match data.get_mut(&volume) { + None => { + data.insert(volume, vec![chap_]); + } + Some(arr) => { + arr.push(chap_); + } + } + } + Ok(data) +} + +pub async fn group_chapter_to_volume_aggregate(input: I) -> Result> +where + I: Stream> + Unpin, +{ + let mut data: Vec = Vec::new(); + for in_ in group_chapter_to_volume_hash_map(input).await? { + data.push(chapter_volume_hashmap_entry_to_volume_aggregate(in_)?); + } + data.sort_by(|a, b| { + let a = match a.volume.parse::() { + Ok(d) => d, + Err(_) => return Ordering::Equal, + }; + let b = match b.volume.parse::() { + Ok(d) => d, + Err(_) => return Ordering::Equal, + }; + a.total_cmp(&b) + }); + Ok(data) +} + +/*pub fn chapter_vec_to_chapter_aggregate_vec(input : Vec>) -> Result<()> { + ChapterAggregate{ + + } + Ok(()) +}*/ + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use crate::{settings::files_dirs::DirsOptions, utils::manga::MangaUtils}; + + use super::*; + + #[tokio::test] + async fn test_to_volume_hash_map() { + let manga_id = "d58eb211-a1ae-426c-b504-fc88253de600".to_string(); + let manga_utils = + MangaUtils::new(Arc::new(DirsOptions::new().unwrap()), Default::default()); + let utils = manga_utils.with_id(manga_id); + let mut data = Box::pin(utils.get_all_downloaded_chapter_data().await.unwrap()); + + for (volume, chapters) in group_chapter_to_volume_hash_map(&mut data).await.unwrap() { + println!( + "\"{}\" : {}", + volume, + serde_json::to_string(&(group_chapter_to_chapter_aggregate(chapters).unwrap())) + .unwrap() + ); + } + } + #[tokio::test] + async fn test_to_volume_aggregate() { + let manga_id = "1c8f0358-d663-4d60-8590-b5e82890a1e3".to_string(); + let manga_utils = + MangaUtils::new(Arc::new(DirsOptions::new().unwrap()), Default::default()) + .with_id(manga_id); + println!( + "{}", + serde_json::to_string(&(manga_utils.aggregate_manga_chapters_async_friendly().await.unwrap())) + .unwrap() + ); + } +} From cad5f1efb621334802e36ce1f70398234c2d12c9 Mon Sep 17 00:00:00 2001 From: tonymushah Date: Fri, 6 Oct 2023 11:30:14 +0300 Subject: [PATCH 3/4] removed required-features --- Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bfdfd5f..9cf6b10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,5 +51,4 @@ required-features = ["use_fern"] [[bench]] name = "aggregate" -harness = false -required-features = ["use_fern"] +harness = false \ No newline at end of file From 6fec93174177798f264cb1b36130976c26bb869f Mon Sep 17 00:00:00 2001 From: tonymushah Date: Fri, 6 Oct 2023 11:39:48 +0300 Subject: [PATCH 4/4] send_wildcard added --- src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.rs b/src/server.rs index f144358..fdf75e1 100644 --- a/src/server.rs +++ b/src/server.rs @@ -94,7 +94,7 @@ pub fn get_actix_app( InitError = (), > + 'static, > { - let cors = Cors::default().allow_any_origin(); + let cors = Cors::default().allow_any_origin().send_wildcard(); App::new() .app_data(app_state) .wrap(ErrorHandlers::new().handler(StatusCode::NOT_FOUND, not_found_message))