From d631985588e7702c6f9a21f122b39beb00588f91 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Thu, 11 Jan 2024 19:37:29 +0700 Subject: [PATCH 01/32] Implement basic stuff --- Cargo.lock | 1804 ++++++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 10 +- src/main.rs | 231 ++++++- 3 files changed, 2041 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c325b82..76d00b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,1808 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aes" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "ahash" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c79fed4cdb43e993fcdadc7e58a09fd0e3e649c4436fa11da71c9f1f3ee7feb9" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "jobserver", + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.48.5", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e92c5c1a78c62968ec57dbc2440366a2d6e5a23faf829970ff1585dc6b18e2" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4323769dc8a61e2c39ad7dc26f6f2800524691a44d74fe3d1071a5c24db6370" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "duration-string" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fcc1d9ae294a15ed05aeae8e11ee5f2b3fe971c077d45a42fb20825fba6ee13" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "h2" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "jobserver" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" + +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl" +version = "0.10.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.48.5", +] + +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro2" +version = "1.0.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +dependencies = [ + "unicode-ident", +] + [[package]] name = "quicksync-rs" -version = "0.0.0" +version = "0.1.0" +dependencies = [ + "chrono", + "clap", + "duration-string", + "futures-util", + "reqwest", + "rusqlite", + "tokio", + "zip", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "reqwest" +version = "0.11.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg", +] + +[[package]] +name = "rusqlite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" +dependencies = [ + "bitflags 2.4.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.38.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "ryu" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.195" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.195" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "2.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "time" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +dependencies = [ + "deranged", + "powerfmt", + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.35.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-bidi" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" + +[[package]] +name = "wasm-streams" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "time", + "zstd", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.9+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 4268b5b..36179c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,16 @@ [package] name = "quicksync-rs" -version = "0.0.0" +version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +chrono = "0.4.31" +clap = { version="4.4.14", features=["derive"] } +duration-string = "0.3.0" +futures-util = "0.3.30" +reqwest = { version = "0.11.23", features = ["json", "stream"] } +rusqlite = "0.30.0" +tokio = { version = "1.35.1", features = ["full"] } +zip = "0.6.6" diff --git a/src/main.rs b/src/main.rs index a8c8771..bfae03b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,232 @@ +use chrono::{DateTime, Utc, Duration}; +use clap::{Parser, Subcommand}; +use duration_string::DurationString; +use reqwest::Client; +use rusqlite::{Connection, params}; +use zip::ZipArchive; +use std::env; +use std::fs::{OpenOptions, create_dir_all}; +use std::io::{Seek, SeekFrom, Write}; +use std::process::Command; +use std::path::{PathBuf, Path}; use std::error::Error; +use futures_util::StreamExt; -fn main() { - println!("Hello world"); +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Cli { + #[clap(subcommand)] + command: Commands, +} + +#[derive(Subcommand, Debug)] +enum Commands { + /// Checks if quicksync is recommended + Check { + /// Path to the node-data directory + #[clap(short, long)] + node_data: String, + /// Genesis time in ISO format + #[clap(short, long, default_value = "2023-07-14T08:00:00Z")] + genesis_time: String, + /// Layer duration + #[clap(short, long, default_value = "5m")] + layer_duration: String, + }, + /// Downloads latest db from official website + Download { + /// Path to the node-data directory + #[clap(short, long)] + node_data: String, + /// Path to go-spacemesh binary + #[clap(short, long, default_value = go_spacemesh_default_path())] + go_spacemesh_path: String, + /// URL to download from (without version at the end). Default: http://localhost:8080/ + #[clap(short, long, default_value = "http://localhost:8080/")] + download_url: String, + }, +} + +// Функция для определения пути по умолчанию в зависимости от ОС +fn go_spacemesh_default_path() -> &'static str { + #[cfg(target_os = "windows")] + { + "./go-spacemesh.exe" + } + #[cfg(not(target_os = "windows"))] + { + "./go-spacemesh" + } +} + +fn parse_iso_date(iso_date: &str) -> Result, chrono::ParseError> { + iso_date.parse::>() +} + +async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> Result<(), Box> { + let path = Path::new(file_path); + + if let Some(dir) = path.parent() { + // Пытаемся создать директорию + create_dir_all(dir).expect("Cannot create directory"); + } + + let mut file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(file_path) + .expect("Cannot create file"); + + let file_size = file.metadata()?.len(); + + let client = Client::new(); + let response = client.get(url) + .header("Range", format!("bytes={}-", file_size)) + .send() + .await?; + let final_url = response.url().clone(); + + std::fs::write(redirect_path, final_url.as_str())?; + + if response.status().is_success() { + file.seek(SeekFrom::End(0))?; + let mut stream = response.bytes_stream(); + + while let Some(item) = stream.next().await { + let chunk = item?; + file.write_all(&chunk)?; + } + } else { + println!("Cannot resume downloading: {:?}", response.status()); + } + + Ok(()) +} + +fn get_go_spacemesh_version(path: &str) -> Result> { + let output = Command::new(path) + .arg("version") + .output() + .expect("Cannot run go-spacemesh version"); + + let version = String::from_utf8(output.stdout)? + .trim() + .to_string(); + + Ok(version) +} + +fn resolve_path(relative_path: &str) -> Result> { + let current_dir = env::current_dir()?; + let resolved_path = current_dir.join(relative_path); + Ok(resolved_path) +} + +fn get_last_layer_from_db(db_path: &str) -> Result> { + let conn = Connection::open(db_path)?; + + let mut stmt = conn.prepare("SELECT * FROM layers ORDER BY id DESC LIMIT 1")?; + let mut layer_iter = stmt.query_map(params![], |row| { + Ok(row.get::<_, i32>(0)?) + })?; + + if let Some(result) = layer_iter.next() { + let last_id = result?; + Ok(last_id) + } else { + Ok(0) + } +} + +fn calculate_latest_layer(genesis_time: String, layer_duration: String) -> Result> { + let genesis = parse_iso_date(&genesis_time)?; + let delta = Utc::now() - genesis; + let dur = Duration::from_std(DurationString::from_string(layer_duration)?.into())?; + Ok(delta.num_milliseconds() / dur.num_milliseconds()) +} + +fn unzip_state_sql(archive_path: &str, output_path: &str) -> Result<(), Box> { + let file = File::open(archive_path)?; + let mut zip = ZipArchive::new(file)?; + + let mut state_sql = zip.by_name("state.sql") + .expect("State.sql file not found in archive"); + let outpath = Path::new(output_path); + + if let Some(p) = outpath.parent() { + std::fs::create_dir_all(&p)?; + } + let mut outfile = File::create(&outpath)?; + std::io::copy(&mut state_sql, &mut outfile)?; + + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let cli = Cli::parse(); + + match cli.command { + Commands::Check { node_data, genesis_time, layer_duration } => { + let dir_path = PathBuf::from(node_data.clone()); + let db_file_path = dir_path.join("state.sql"); + let db_file_str = db_file_path.to_str().expect("Cannot compose path"); + println!("Checking database: {}", &db_file_str); + let db_layer = i64::from(get_last_layer_from_db(db_file_str)?); + let time_layer = calculate_latest_layer(genesis_time, layer_duration)?; + println!("Latest layer in db: {}", db_layer); + println!("Latest calculated layer: {}", time_layer); + if db_layer - time_layer > 100 { + println!("Too far behind"); + } else { + println!("OK!"); + } + Ok(()) + } + Commands::Download { node_data, go_spacemesh_path, download_url } => { + let dir_path = PathBuf::from(node_data); + let temp_file_path = dir_path.join("state.download"); + let redirect_file_path = dir_path.join("state.url"); + let archive_file_path = dir_path.join("state.zip"); + let final_file_path = dir_path.join("state.sql"); + let backup_file_path = dir_path.join("state.sql.bak"); + + let temp_file_str = temp_file_path.to_str().expect("Cannot compose path"); + let redirect_file_str = redirect_file_path.to_str().expect("Cannot compose path"); + let archive_file_str = archive_file_path.to_str().expect("Cannot compose path"); + let final_file_str = final_file_path.to_str().expect("Cannot compose path"); + let backup_file_str = backup_file_path.to_str().expect("Cannot compose path"); + + let url = if redirect_file_path.exists() { + std::fs::read_to_string(redirect_file_str)? + } else { + let go_path = resolve_path(&go_spacemesh_path).unwrap(); + let go_path_str = go_path.to_str().expect("Cannot resolve path to go-spacemesh"); + format!("{}{}", &download_url, get_go_spacemesh_version(&go_path_str)?) + }; + + download_file(&url, temp_file_str, redirect_file_str).await?; + // TODO: Display a progress + + // Rename `state.download` -> `state.zip` + std::fs::rename(temp_file_str, archive_file_str)?; + + if final_file_path.exists() { + println!("Renaming current state.sql file into state.sql.bak"); + // Rename original State.Sql (backup) + std::fs::rename(final_file_str, backup_file_str)?; + } + + // Unzip + println!("Unzipping downloaded archive"); + unzip_state_sql(archive_file_str, final_file_str) + .expect("Cannot unzip archive"); + + // TODO: Delete state.url + // TODO: Download the checksum and validate (e.g. http://localhost:8080/abcdef.checksum) + + Ok(()) + } + } } From 6588bb3e2609b028e730dcb6b325faea54deb23f Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Thu, 11 Jan 2024 19:42:22 +0700 Subject: [PATCH 02/32] Remove build from go-spacemesh version --- src/main.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index bfae03b..d973331 100644 --- a/src/main.rs +++ b/src/main.rs @@ -104,6 +104,10 @@ async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> Resul Ok(()) } +fn trim_version(version: &str) -> &str { + version.split('+').next().unwrap_or(version) +} + fn get_go_spacemesh_version(path: &str) -> Result> { let output = Command::new(path) .arg("version") @@ -111,10 +115,10 @@ fn get_go_spacemesh_version(path: &str) -> Result> { .expect("Cannot run go-spacemesh version"); let version = String::from_utf8(output.stdout)? - .trim() - .to_string(); + .trim(); + let trimmed = trim_version(version).to_string(); - Ok(version) + Ok(trimmed) } fn resolve_path(relative_path: &str) -> Result> { From 817fedd1e6d756d805023606f3028ba7ac52c638 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Fri, 12 Jan 2024 01:04:31 +0700 Subject: [PATCH 03/32] Add printing downloading & unzipping progress --- src/main.rs | 113 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 89 insertions(+), 24 deletions(-) diff --git a/src/main.rs b/src/main.rs index d973331..7726d8a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,15 +1,16 @@ use chrono::{DateTime, Utc, Duration}; use clap::{Parser, Subcommand}; use duration_string::DurationString; -use reqwest::Client; +use reqwest::{Client, header}; use rusqlite::{Connection, params}; use zip::ZipArchive; use std::env; -use std::fs::{OpenOptions, create_dir_all}; -use std::io::{Seek, SeekFrom, Write}; +use std::fs::{OpenOptions, create_dir_all, File}; +use std::io::{Seek, SeekFrom, Write, Read}; use std::process::Command; use std::path::{PathBuf, Path}; use std::error::Error; +use std::time::Instant; use futures_util::StreamExt; #[derive(Parser, Debug)] @@ -41,8 +42,8 @@ enum Commands { /// Path to go-spacemesh binary #[clap(short, long, default_value = go_spacemesh_default_path())] go_spacemesh_path: String, - /// URL to download from (without version at the end). Default: http://localhost:8080/ - #[clap(short, long, default_value = "http://localhost:8080/")] + /// URL to download from (without version at the end). Default: http://localhost:8000/ + #[clap(short, long, default_value = "http://localhost:8000/")] download_url: String, }, } @@ -67,7 +68,6 @@ async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> Resul let path = Path::new(file_path); if let Some(dir) = path.parent() { - // Пытаемся создать директорию create_dir_all(dir).expect("Cannot create directory"); } @@ -90,12 +90,46 @@ async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> Resul std::fs::write(redirect_path, final_url.as_str())?; if response.status().is_success() { + let total_size = response + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|ct_len| ct_len.to_str().ok()) + .and_then(|ct_len| ct_len.parse::().ok()) + .unwrap_or(0) + file_size; + file.seek(SeekFrom::End(0))?; let mut stream = response.bytes_stream(); + let start = Instant::now(); + + let mut downloaded: u64 = file_size; + let mut last_reported_progress: i64 = -1; while let Some(item) = stream.next().await { let chunk = item?; file.write_all(&chunk)?; + downloaded += chunk.len() as u64; + + let elapsed = start.elapsed().as_secs_f64(); + let speed = if elapsed > 0.0 { + downloaded as f64 / elapsed + } else { + 0.0 + }; + let eta = if speed > 0.0 { + (total_size as f64 - downloaded as f64) / speed + } else { + 0.0 + }; + + let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; + if progress > last_reported_progress { + println!("Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", + progress, + downloaded as f64 / 1_024_000.0, + total_size as f64 / 1_024_000.0, + eta); + last_reported_progress = progress; + } } } else { println!("Cannot resume downloading: {:?}", response.status()); @@ -114,9 +148,8 @@ fn get_go_spacemesh_version(path: &str) -> Result> { .output() .expect("Cannot run go-spacemesh version"); - let version = String::from_utf8(output.stdout)? - .trim(); - let trimmed = trim_version(version).to_string(); + let version = String::from_utf8(output.stdout)?; + let trimmed = trim_version(version.trim()).to_string(); Ok(trimmed) } @@ -162,7 +195,30 @@ fn unzip_state_sql(archive_path: &str, output_path: &str) -> Result<(), Box Result<(), Box> { let final_file_str = final_file_path.to_str().expect("Cannot compose path"); let backup_file_str = backup_file_path.to_str().expect("Cannot compose path"); - let url = if redirect_file_path.exists() { - std::fs::read_to_string(redirect_file_str)? + if !archive_file_path.exists() { + println!("Downloading the latest database..."); + let url = if redirect_file_path.exists() { + std::fs::read_to_string(redirect_file_str)? + } else { + let go_path = resolve_path(&go_spacemesh_path).unwrap(); + let go_path_str = go_path.to_str().expect("Cannot resolve path to go-spacemesh"); + format!("{}{}", &download_url, get_go_spacemesh_version(&go_path_str)?) + }; + + download_file(&url, temp_file_str, redirect_file_str).await?; + + // Rename `state.download` -> `state.zip` + std::fs::rename(temp_file_str, archive_file_str)?; + println!("Archive downloaded!"); } else { - let go_path = resolve_path(&go_spacemesh_path).unwrap(); - let go_path_str = go_path.to_str().expect("Cannot resolve path to go-spacemesh"); - format!("{}{}", &download_url, get_go_spacemesh_version(&go_path_str)?) - }; - - download_file(&url, temp_file_str, redirect_file_str).await?; - // TODO: Display a progress - - // Rename `state.download` -> `state.zip` - std::fs::rename(temp_file_str, archive_file_str)?; + println!("Archive found..."); + } if final_file_path.exists() { println!("Renaming current state.sql file into state.sql.bak"); @@ -223,13 +284,17 @@ async fn main() -> Result<(), Box> { } // Unzip - println!("Unzipping downloaded archive"); unzip_state_sql(archive_file_str, final_file_str) .expect("Cannot unzip archive"); - // TODO: Delete state.url // TODO: Download the checksum and validate (e.g. http://localhost:8080/abcdef.checksum) + std::fs::remove_file(redirect_file_str)?; + std::fs::remove_file(archive_file_str)?; + + println!("Done!"); + println!("Now you can run go-spacemesh as usually."); + Ok(()) } } From 1ef8ef77c5563304b5e6d6096fe16c5533c65b2a Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Fri, 12 Jan 2024 01:08:06 +0700 Subject: [PATCH 04/32] Move readme --- README.md | 12 ++++++++++++ src/README.md | 0 2 files changed, 12 insertions(+) create mode 100644 README.md delete mode 100644 src/README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..14ec28e --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +Quicksync-rs +============ + +How to use: +``` +quicksync-rs help +``` + +In development: +``` +cargo run -- help +``` \ No newline at end of file diff --git a/src/README.md b/src/README.md deleted file mode 100644 index e69de29..0000000 From d0db29bb897f871964ee21c26cfc59a6f9fcc9b2 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Fri, 12 Jan 2024 01:23:26 +0700 Subject: [PATCH 05/32] Change argument defaults --- src/main.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index 7726d8a..1407160 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,25 +25,25 @@ enum Commands { /// Checks if quicksync is recommended Check { /// Path to the node-data directory - #[clap(short, long)] + #[clap(short = 'd', long)] node_data: String, /// Genesis time in ISO format - #[clap(short, long, default_value = "2023-07-14T08:00:00Z")] + #[clap(short = 'g', long, default_value = "2023-07-14T08:00:00Z")] genesis_time: String, /// Layer duration - #[clap(short, long, default_value = "5m")] + #[clap(short = 'l', long, default_value = "5m")] layer_duration: String, }, /// Downloads latest db from official website Download { /// Path to the node-data directory - #[clap(short, long)] + #[clap(short = 'd', long)] node_data: String, /// Path to go-spacemesh binary - #[clap(short, long, default_value = go_spacemesh_default_path())] + #[clap(short = 'p', long, default_value = go_spacemesh_default_path())] go_spacemesh_path: String, - /// URL to download from (without version at the end). Default: http://localhost:8000/ - #[clap(short, long, default_value = "http://localhost:8000/")] + /// URL to download database from. Node version will be appended at the end + #[clap(short = 'u', long, default_value = "https://quicksync.spacemesh.network/")] download_url: String, }, } From f5698cc352b9d52864e82ee02af348447bae04d1 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Fri, 12 Jan 2024 02:00:15 +0700 Subject: [PATCH 06/32] Handle non-200 responses properly --- Cargo.lock | 1 + Cargo.toml | 1 + src/main.rs | 25 +++++++++++++++++++++---- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76d00b6..6f4eaa4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1010,6 +1010,7 @@ dependencies = [ "reqwest", "rusqlite", "tokio", + "url", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index 36179c0..ff0c6c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,4 +13,5 @@ futures-util = "0.3.30" reqwest = { version = "0.11.23", features = ["json", "stream"] } rusqlite = "0.30.0" tokio = { version = "1.35.1", features = ["full"] } +url = "2.5.0" zip = "0.6.6" diff --git a/src/main.rs b/src/main.rs index 1407160..dd239de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ use clap::{Parser, Subcommand}; use duration_string::DurationString; use reqwest::{Client, header}; use rusqlite::{Connection, params}; +use url::{Url, ParseError}; use zip::ZipArchive; use std::env; use std::fs::{OpenOptions, create_dir_all, File}; @@ -131,11 +132,19 @@ async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> Resul last_reported_progress = progress; } } + Ok(()) } else { - println!("Cannot resume downloading: {:?}", response.status()); + let err_message = format!("Cannot download: {:?}", response.status()); + + std::fs::remove_file(redirect_path)?; + std::fs::remove_file(file_path)?; + Err( + Box::new(std::io::Error::new( + std::io::ErrorKind::NotFound, + err_message + )) + ) } - - Ok(()) } fn trim_version(version: &str) -> &str { @@ -223,6 +232,12 @@ fn unzip_state_sql(archive_path: &str, output_path: &str) -> Result<(), Box Result { + let mut url = Url::parse(base)?; + url.path_segments_mut().expect("cannot be base").extend(path.split('/')); + Ok(url) +} + #[tokio::main] async fn main() -> Result<(), Box> { let cli = Cli::parse(); @@ -265,7 +280,9 @@ async fn main() -> Result<(), Box> { } else { let go_path = resolve_path(&go_spacemesh_path).unwrap(); let go_path_str = go_path.to_str().expect("Cannot resolve path to go-spacemesh"); - format!("{}{}", &download_url, get_go_spacemesh_version(&go_path_str)?) + let path = format!("{}/state.zip", &get_go_spacemesh_version(&go_path_str)?); + let url = build_url(&download_url, &path)?; + url.to_string() }; download_file(&url, temp_file_str, redirect_file_str).await?; From 67c9ea35e885a7b1de6f2c94f5664c125dcf0039 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Fri, 12 Jan 2024 03:57:51 +0700 Subject: [PATCH 07/32] Validate md5 checksum on unpacked state.sql --- Cargo.lock | 7 +++++ Cargo.toml | 1 + src/main.rs | 80 +++++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f4eaa4..bfb368c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -786,6 +786,12 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.7.1" @@ -1007,6 +1013,7 @@ dependencies = [ "clap", "duration-string", "futures-util", + "md5", "reqwest", "rusqlite", "tokio", diff --git a/Cargo.toml b/Cargo.toml index ff0c6c2..2112446 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ chrono = "0.4.31" clap = { version="4.4.14", features=["derive"] } duration-string = "0.3.0" futures-util = "0.3.30" +md5 = "0.7.0" reqwest = { version = "0.11.23", features = ["json", "stream"] } rusqlite = "0.30.0" tokio = { version = "1.35.1", features = ["full"] } diff --git a/src/main.rs b/src/main.rs index dd239de..c2606c1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,9 +3,10 @@ use clap::{Parser, Subcommand}; use duration_string::DurationString; use reqwest::{Client, header}; use rusqlite::{Connection, params}; +use tokio::time::sleep; use url::{Url, ParseError}; use zip::ZipArchive; -use std::env; +use std::{env, io}; use std::fs::{OpenOptions, create_dir_all, File}; use std::io::{Seek, SeekFrom, Write, Read}; use std::process::Command; @@ -65,6 +66,45 @@ fn parse_iso_date(iso_date: &str) -> Result, chrono::ParseError> { iso_date.parse::>() } +fn strip_trailing_newline(input: &str) -> &str { + input + .strip_suffix("\r\n") + .or(input.strip_suffix("\n")) + .unwrap_or(input) +} + +async fn download_md5(url: &str) -> Result> { + let mut u = Url::parse(url)?; + u.path_segments_mut().expect("Wrong URL").pop().push("state.sql.md5"); + let md5_url = u.to_string(); + + let client = Client::new(); + let response = client.get(md5_url) + .send() + .await?; + + if response.status().is_success() { + let md5 = response.text().await?; + let stripped = strip_trailing_newline(&md5); + Ok(stripped.to_string()) + } else { + Err( + Box::new(std::io::Error::new( + std::io::ErrorKind::NotFound, + "Cannot download MD5 checksum" + )) + ) + } +} + +fn calculate_md5(file_path: &str) -> io::Result { + let mut file = File::open(file_path)?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + let hash = md5::compute(buffer); + Ok(format!("{:x}", hash)) +} + async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> Result<(), Box> { let path = Path::new(file_path); @@ -147,6 +187,22 @@ async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> Resul } } +async fn download_with_retries(url: &str, file_path: &str, redirect_path: &str, max_retries: u32) -> Result<(), Box> { + let mut attempts = 0; + + loop { + match download_file(url, file_path, redirect_path).await { + Ok(()) => return Ok(()), + Err(e) if attempts < max_retries => { + eprintln!("Download error: {}. Attemmpt {} / {}", e, attempts + 1, max_retries); + attempts += 1; + sleep(std::time::Duration::from_secs(5)).await; + } + Err(e) => return Err(e), + } + } +} + fn trim_version(version: &str) -> &str { version.split('+').next().unwrap_or(version) } @@ -192,7 +248,7 @@ fn calculate_latest_layer(genesis_time: String, layer_duration: String) -> Resul Ok(delta.num_milliseconds() / dur.num_milliseconds()) } -fn unzip_state_sql(archive_path: &str, output_path: &str) -> Result<(), Box> { +async fn unzip_state_sql(archive_path: &str, output_path: &str) -> Result<(), Box> { let file = File::open(archive_path)?; let mut zip = ZipArchive::new(file)?; @@ -241,6 +297,7 @@ fn build_url(base: &str, path: &str) -> Result { #[tokio::main] async fn main() -> Result<(), Box> { let cli = Cli::parse(); + let max_retries = 5; match cli.command { Commands::Check { node_data, genesis_time, layer_duration } => { @@ -285,7 +342,9 @@ async fn main() -> Result<(), Box> { url.to_string() }; - download_file(&url, temp_file_str, redirect_file_str).await?; + if let Err(e) = download_with_retries(&url, temp_file_str, redirect_file_str, max_retries).await { + eprintln!("Failed to download a file after {} attempts: {}", max_retries, e); + } // Rename `state.download` -> `state.zip` std::fs::rename(temp_file_str, archive_file_str)?; @@ -297,14 +356,25 @@ async fn main() -> Result<(), Box> { if final_file_path.exists() { println!("Renaming current state.sql file into state.sql.bak"); // Rename original State.Sql (backup) - std::fs::rename(final_file_str, backup_file_str)?; + std::fs::rename(final_file_str, backup_file_str).expect("Cannot rename state.sql -> state.sql.bak"); } // Unzip unzip_state_sql(archive_file_str, final_file_str) + .await .expect("Cannot unzip archive"); - // TODO: Download the checksum and validate (e.g. http://localhost:8080/abcdef.checksum) + println!("Checking MD5 checksum..."); + let archive_url = String::from_utf8( + std::fs::read(redirect_file_str).expect("Cannot read state.url") + )?; + let md5_expected = download_md5(&archive_url).await.expect("Cannot download md5"); + let md5_actual = calculate_md5(final_file_str).expect("Cannot calculate md5"); + + assert_eq!( + md5_actual, md5_expected, + "MD5 checksums are not equal" + ); std::fs::remove_file(redirect_file_str)?; std::fs::remove_file(archive_file_str)?; From 952db9524ccbe638f3be65a4cacc34bfebc87945 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Fri, 12 Jan 2024 15:09:27 +0700 Subject: [PATCH 08/32] Refactor: move functions in separate modules --- src/checksum.rs | 37 ++++++ src/download.rs | 106 +++++++++++++++++ src/go_spacemesh.rs | 16 +++ src/main.rs | 270 ++++---------------------------------------- src/sql.rs | 18 +++ src/utils.rs | 38 +++++++ src/zip.rs | 45 ++++++++ 7 files changed, 279 insertions(+), 251 deletions(-) create mode 100644 src/checksum.rs create mode 100644 src/download.rs create mode 100644 src/go_spacemesh.rs create mode 100644 src/sql.rs create mode 100644 src/utils.rs create mode 100644 src/zip.rs diff --git a/src/checksum.rs b/src/checksum.rs new file mode 100644 index 0000000..0fedd15 --- /dev/null +++ b/src/checksum.rs @@ -0,0 +1,37 @@ +use reqwest::Client; +use url::Url; +use std::{error::Error, fs::File, io, io::Read}; + +use crate::utils::strip_trailing_newline; + +pub async fn download_checksum(url: &str) -> Result> { + let mut u = Url::parse(url)?; + u.path_segments_mut().expect("Wrong URL").pop().push("state.sql.md5"); + let md5_url = u.to_string(); + + let client = Client::new(); + let response = client.get(md5_url) + .send() + .await?; + + if response.status().is_success() { + let md5 = response.text().await?; + let stripped = strip_trailing_newline(&md5); + Ok(stripped.to_string()) + } else { + Err( + Box::new(std::io::Error::new( + std::io::ErrorKind::NotFound, + "Cannot download MD5 checksum" + )) + ) + } +} + +pub fn calculate_checksum(file_path: &str) -> io::Result { + let mut file = File::open(file_path)?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + let hash = md5::compute(buffer); + Ok(format!("{:x}", hash)) +} \ No newline at end of file diff --git a/src/download.rs b/src/download.rs new file mode 100644 index 0000000..7a6b07a --- /dev/null +++ b/src/download.rs @@ -0,0 +1,106 @@ +use reqwest::{Client, header}; +use tokio::time::sleep; +use std::error::Error; +use std::path::Path; +use std::time::Instant; +use std::fs::{OpenOptions, create_dir_all}; +use std::io::{Seek, SeekFrom, Write}; +use futures_util::StreamExt; + +pub async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> Result<(), Box> { + let path = Path::new(file_path); + + if let Some(dir) = path.parent() { + create_dir_all(dir).expect("Cannot create directory"); + } + + let mut file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(file_path) + .expect("Cannot create file"); + + let file_size = file.metadata()?.len(); + + let client = Client::new(); + let response = client.get(url) + .header("Range", format!("bytes={}-", file_size)) + .send() + .await?; + let final_url = response.url().clone(); + + std::fs::write(redirect_path, final_url.as_str())?; + + if response.status().is_success() { + let total_size = response + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|ct_len| ct_len.to_str().ok()) + .and_then(|ct_len| ct_len.parse::().ok()) + .unwrap_or(0) + file_size; + + file.seek(SeekFrom::End(0))?; + let mut stream = response.bytes_stream(); + let start = Instant::now(); + + let mut downloaded: u64 = file_size; + let mut last_reported_progress: i64 = -1; + + while let Some(item) = stream.next().await { + let chunk = item?; + file.write_all(&chunk)?; + downloaded += chunk.len() as u64; + + let elapsed = start.elapsed().as_secs_f64(); + let speed = if elapsed > 0.0 { + downloaded as f64 / elapsed + } else { + 0.0 + }; + let eta = if speed > 0.0 { + (total_size as f64 - downloaded as f64) / speed + } else { + 0.0 + }; + + let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; + if progress > last_reported_progress { + println!("Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", + progress, + downloaded as f64 / 1_024_000.0, + total_size as f64 / 1_024_000.0, + eta); + last_reported_progress = progress; + } + } + Ok(()) + } else { + let err_message = format!("Cannot download: {:?}", response.status()); + + std::fs::remove_file(redirect_path)?; + std::fs::remove_file(file_path)?; + Err( + Box::new(std::io::Error::new( + std::io::ErrorKind::NotFound, + err_message + )) + ) + } +} + +pub async fn download_with_retries(url: &str, file_path: &str, redirect_path: &str, max_retries: u32) -> Result<(), Box> { + let mut attempts = 0; + + loop { + match download_file(url, file_path, redirect_path).await { + Ok(()) => return Ok(()), + Err(e) if attempts < max_retries => { + eprintln!("Download error: {}. Attemmpt {} / {}", e, attempts + 1, max_retries); + attempts += 1; + sleep(std::time::Duration::from_secs(5)).await; + } + Err(e) => return Err(e), + } + } +} \ No newline at end of file diff --git a/src/go_spacemesh.rs b/src/go_spacemesh.rs new file mode 100644 index 0000000..bf8fa24 --- /dev/null +++ b/src/go_spacemesh.rs @@ -0,0 +1,16 @@ +use std::error::Error; +use std::process::Command; + +use crate::utils::trim_version; + +pub fn get_version(path: &str) -> Result> { + let output = Command::new(path) + .arg("version") + .output() + .expect("Cannot run go-spacemesh version"); + + let version = String::from_utf8(output.stdout)?; + let trimmed = trim_version(version.trim()).to_string(); + + Ok(trimmed) +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index c2606c1..96f8d90 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,19 +1,20 @@ -use chrono::{DateTime, Utc, Duration}; use clap::{Parser, Subcommand}; -use duration_string::DurationString; -use reqwest::{Client, header}; -use rusqlite::{Connection, params}; -use tokio::time::sleep; -use url::{Url, ParseError}; -use zip::ZipArchive; -use std::{env, io}; -use std::fs::{OpenOptions, create_dir_all, File}; -use std::io::{Seek, SeekFrom, Write, Read}; -use std::process::Command; -use std::path::{PathBuf, Path}; +use std::path::PathBuf; use std::error::Error; -use std::time::Instant; -use futures_util::StreamExt; + +mod utils; +mod checksum; +mod download; +mod sql; +mod go_spacemesh; +mod zip; + +use utils::*; +use checksum::*; +use download::download_with_retries; +use sql::get_last_layer_from_db; +use go_spacemesh::get_version; +use zip::unpack; #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] @@ -50,7 +51,6 @@ enum Commands { }, } -// Функция для определения пути по умолчанию в зависимости от ОС fn go_spacemesh_default_path() -> &'static str { #[cfg(target_os = "windows")] { @@ -62,238 +62,6 @@ fn go_spacemesh_default_path() -> &'static str { } } -fn parse_iso_date(iso_date: &str) -> Result, chrono::ParseError> { - iso_date.parse::>() -} - -fn strip_trailing_newline(input: &str) -> &str { - input - .strip_suffix("\r\n") - .or(input.strip_suffix("\n")) - .unwrap_or(input) -} - -async fn download_md5(url: &str) -> Result> { - let mut u = Url::parse(url)?; - u.path_segments_mut().expect("Wrong URL").pop().push("state.sql.md5"); - let md5_url = u.to_string(); - - let client = Client::new(); - let response = client.get(md5_url) - .send() - .await?; - - if response.status().is_success() { - let md5 = response.text().await?; - let stripped = strip_trailing_newline(&md5); - Ok(stripped.to_string()) - } else { - Err( - Box::new(std::io::Error::new( - std::io::ErrorKind::NotFound, - "Cannot download MD5 checksum" - )) - ) - } -} - -fn calculate_md5(file_path: &str) -> io::Result { - let mut file = File::open(file_path)?; - let mut buffer = Vec::new(); - file.read_to_end(&mut buffer)?; - let hash = md5::compute(buffer); - Ok(format!("{:x}", hash)) -} - -async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> Result<(), Box> { - let path = Path::new(file_path); - - if let Some(dir) = path.parent() { - create_dir_all(dir).expect("Cannot create directory"); - } - - let mut file = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(file_path) - .expect("Cannot create file"); - - let file_size = file.metadata()?.len(); - - let client = Client::new(); - let response = client.get(url) - .header("Range", format!("bytes={}-", file_size)) - .send() - .await?; - let final_url = response.url().clone(); - - std::fs::write(redirect_path, final_url.as_str())?; - - if response.status().is_success() { - let total_size = response - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|ct_len| ct_len.to_str().ok()) - .and_then(|ct_len| ct_len.parse::().ok()) - .unwrap_or(0) + file_size; - - file.seek(SeekFrom::End(0))?; - let mut stream = response.bytes_stream(); - let start = Instant::now(); - - let mut downloaded: u64 = file_size; - let mut last_reported_progress: i64 = -1; - - while let Some(item) = stream.next().await { - let chunk = item?; - file.write_all(&chunk)?; - downloaded += chunk.len() as u64; - - let elapsed = start.elapsed().as_secs_f64(); - let speed = if elapsed > 0.0 { - downloaded as f64 / elapsed - } else { - 0.0 - }; - let eta = if speed > 0.0 { - (total_size as f64 - downloaded as f64) / speed - } else { - 0.0 - }; - - let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; - if progress > last_reported_progress { - println!("Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", - progress, - downloaded as f64 / 1_024_000.0, - total_size as f64 / 1_024_000.0, - eta); - last_reported_progress = progress; - } - } - Ok(()) - } else { - let err_message = format!("Cannot download: {:?}", response.status()); - - std::fs::remove_file(redirect_path)?; - std::fs::remove_file(file_path)?; - Err( - Box::new(std::io::Error::new( - std::io::ErrorKind::NotFound, - err_message - )) - ) - } -} - -async fn download_with_retries(url: &str, file_path: &str, redirect_path: &str, max_retries: u32) -> Result<(), Box> { - let mut attempts = 0; - - loop { - match download_file(url, file_path, redirect_path).await { - Ok(()) => return Ok(()), - Err(e) if attempts < max_retries => { - eprintln!("Download error: {}. Attemmpt {} / {}", e, attempts + 1, max_retries); - attempts += 1; - sleep(std::time::Duration::from_secs(5)).await; - } - Err(e) => return Err(e), - } - } -} - -fn trim_version(version: &str) -> &str { - version.split('+').next().unwrap_or(version) -} - -fn get_go_spacemesh_version(path: &str) -> Result> { - let output = Command::new(path) - .arg("version") - .output() - .expect("Cannot run go-spacemesh version"); - - let version = String::from_utf8(output.stdout)?; - let trimmed = trim_version(version.trim()).to_string(); - - Ok(trimmed) -} - -fn resolve_path(relative_path: &str) -> Result> { - let current_dir = env::current_dir()?; - let resolved_path = current_dir.join(relative_path); - Ok(resolved_path) -} - -fn get_last_layer_from_db(db_path: &str) -> Result> { - let conn = Connection::open(db_path)?; - - let mut stmt = conn.prepare("SELECT * FROM layers ORDER BY id DESC LIMIT 1")?; - let mut layer_iter = stmt.query_map(params![], |row| { - Ok(row.get::<_, i32>(0)?) - })?; - - if let Some(result) = layer_iter.next() { - let last_id = result?; - Ok(last_id) - } else { - Ok(0) - } -} - -fn calculate_latest_layer(genesis_time: String, layer_duration: String) -> Result> { - let genesis = parse_iso_date(&genesis_time)?; - let delta = Utc::now() - genesis; - let dur = Duration::from_std(DurationString::from_string(layer_duration)?.into())?; - Ok(delta.num_milliseconds() / dur.num_milliseconds()) -} - -async fn unzip_state_sql(archive_path: &str, output_path: &str) -> Result<(), Box> { - let file = File::open(archive_path)?; - let mut zip = ZipArchive::new(file)?; - - let mut state_sql = zip.by_name("state.sql") - .expect("State.sql file not found in archive"); - let outpath = Path::new(output_path); - - if let Some(p) = outpath.parent() { - std::fs::create_dir_all(&p)?; - } - let mut outfile = File::create(&outpath)?; - - let total_size = state_sql.size(); - let mut extracted_size: u64 = 0; - let mut buffer = [0; 4096]; - - let mut last_reported_progress: i64 = -1; - - while let Ok(bytes_read) = state_sql.read(&mut buffer) { - if bytes_read == 0 { - break; - } - outfile.write_all(&buffer[..bytes_read])?; - extracted_size += bytes_read as u64; - - let progress = (extracted_size as f64 / total_size as f64 * 100.0).round() as i64; - if last_reported_progress != progress { - last_reported_progress = progress; - println!("Unzipping... {}%", progress); - } - } - if last_reported_progress < 100 { - // Ensure that 100% will be printed - println!("Unzipping... 100%"); - } - - Ok(()) -} - -fn build_url(base: &str, path: &str) -> Result { - let mut url = Url::parse(base)?; - url.path_segments_mut().expect("cannot be base").extend(path.split('/')); - Ok(url) -} - #[tokio::main] async fn main() -> Result<(), Box> { let cli = Cli::parse(); @@ -337,7 +105,7 @@ async fn main() -> Result<(), Box> { } else { let go_path = resolve_path(&go_spacemesh_path).unwrap(); let go_path_str = go_path.to_str().expect("Cannot resolve path to go-spacemesh"); - let path = format!("{}/state.zip", &get_go_spacemesh_version(&go_path_str)?); + let path = format!("{}/state.zip", &get_version(&go_path_str)?); let url = build_url(&download_url, &path)?; url.to_string() }; @@ -360,7 +128,7 @@ async fn main() -> Result<(), Box> { } // Unzip - unzip_state_sql(archive_file_str, final_file_str) + unpack(archive_file_str, final_file_str) .await .expect("Cannot unzip archive"); @@ -368,8 +136,8 @@ async fn main() -> Result<(), Box> { let archive_url = String::from_utf8( std::fs::read(redirect_file_str).expect("Cannot read state.url") )?; - let md5_expected = download_md5(&archive_url).await.expect("Cannot download md5"); - let md5_actual = calculate_md5(final_file_str).expect("Cannot calculate md5"); + let md5_expected = download_checksum(&archive_url).await.expect("Cannot download md5"); + let md5_actual = calculate_checksum(final_file_str).expect("Cannot calculate md5"); assert_eq!( md5_actual, md5_expected, diff --git a/src/sql.rs b/src/sql.rs new file mode 100644 index 0000000..c5cb0d3 --- /dev/null +++ b/src/sql.rs @@ -0,0 +1,18 @@ +use std::error::Error; +use rusqlite::{Connection, params}; + +pub fn get_last_layer_from_db(db_path: &str) -> Result> { + let conn = Connection::open(db_path)?; + + let mut stmt = conn.prepare("SELECT * FROM layers ORDER BY id DESC LIMIT 1")?; + let mut layer_iter = stmt.query_map(params![], |row| { + Ok(row.get::<_, i32>(0)?) + })?; + + if let Some(result) = layer_iter.next() { + let last_id = result?; + Ok(last_id) + } else { + Ok(0) + } +} \ No newline at end of file diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..b7f1d9e --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,38 @@ +use chrono::{DateTime, Utc, Duration}; +use duration_string::DurationString; +use url::{Url, ParseError}; +use std::{error::Error, path::PathBuf, env}; + +pub fn parse_iso_date(iso_date: &str) -> Result, chrono::ParseError> { + iso_date.parse::>() +} + +pub fn strip_trailing_newline(input: &str) -> &str { + input + .strip_suffix("\r\n") + .or(input.strip_suffix("\n")) + .unwrap_or(input) +} + +pub fn calculate_latest_layer(genesis_time: String, layer_duration: String) -> Result> { + let genesis = parse_iso_date(&genesis_time)?; + let delta = Utc::now() - genesis; + let dur = Duration::from_std(DurationString::from_string(layer_duration)?.into())?; + Ok(delta.num_milliseconds() / dur.num_milliseconds()) +} + +pub fn resolve_path(relative_path: &str) -> Result> { + let current_dir = env::current_dir()?; + let resolved_path = current_dir.join(relative_path); + Ok(resolved_path) +} + +pub fn trim_version(version: &str) -> &str { + version.split('+').next().unwrap_or(version) +} + +pub fn build_url(base: &str, path: &str) -> Result { + let mut url = Url::parse(base)?; + url.path_segments_mut().expect("cannot be base").extend(path.split('/')); + Ok(url) +} \ No newline at end of file diff --git a/src/zip.rs b/src/zip.rs new file mode 100644 index 0000000..eb3bee0 --- /dev/null +++ b/src/zip.rs @@ -0,0 +1,45 @@ +use std::fs::File; +use std::error::Error; +use std::io::{Write, Read}; +use std::path::Path; +use zip::ZipArchive; + +pub async fn unpack(archive_path: &str, output_path: &str) -> Result<(), Box> { + let file = File::open(archive_path)?; + let mut zip = ZipArchive::new(file)?; + + let mut state_sql = zip.by_name("state.sql") + .expect("State.sql file not found in archive"); + let outpath = Path::new(output_path); + + if let Some(p) = outpath.parent() { + std::fs::create_dir_all(&p)?; + } + let mut outfile = File::create(&outpath)?; + + let total_size = state_sql.size(); + let mut extracted_size: u64 = 0; + let mut buffer = [0; 4096]; + + let mut last_reported_progress: i64 = -1; + + while let Ok(bytes_read) = state_sql.read(&mut buffer) { + if bytes_read == 0 { + break; + } + outfile.write_all(&buffer[..bytes_read])?; + extracted_size += bytes_read as u64; + + let progress = (extracted_size as f64 / total_size as f64 * 100.0).round() as i64; + if last_reported_progress != progress { + last_reported_progress = progress; + println!("Unzipping... {}%", progress); + } + } + if last_reported_progress < 100 { + // Ensure that 100% will be printed + println!("Unzipping... 100%"); + } + + Ok(()) +} \ No newline at end of file From ad75598e761abbff2c00844f1eab6b2c51d53e82 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Fri, 12 Jan 2024 15:59:16 +0700 Subject: [PATCH 09/32] Show error if archive is not unpacked well / entirely --- src/zip.rs | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/zip.rs b/src/zip.rs index eb3bee0..fa2a437 100644 --- a/src/zip.rs +++ b/src/zip.rs @@ -23,22 +23,34 @@ pub async fn unpack(archive_path: &str, output_path: &str) -> Result<(), Box { + if last_reported_progress != 100 { + last_reported_progress = 100; + println!("Unzipping... {}%", last_reported_progress); + } + break + }, + Ok(bytes_read) => { + outfile.write_all(&buffer[..bytes_read])?; + extracted_size += bytes_read as u64; + + let progress = (extracted_size as f64 / total_size as f64 * 100.0).round() as i64; + if last_reported_progress != progress { + last_reported_progress = progress; + println!("Unzipping... {}%", progress); + } + } + Err(e) => return Err(Box::new(e)), + } } + if last_reported_progress < 100 { - // Ensure that 100% will be printed - println!("Unzipping... 100%"); + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "Archive was not fully unpacked", + ))); } Ok(()) From a1c0ea9a2ba12e8b2969e680d58b2f884449f948 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Fri, 12 Jan 2024 23:14:33 +0700 Subject: [PATCH 10/32] Handle errors properly --- src/checksum.rs | 4 +-- src/download.rs | 4 +-- src/main.rs | 70 +++++++++++++++++++++++++++++-------------------- src/zip.rs | 2 +- 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/src/checksum.rs b/src/checksum.rs index 0fedd15..964fb2a 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -1,6 +1,6 @@ use reqwest::Client; use url::Url; -use std::{error::Error, fs::File, io, io::Read}; +use std::{error::Error, fs::File, io, io::Read, path::Path}; use crate::utils::strip_trailing_newline; @@ -28,7 +28,7 @@ pub async fn download_checksum(url: &str) -> Result> { } } -pub fn calculate_checksum(file_path: &str) -> io::Result { +pub fn calculate_checksum(file_path: &Path) -> io::Result { let mut file = File::open(file_path)?; let mut buffer = Vec::new(); file.read_to_end(&mut buffer)?; diff --git a/src/download.rs b/src/download.rs index 7a6b07a..5072cd6 100644 --- a/src/download.rs +++ b/src/download.rs @@ -7,7 +7,7 @@ use std::fs::{OpenOptions, create_dir_all}; use std::io::{Seek, SeekFrom, Write}; use futures_util::StreamExt; -pub async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> Result<(), Box> { +pub async fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> Result<(), Box> { let path = Path::new(file_path); if let Some(dir) = path.parent() { @@ -89,7 +89,7 @@ pub async fn download_file(url: &str, file_path: &str, redirect_path: &str) -> R } } -pub async fn download_with_retries(url: &str, file_path: &str, redirect_path: &str, max_retries: u32) -> Result<(), Box> { +pub async fn download_with_retries(url: &str, file_path: &Path, redirect_path: &Path, max_retries: u32) -> Result<(), Box> { let mut attempts = 0; loop { diff --git a/src/main.rs b/src/main.rs index 96f8d90..693a46c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ use clap::{Parser, Subcommand}; use std::path::PathBuf; use std::error::Error; +use std::process; mod utils; mod checksum; @@ -89,19 +90,14 @@ async fn main() -> Result<(), Box> { let temp_file_path = dir_path.join("state.download"); let redirect_file_path = dir_path.join("state.url"); let archive_file_path = dir_path.join("state.zip"); + let unpacked_file_path = dir_path.join("state_downloaded.sql"); let final_file_path = dir_path.join("state.sql"); let backup_file_path = dir_path.join("state.sql.bak"); - let temp_file_str = temp_file_path.to_str().expect("Cannot compose path"); - let redirect_file_str = redirect_file_path.to_str().expect("Cannot compose path"); - let archive_file_str = archive_file_path.to_str().expect("Cannot compose path"); - let final_file_str = final_file_path.to_str().expect("Cannot compose path"); - let backup_file_str = backup_file_path.to_str().expect("Cannot compose path"); - if !archive_file_path.exists() { println!("Downloading the latest database..."); let url = if redirect_file_path.exists() { - std::fs::read_to_string(redirect_file_str)? + std::fs::read_to_string(&redirect_file_path)? } else { let go_path = resolve_path(&go_spacemesh_path).unwrap(); let go_path_str = go_path.to_str().expect("Cannot resolve path to go-spacemesh"); @@ -110,42 +106,60 @@ async fn main() -> Result<(), Box> { url.to_string() }; - if let Err(e) = download_with_retries(&url, temp_file_str, redirect_file_str, max_retries).await { + if let Err(e) = download_with_retries(&url, &temp_file_path, &redirect_file_path, max_retries).await { eprintln!("Failed to download a file after {} attempts: {}", max_retries, e); + process::exit(1); } // Rename `state.download` -> `state.zip` - std::fs::rename(temp_file_str, archive_file_str)?; + std::fs::rename(&temp_file_path, &archive_file_path)?; println!("Archive downloaded!"); - } else { - println!("Archive found..."); - } - - if final_file_path.exists() { - println!("Renaming current state.sql file into state.sql.bak"); - // Rename original State.Sql (backup) - std::fs::rename(final_file_str, backup_file_str).expect("Cannot rename state.sql -> state.sql.bak"); } // Unzip - unpack(archive_file_str, final_file_str) - .await - .expect("Cannot unzip archive"); + match unpack(&archive_file_path, &unpacked_file_path).await { + Ok(_) => { + println!("Archive unpacked successfully"); + }, + Err(b) => { + let dyn_err = b.as_ref(); + let e = dyn_err.downcast_ref::().expect("Cannot read Error message"); + if e.raw_os_error() == Some(28) { + println!("Cannot unpack archive: not enough disk space"); + std::fs::remove_file(&unpacked_file_path)?; + process::exit(2); + } else { + println!("Cannot unpack archive: {}", e); + std::fs::remove_file(&unpacked_file_path)?; + std::fs::remove_file(&archive_file_path)?; + process::exit(3); + } + } + } println!("Checking MD5 checksum..."); let archive_url = String::from_utf8( - std::fs::read(redirect_file_str).expect("Cannot read state.url") + std::fs::read(&redirect_file_path).expect("Cannot read state.url") )?; let md5_expected = download_checksum(&archive_url).await.expect("Cannot download md5"); - let md5_actual = calculate_checksum(final_file_str).expect("Cannot calculate md5"); + let md5_actual = calculate_checksum(&unpacked_file_path).expect("Cannot calculate md5"); + + if md5_actual != md5_expected { + println!("MD5 checksums are not equal. Deleting archive and unpacked state.sql"); + std::fs::remove_file(&unpacked_file_path)?; + std::fs::remove_file(&archive_file_path)?; + process::exit(4); + } - assert_eq!( - md5_actual, md5_expected, - "MD5 checksums are not equal" - ); + if final_file_path.exists() { + println!("Renaming current state.sql file into state.sql.bak"); + // Rename original State.Sql (backup) + std::fs::rename(&final_file_path, &backup_file_path).expect("Cannot rename state.sql -> state.sql.bak"); + } + std::fs::rename(&unpacked_file_path, &final_file_path).expect("Cannot rename downloaded file into state.sql"); - std::fs::remove_file(redirect_file_str)?; - std::fs::remove_file(archive_file_str)?; + std::fs::remove_file(&redirect_file_path)?; + std::fs::remove_file(&archive_file_path)?; println!("Done!"); println!("Now you can run go-spacemesh as usually."); diff --git a/src/zip.rs b/src/zip.rs index fa2a437..0eea035 100644 --- a/src/zip.rs +++ b/src/zip.rs @@ -4,7 +4,7 @@ use std::io::{Write, Read}; use std::path::Path; use zip::ZipArchive; -pub async fn unpack(archive_path: &str, output_path: &str) -> Result<(), Box> { +pub async fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Box> { let file = File::open(archive_path)?; let mut zip = ZipArchive::new(file)?; From 0103411787a1f406dbfe6d0d32ce02a773ca6319 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Fri, 12 Jan 2024 23:14:46 +0700 Subject: [PATCH 11/32] Add error codes in README.md --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 14ec28e..70174c2 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,18 @@ -Quicksync-rs -============ +# Quicksync-rs How to use: ``` quicksync-rs help ``` -In development: +Development: ``` cargo run -- help -``` \ No newline at end of file +``` + +## Exit codes +- `0` - all good +- `1` - failed to download archive within 5 retries (any reason) +- `2` - cannot unpack archive: not enough disk space +- `3` - cannot unpack archive: any other reason +- `4` - invalid checksum \ No newline at end of file From 7ac1e8cab54ebe1bd817ef769f8f88f9946a63c6 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Wed, 31 Jan 2024 23:58:44 +0700 Subject: [PATCH 12/32] refactor: follow @poszu recommendations --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 5 +- src/checksum.rs | 66 +++++++---- src/download.rs | 152 +++++++++++++------------ src/go_spacemesh.rs | 5 +- src/main.rs | 268 +++++++++++++++++++++++--------------------- src/parsers.rs | 17 +++ src/utils.rs | 48 +++++--- src/zip.rs | 15 ++- 10 files changed, 328 insertions(+), 252 deletions(-) create mode 100644 src/parsers.rs diff --git a/Cargo.lock b/Cargo.lock index bfb368c..b620164 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1006,7 +1006,7 @@ dependencies = [ ] [[package]] -name = "quicksync-rs" +name = "quicksync" version = "0.1.0" dependencies = [ "chrono", diff --git a/Cargo.toml b/Cargo.toml index 2112446..15f1b43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "quicksync-rs" +name = "quicksync" version = "0.1.0" edition = "2021" diff --git a/README.md b/README.md index 70174c2..f8b3fa9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ cargo run -- help ## Exit codes - `0` - all good -- `1` - failed to download archive within 5 retries (any reason) +- `1` - failed to download archive within max retries (any reason) - `2` - cannot unpack archive: not enough disk space - `3` - cannot unpack archive: any other reason -- `4` - invalid checksum \ No newline at end of file +- `4` - invalid checksum +- `5` - cannot verify checksum for some reason \ No newline at end of file diff --git a/src/checksum.rs b/src/checksum.rs index 964fb2a..a3d3954 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -1,37 +1,65 @@ use reqwest::Client; use url::Url; -use std::{error::Error, fs::File, io, io::Read, path::Path}; +use std::{fs::File, io::{self, BufReader}, io::{Error, BufRead}, path::Path}; use crate::utils::strip_trailing_newline; -pub async fn download_checksum(url: &str) -> Result> { - let mut u = Url::parse(url)?; +pub async fn download_checksum(url: &Url) -> Result { + let mut u = url.clone(); u.path_segments_mut().expect("Wrong URL").pop().push("state.sql.md5"); let md5_url = u.to_string(); let client = Client::new(); let response = client.get(md5_url) - .send() - .await?; + .send() + .await + .map_err(|e| Error::new(std::io::ErrorKind::Other, e.to_string()))?; if response.status().is_success() { - let md5 = response.text().await?; - let stripped = strip_trailing_newline(&md5); - Ok(stripped.to_string()) + let md5 = response.text().await + .map_err(|e| Error::new( + std::io::ErrorKind::Other, e.to_string()) + )?; + let stripped = strip_trailing_newline(&md5); + Ok(stripped.to_string()) } else { - Err( - Box::new(std::io::Error::new( - std::io::ErrorKind::NotFound, - "Cannot download MD5 checksum" - )) + Err( + std::io::Error::new( + std::io::ErrorKind::NotFound, + "Cannot download MD5 checksum" ) + ) } } pub fn calculate_checksum(file_path: &Path) -> io::Result { - let mut file = File::open(file_path)?; - let mut buffer = Vec::new(); - file.read_to_end(&mut buffer)?; - let hash = md5::compute(buffer); - Ok(format!("{:x}", hash)) -} \ No newline at end of file + let file = File::open(file_path)?; + let mut reader = BufReader::with_capacity(16 * 1024 *1024, file); + let mut hasher = md5::Context::new(); + + loop { + let chunk = reader.fill_buf()?; + if chunk.is_empty() { + break; + } + hasher.consume(chunk); + let chunk_len = chunk.len(); + reader.consume(chunk_len); + } + + let hash = hasher.compute(); + Ok(format!("{:x}", hash)) +} + +pub async fn verify(redirect_file_path: &Path, unpacked_file_path: &Path) -> Result { + let archive_url_str = String::from_utf8(std::fs::read(&redirect_file_path)?) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + + let archive_url = Url::parse(&archive_url_str) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + + let md5_expected = download_checksum(&archive_url).await?; + let md5_actual = calculate_checksum(&unpacked_file_path)?; + + Ok(md5_actual == md5_expected) +} diff --git a/src/download.rs b/src/download.rs index 5072cd6..e640e7f 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,5 +1,6 @@ use reqwest::{Client, header}; use tokio::time::sleep; +use std::collections::VecDeque; use std::error::Error; use std::path::Path; use std::time::Instant; @@ -8,99 +9,102 @@ use std::io::{Seek, SeekFrom, Write}; use futures_util::StreamExt; pub async fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> Result<(), Box> { - let path = Path::new(file_path); - - if let Some(dir) = path.parent() { - create_dir_all(dir).expect("Cannot create directory"); + if let Some(dir) = file_path.parent() { + create_dir_all(dir)?; } let mut file = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(file_path) - .expect("Cannot create file"); + .create(true) + .read(true) + .write(true) + .open(file_path)?; let file_size = file.metadata()?.len(); let client = Client::new(); let response = client.get(url) - .header("Range", format!("bytes={}-", file_size)) - .send() - .await?; + .header("Range", format!("bytes={}-", file_size)) + .send() + .await?; let final_url = response.url().clone(); std::fs::write(redirect_path, final_url.as_str())?; - if response.status().is_success() { - let total_size = response - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|ct_len| ct_len.to_str().ok()) - .and_then(|ct_len| ct_len.parse::().ok()) - .unwrap_or(0) + file_size; - - file.seek(SeekFrom::End(0))?; - let mut stream = response.bytes_stream(); - let start = Instant::now(); - - let mut downloaded: u64 = file_size; - let mut last_reported_progress: i64 = -1; - - while let Some(item) = stream.next().await { - let chunk = item?; - file.write_all(&chunk)?; - downloaded += chunk.len() as u64; - - let elapsed = start.elapsed().as_secs_f64(); - let speed = if elapsed > 0.0 { - downloaded as f64 / elapsed - } else { - 0.0 - }; - let eta = if speed > 0.0 { - (total_size as f64 - downloaded as f64) / speed - } else { - 0.0 - }; - - let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; - if progress > last_reported_progress { - println!("Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", - progress, - downloaded as f64 / 1_024_000.0, - total_size as f64 / 1_024_000.0, - eta); - last_reported_progress = progress; - } - } - Ok(()) - } else { - let err_message = format!("Cannot download: {:?}", response.status()); - - std::fs::remove_file(redirect_path)?; - std::fs::remove_file(file_path)?; - Err( - Box::new(std::io::Error::new( - std::io::ErrorKind::NotFound, - err_message - )) - ) + if !response.status().is_success() { + let err_message = format!("Cannot download: {:?}", response.status()); + + std::fs::remove_file(redirect_path)?; + std::fs::remove_file(file_path)?; + return Err( + Box::new(std::io::Error::new( + std::io::ErrorKind::NotFound, + err_message + )) + ); + } + + let total_size = response + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|ct_len| ct_len.to_str().ok()) + .and_then(|ct_len| ct_len.parse::().ok()) + .unwrap_or(0) + file_size; + + file.seek(SeekFrom::End(0))?; + let mut stream = response.bytes_stream(); + + let start = Instant::now(); + let mut downloaded: u64 = file_size; + let mut newly_downloaded: u64 = 0; + let mut last_reported_progress: i64 = -1; + + let mut measurements = VecDeque::with_capacity(10); + + while let Some(item) = stream.next().await { + let chunk = item?; + file.write_all(&chunk)?; + newly_downloaded += chunk.len() as u64; + downloaded += chunk.len() as u64; + + let elapsed = start.elapsed().as_secs_f64(); + let speed = if elapsed > 0.0 { + newly_downloaded as f64 / elapsed + } else { + 0.0 + }; + measurements.push_back(speed); + let avg_speed = measurements.iter().sum::() / measurements.len() as f64; + let eta = if avg_speed > 0.0 { + (total_size as f64 - downloaded as f64) / avg_speed + } else { + 0.0 + }; + + let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; + if progress > last_reported_progress { + println!("Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", + progress, + downloaded as f64 / 1_024_000.00, + total_size as f64 / 1_024_000.00, + eta); + last_reported_progress = progress; + } } + Ok(()) } pub async fn download_with_retries(url: &str, file_path: &Path, redirect_path: &Path, max_retries: u32) -> Result<(), Box> { let mut attempts = 0; loop { - match download_file(url, file_path, redirect_path).await { - Ok(()) => return Ok(()), - Err(e) if attempts < max_retries => { - eprintln!("Download error: {}. Attemmpt {} / {}", e, attempts + 1, max_retries); - attempts += 1; - sleep(std::time::Duration::from_secs(5)).await; - } - Err(e) => return Err(e), + match download_file(url, file_path, redirect_path).await { + Ok(()) => return Ok(()), + Err(e) if attempts < max_retries => { + eprintln!("Download error: {}. Attemmpt {} / {}", e, attempts + 1, max_retries); + attempts += 1; + sleep(std::time::Duration::from_secs(5)).await; } + Err(e) => return Err(e), + } } } \ No newline at end of file diff --git a/src/go_spacemesh.rs b/src/go_spacemesh.rs index bf8fa24..2c6caf6 100644 --- a/src/go_spacemesh.rs +++ b/src/go_spacemesh.rs @@ -5,9 +5,8 @@ use crate::utils::trim_version; pub fn get_version(path: &str) -> Result> { let output = Command::new(path) - .arg("version") - .output() - .expect("Cannot run go-spacemesh version"); + .arg("version") + .output()?; let version = String::from_utf8(output.stdout)?; let trimmed = trim_version(version.trim()).to_string(); diff --git a/src/main.rs b/src/main.rs index 693a46c..5dab66f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,6 @@ +use chrono::Duration; use clap::{Parser, Subcommand}; +use url::Url; use std::path::PathBuf; use std::error::Error; use std::process; @@ -9,6 +11,7 @@ mod download; mod sql; mod go_spacemesh; mod zip; +mod parsers; use utils::*; use checksum::*; @@ -16,6 +19,7 @@ use download::download_with_retries; use sql::get_last_layer_from_db; use go_spacemesh::get_version; use zip::unpack; +use parsers::*; #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] @@ -26,145 +30,155 @@ struct Cli { #[derive(Subcommand, Debug)] enum Commands { - /// Checks if quicksync is recommended - Check { - /// Path to the node-data directory - #[clap(short = 'd', long)] - node_data: String, - /// Genesis time in ISO format - #[clap(short = 'g', long, default_value = "2023-07-14T08:00:00Z")] - genesis_time: String, - /// Layer duration - #[clap(short = 'l', long, default_value = "5m")] - layer_duration: String, - }, - /// Downloads latest db from official website - Download { - /// Path to the node-data directory - #[clap(short = 'd', long)] - node_data: String, - /// Path to go-spacemesh binary - #[clap(short = 'p', long, default_value = go_spacemesh_default_path())] - go_spacemesh_path: String, - /// URL to download database from. Node version will be appended at the end - #[clap(short = 'u', long, default_value = "https://quicksync.spacemesh.network/")] - download_url: String, - }, + /// Checks if quicksync is recommended + Check { + /// Path to the node-data directory + #[clap(short = 'd', long)] + node_data: PathBuf, + /// Genesis time in ISO format + #[clap(short = 'g', long, default_value = "2023-07-14T08:00:00Z")] + genesis_time: chrono::DateTime, + /// Layer duration + #[clap(short = 'l', long, default_value = "5m", value_parser = parse_duration)] + layer_duration: Duration, + }, + /// Downloads latest db from official website + Download { + /// Path to the node-data directory + #[clap(short = 'd', long)] + node_data: PathBuf, + /// Path to go-spacemesh binary + #[clap(short = 'g', long, default_value = go_spacemesh_default_path())] + go_spacemesh_path: PathBuf, + /// URL to download database from. Node version will be appended at the end + #[clap(short = 'u', long, default_value = "https://quicksync.spacemesh.network/")] + download_url: Url, + /// Maximum retries amount for downloading (or resuming download) if something went wrong + #[clap(short = 'r', long, default_value = "5")] + max_retries: u32, + }, } fn go_spacemesh_default_path() -> &'static str { - #[cfg(target_os = "windows")] - { - "./go-spacemesh.exe" - } - #[cfg(not(target_os = "windows"))] - { - "./go-spacemesh" - } + #[cfg(target_os = "windows")] + { + "./go-spacemesh.exe" + } + #[cfg(not(target_os = "windows"))] + { + "./go-spacemesh" + } } #[tokio::main] async fn main() -> Result<(), Box> { - let cli = Cli::parse(); - let max_retries = 5; - - match cli.command { - Commands::Check { node_data, genesis_time, layer_duration } => { - let dir_path = PathBuf::from(node_data.clone()); - let db_file_path = dir_path.join("state.sql"); - let db_file_str = db_file_path.to_str().expect("Cannot compose path"); - println!("Checking database: {}", &db_file_str); - let db_layer = i64::from(get_last_layer_from_db(db_file_str)?); - let time_layer = calculate_latest_layer(genesis_time, layer_duration)?; - println!("Latest layer in db: {}", db_layer); - println!("Latest calculated layer: {}", time_layer); - if db_layer - time_layer > 100 { - println!("Too far behind"); - } else { - println!("OK!"); - } - Ok(()) + let cli = Cli::parse(); + + match cli.command { + Commands::Check { node_data, genesis_time, layer_duration } => { + let dir_path = node_data.clone(); + let db_file_path = dir_path.join("state.sql"); + let db_file_str = db_file_path.to_str().expect("Cannot compose path"); + println!("Checking database: {}", &db_file_str); + let db_layer = i64::from(get_last_layer_from_db(db_file_str)?); + let time_layer = calculate_latest_layer(genesis_time, layer_duration)?; + println!("Latest layer in db: {}", db_layer); + println!("Latest calculated layer: {}", time_layer); + if time_layer - db_layer > 100 { + println!("Too far behind"); + } else { + println!("OK!"); + } + Ok(()) + } + Commands::Download { node_data, go_spacemesh_path, download_url, max_retries } => { + let dir_path = node_data; + let temp_file_path = dir_path.join("state.download"); + let redirect_file_path = dir_path.join("state.url"); + let archive_file_path = dir_path.join("state.zip"); + let unpacked_file_path = dir_path.join("state_downloaded.sql"); + let final_file_path = dir_path.join("state.sql"); + + // Download archive if needed + if !archive_file_path.exists() { + println!("Downloading the latest database..."); + let url = if redirect_file_path.exists() { + std::fs::read_to_string(&redirect_file_path)? + } else { + let go_path = resolve_path(&go_spacemesh_path).unwrap(); + let go_path_str = go_path.to_str().expect("Cannot resolve path to go-spacemesh"); + let path = format!("{}/state.zip", &get_version(&go_path_str)?); + let url = build_url(&download_url, &path)?; + url.to_string() + }; + + if let Err(e) = download_with_retries(&url, &temp_file_path, &redirect_file_path, max_retries).await { + eprintln!("Failed to download a file after {} attempts: {}", max_retries, e); + process::exit(1); } - Commands::Download { node_data, go_spacemesh_path, download_url } => { - let dir_path = PathBuf::from(node_data); - let temp_file_path = dir_path.join("state.download"); - let redirect_file_path = dir_path.join("state.url"); - let archive_file_path = dir_path.join("state.zip"); - let unpacked_file_path = dir_path.join("state_downloaded.sql"); - let final_file_path = dir_path.join("state.sql"); - let backup_file_path = dir_path.join("state.sql.bak"); - - if !archive_file_path.exists() { - println!("Downloading the latest database..."); - let url = if redirect_file_path.exists() { - std::fs::read_to_string(&redirect_file_path)? - } else { - let go_path = resolve_path(&go_spacemesh_path).unwrap(); - let go_path_str = go_path.to_str().expect("Cannot resolve path to go-spacemesh"); - let path = format!("{}/state.zip", &get_version(&go_path_str)?); - let url = build_url(&download_url, &path)?; - url.to_string() - }; - - if let Err(e) = download_with_retries(&url, &temp_file_path, &redirect_file_path, max_retries).await { - eprintln!("Failed to download a file after {} attempts: {}", max_retries, e); - process::exit(1); - } - - // Rename `state.download` -> `state.zip` - std::fs::rename(&temp_file_path, &archive_file_path)?; - println!("Archive downloaded!"); - } - - // Unzip - match unpack(&archive_file_path, &unpacked_file_path).await { - Ok(_) => { - println!("Archive unpacked successfully"); - }, - Err(b) => { - let dyn_err = b.as_ref(); - let e = dyn_err.downcast_ref::().expect("Cannot read Error message"); - if e.raw_os_error() == Some(28) { - println!("Cannot unpack archive: not enough disk space"); - std::fs::remove_file(&unpacked_file_path)?; - process::exit(2); - } else { - println!("Cannot unpack archive: {}", e); - std::fs::remove_file(&unpacked_file_path)?; - std::fs::remove_file(&archive_file_path)?; - process::exit(3); - } - } - } - - println!("Checking MD5 checksum..."); - let archive_url = String::from_utf8( - std::fs::read(&redirect_file_path).expect("Cannot read state.url") - )?; - let md5_expected = download_checksum(&archive_url).await.expect("Cannot download md5"); - let md5_actual = calculate_checksum(&unpacked_file_path).expect("Cannot calculate md5"); - if md5_actual != md5_expected { - println!("MD5 checksums are not equal. Deleting archive and unpacked state.sql"); - std::fs::remove_file(&unpacked_file_path)?; - std::fs::remove_file(&archive_file_path)?; - process::exit(4); - } + // Rename `state.download` -> `state.zip` + std::fs::rename(&temp_file_path, &archive_file_path)?; + println!("Archive downloaded!"); + } + + // Unzip + match unpack(&archive_file_path, &unpacked_file_path).await { + Ok(_) => { + println!("Archive unpacked successfully"); + }, + Err(e) if e.raw_os_error() == Some(28) => { + println!("Cannot unpack archive: not enough disk space"); + std::fs::remove_file(&unpacked_file_path)?; + process::exit(2); + } + Err(e) => { + println!("Cannot unpack archive: {}", e); + std::fs::remove_file(&unpacked_file_path)?; + std::fs::remove_file(&archive_file_path)?; + process::exit(3); + } + } - if final_file_path.exists() { - println!("Renaming current state.sql file into state.sql.bak"); - // Rename original State.Sql (backup) - std::fs::rename(&final_file_path, &backup_file_path).expect("Cannot rename state.sql -> state.sql.bak"); - } - std::fs::rename(&unpacked_file_path, &final_file_path).expect("Cannot rename downloaded file into state.sql"); + // Verify checksum + println!("Verifying MD5 checksum..."); + match verify(&redirect_file_path, &unpacked_file_path).await { + Ok(true) => { + println!("Checksum is valid"); + } + Ok(false) => { + println!("MD5 checksums are not equal. Deleting archive and unpacked state.sql"); + std::fs::remove_file(&unpacked_file_path)?; + std::fs::remove_file(&archive_file_path)?; + process::exit(4); + } + Err(e) => { + println!("Cannot verify checksum: {}", e.to_string()); + process::exit(5); + } + } + + if final_file_path.exists() { + println!("Backing up current state.sql file"); + match backup_file(&final_file_path) { + Ok(b) => { + let backup_name = b.to_str().expect("Cannot get a path of backed up file"); + println!("File backed up to: {}", backup_name); + } + Err(e) => { + println!("Cannot create a backup file: {}", e.to_string()) + } + } + } + std::fs::rename(&unpacked_file_path, &final_file_path).expect("Cannot rename downloaded file into state.sql"); - std::fs::remove_file(&redirect_file_path)?; - std::fs::remove_file(&archive_file_path)?; + std::fs::remove_file(&redirect_file_path)?; + std::fs::remove_file(&archive_file_path)?; - println!("Done!"); - println!("Now you can run go-spacemesh as usually."); + println!("Done!"); + println!("Now you can run go-spacemesh as usually."); - Ok(()) - } + Ok(()) } + } } diff --git a/src/parsers.rs b/src/parsers.rs new file mode 100644 index 0000000..0002508 --- /dev/null +++ b/src/parsers.rs @@ -0,0 +1,17 @@ +use std::io::{Error, ErrorKind}; + +pub fn parse_duration(v: &str) -> Result { + let ds = v.parse::().map_err( + |e| Error::new( + ErrorKind::InvalidInput, + e.to_string() + ) + )?; + let res = chrono::Duration::from_std(ds.into()) + .map_err(|e| Error::new( + ErrorKind::InvalidInput, + e.to_string() + ))?; + + Ok(res) +} diff --git a/src/utils.rs b/src/utils.rs index b7f1d9e..f873df9 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,27 +1,17 @@ use chrono::{DateTime, Utc, Duration}; -use duration_string::DurationString; use url::{Url, ParseError}; use std::{error::Error, path::PathBuf, env}; -pub fn parse_iso_date(iso_date: &str) -> Result, chrono::ParseError> { - iso_date.parse::>() -} - pub fn strip_trailing_newline(input: &str) -> &str { - input - .strip_suffix("\r\n") - .or(input.strip_suffix("\n")) - .unwrap_or(input) + input.trim_end() } -pub fn calculate_latest_layer(genesis_time: String, layer_duration: String) -> Result> { - let genesis = parse_iso_date(&genesis_time)?; - let delta = Utc::now() - genesis; - let dur = Duration::from_std(DurationString::from_string(layer_duration)?.into())?; - Ok(delta.num_milliseconds() / dur.num_milliseconds()) +pub fn calculate_latest_layer(genesis_time: DateTime, layer_duration: Duration) -> Result> { + let delta = Utc::now() - genesis_time; + Ok(delta.num_milliseconds() / layer_duration.num_milliseconds()) } -pub fn resolve_path(relative_path: &str) -> Result> { +pub fn resolve_path(relative_path: &PathBuf) -> Result> { let current_dir = env::current_dir()?; let resolved_path = current_dir.join(relative_path); Ok(resolved_path) @@ -31,8 +21,32 @@ pub fn trim_version(version: &str) -> &str { version.split('+').next().unwrap_or(version) } -pub fn build_url(base: &str, path: &str) -> Result { - let mut url = Url::parse(base)?; +pub fn build_url(base: &Url, path: &str) -> Result { + let mut url = base.clone(); url.path_segments_mut().expect("cannot be base").extend(path.split('/')); Ok(url) +} + +pub fn backup_file(original_path: &PathBuf) -> Result { + if !original_path.exists() { + return Err( + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "No file to make a backup" + ) + ); + } + + let mut backup_path = original_path.with_extension("sql.bak"); + let mut counter = 1; + + while backup_path.exists() { + let new_name = format!("state.sql.bak.{}", counter); + backup_path = original_path.with_file_name(new_name); + counter += 1; + } + + std::fs::rename(original_path, &backup_path)?; + + Ok(backup_path) } \ No newline at end of file diff --git a/src/zip.rs b/src/zip.rs index 0eea035..1826979 100644 --- a/src/zip.rs +++ b/src/zip.rs @@ -1,15 +1,14 @@ use std::fs::File; -use std::error::Error; -use std::io::{Write, Read}; +use std::io::{Write, Read, Error}; use std::path::Path; use zip::ZipArchive; -pub async fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Box> { +pub async fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Error> { let file = File::open(archive_path)?; let mut zip = ZipArchive::new(file)?; let mut state_sql = zip.by_name("state.sql") - .expect("State.sql file not found in archive"); + .map_err(|e| Error::new(std::io::ErrorKind::NotFound, e.to_string()))?; let outpath = Path::new(output_path); if let Some(p) = outpath.parent() { @@ -30,7 +29,7 @@ pub async fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Box { outfile.write_all(&buffer[..bytes_read])?; @@ -42,15 +41,15 @@ pub async fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Box return Err(Box::new(e)), + Err(e) => return Err(e), } } if last_reported_progress < 100 { - return Err(Box::new(std::io::Error::new( + return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, "Archive was not fully unpacked", - ))); + )); } Ok(()) From 85ddd834b8aaa021ab6de88686ec3ea09696edec Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Thu, 1 Feb 2024 15:09:09 +0700 Subject: [PATCH 13/32] refactor: rustfmt --- rustfmt.toml | 1 + src/checksum.rs | 78 +++++++++++++++++++++++++-------------------- src/download.rs | 68 ++++++++++++++++++++++++--------------- src/go_spacemesh.rs | 6 ++-- src/main.rs | 63 ++++++++++++++++++++++++------------ src/parsers.rs | 16 +++------- src/sql.rs | 14 ++++---- src/utils.rs | 34 +++++++++++--------- src/zip.rs | 47 ++++++++++++++------------- 9 files changed, 185 insertions(+), 142 deletions(-) create mode 100644 rustfmt.toml diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..47874a2 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1 @@ +tab_spaces=2 \ No newline at end of file diff --git a/src/checksum.rs b/src/checksum.rs index a3d3954..c325f44 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -1,65 +1,75 @@ use reqwest::Client; +use std::{ + fs::File, + io::{self, BufReader}, + io::{BufRead, Error}, + path::Path, +}; use url::Url; -use std::{fs::File, io::{self, BufReader}, io::{Error, BufRead}, path::Path}; use crate::utils::strip_trailing_newline; pub async fn download_checksum(url: &Url) -> Result { let mut u = url.clone(); - u.path_segments_mut().expect("Wrong URL").pop().push("state.sql.md5"); + u.path_segments_mut() + .expect("Wrong URL") + .pop() + .push("state.sql.md5"); let md5_url = u.to_string(); let client = Client::new(); - let response = client.get(md5_url) + let response = client + .get(md5_url) .send() .await .map_err(|e| Error::new(std::io::ErrorKind::Other, e.to_string()))?; if response.status().is_success() { - let md5 = response.text().await - .map_err(|e| Error::new( - std::io::ErrorKind::Other, e.to_string()) - )?; + let md5 = response + .text() + .await + .map_err(|e| Error::new(std::io::ErrorKind::Other, e.to_string()))?; let stripped = strip_trailing_newline(&md5); Ok(stripped.to_string()) } else { - Err( - std::io::Error::new( - std::io::ErrorKind::NotFound, - "Cannot download MD5 checksum" - ) - ) + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "Cannot download MD5 checksum", + )) } } pub fn calculate_checksum(file_path: &Path) -> io::Result { - let file = File::open(file_path)?; - let mut reader = BufReader::with_capacity(16 * 1024 *1024, file); - let mut hasher = md5::Context::new(); + let file = File::open(file_path)?; + let mut reader = BufReader::with_capacity(16 * 1024 * 1024, file); + let mut hasher = md5::Context::new(); - loop { - let chunk = reader.fill_buf()?; - if chunk.is_empty() { - break; - } - hasher.consume(chunk); - let chunk_len = chunk.len(); - reader.consume(chunk_len); + loop { + let chunk = reader.fill_buf()?; + if chunk.is_empty() { + break; } + hasher.consume(chunk); + let chunk_len = chunk.len(); + reader.consume(chunk_len); + } - let hash = hasher.compute(); - Ok(format!("{:x}", hash)) + let hash = hasher.compute(); + Ok(format!("{:x}", hash)) } -pub async fn verify(redirect_file_path: &Path, unpacked_file_path: &Path) -> Result { - let archive_url_str = String::from_utf8(std::fs::read(&redirect_file_path)?) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; +pub async fn verify( + redirect_file_path: &Path, + unpacked_file_path: &Path, +) -> Result { + let archive_url_str = String::from_utf8(std::fs::read(&redirect_file_path)?) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; - let archive_url = Url::parse(&archive_url_str) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + let archive_url = Url::parse(&archive_url_str) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; - let md5_expected = download_checksum(&archive_url).await?; - let md5_actual = calculate_checksum(&unpacked_file_path)?; + let md5_expected = download_checksum(&archive_url).await?; + let md5_actual = calculate_checksum(&unpacked_file_path)?; - Ok(md5_actual == md5_expected) + Ok(md5_actual == md5_expected) } diff --git a/src/download.rs b/src/download.rs index e640e7f..d3f3753 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,16 +1,20 @@ -use reqwest::{Client, header}; -use tokio::time::sleep; +use futures_util::StreamExt; +use reqwest::{header, Client}; use std::collections::VecDeque; use std::error::Error; +use std::fs::{create_dir_all, OpenOptions}; +use std::io::{Seek, SeekFrom, Write}; use std::path::Path; use std::time::Instant; -use std::fs::{OpenOptions, create_dir_all}; -use std::io::{Seek, SeekFrom, Write}; -use futures_util::StreamExt; +use tokio::time::sleep; -pub async fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> Result<(), Box> { +pub async fn download_file( + url: &str, + file_path: &Path, + redirect_path: &Path, +) -> Result<(), Box> { if let Some(dir) = file_path.parent() { - create_dir_all(dir)?; + create_dir_all(dir)?; } let mut file = OpenOptions::new() @@ -22,7 +26,8 @@ pub async fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> let file_size = file.metadata()?.len(); let client = Client::new(); - let response = client.get(url) + let response = client + .get(url) .header("Range", format!("bytes={}-", file_size)) .send() .await?; @@ -35,20 +40,19 @@ pub async fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> std::fs::remove_file(redirect_path)?; std::fs::remove_file(file_path)?; - return Err( - Box::new(std::io::Error::new( - std::io::ErrorKind::NotFound, - err_message - )) - ); + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::NotFound, + err_message, + ))); } let total_size = response - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|ct_len| ct_len.to_str().ok()) - .and_then(|ct_len| ct_len.parse::().ok()) - .unwrap_or(0) + file_size; + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|ct_len| ct_len.to_str().ok()) + .and_then(|ct_len| ct_len.parse::().ok()) + .unwrap_or(0) + + file_size; file.seek(SeekFrom::End(0))?; let mut stream = response.bytes_stream(); @@ -82,29 +86,41 @@ pub async fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; if progress > last_reported_progress { - println!("Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", + println!( + "Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", progress, downloaded as f64 / 1_024_000.00, total_size as f64 / 1_024_000.00, - eta); + eta + ); last_reported_progress = progress; } } Ok(()) } -pub async fn download_with_retries(url: &str, file_path: &Path, redirect_path: &Path, max_retries: u32) -> Result<(), Box> { +pub async fn download_with_retries( + url: &str, + file_path: &Path, + redirect_path: &Path, + max_retries: u32, +) -> Result<(), Box> { let mut attempts = 0; loop { match download_file(url, file_path, redirect_path).await { Ok(()) => return Ok(()), Err(e) if attempts < max_retries => { - eprintln!("Download error: {}. Attemmpt {} / {}", e, attempts + 1, max_retries); - attempts += 1; - sleep(std::time::Duration::from_secs(5)).await; + eprintln!( + "Download error: {}. Attemmpt {} / {}", + e, + attempts + 1, + max_retries + ); + attempts += 1; + sleep(std::time::Duration::from_secs(5)).await; } Err(e) => return Err(e), } } -} \ No newline at end of file +} diff --git a/src/go_spacemesh.rs b/src/go_spacemesh.rs index 2c6caf6..703706f 100644 --- a/src/go_spacemesh.rs +++ b/src/go_spacemesh.rs @@ -4,12 +4,10 @@ use std::process::Command; use crate::utils::trim_version; pub fn get_version(path: &str) -> Result> { - let output = Command::new(path) - .arg("version") - .output()?; + let output = Command::new(path).arg("version").output()?; let version = String::from_utf8(output.stdout)?; let trimmed = trim_version(version.trim()).to_string(); Ok(trimmed) -} \ No newline at end of file +} diff --git a/src/main.rs b/src/main.rs index 5dab66f..7ff4d62 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,31 +1,31 @@ use chrono::Duration; use clap::{Parser, Subcommand}; -use url::Url; -use std::path::PathBuf; use std::error::Error; +use std::path::PathBuf; use std::process; +use url::Url; -mod utils; mod checksum; mod download; -mod sql; mod go_spacemesh; -mod zip; mod parsers; +mod sql; +mod utils; +mod zip; -use utils::*; use checksum::*; use download::download_with_retries; -use sql::get_last_layer_from_db; use go_spacemesh::get_version; -use zip::unpack; use parsers::*; +use sql::get_last_layer_from_db; +use utils::*; +use zip::unpack; #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] struct Cli { - #[clap(subcommand)] - command: Commands, + #[clap(subcommand)] + command: Commands, } #[derive(Subcommand, Debug)] @@ -51,7 +51,11 @@ enum Commands { #[clap(short = 'g', long, default_value = go_spacemesh_default_path())] go_spacemesh_path: PathBuf, /// URL to download database from. Node version will be appended at the end - #[clap(short = 'u', long, default_value = "https://quicksync.spacemesh.network/")] + #[clap( + short = 'u', + long, + default_value = "https://quicksync.spacemesh.network/" + )] download_url: Url, /// Maximum retries amount for downloading (or resuming download) if something went wrong #[clap(short = 'r', long, default_value = "5")] @@ -75,7 +79,11 @@ async fn main() -> Result<(), Box> { let cli = Cli::parse(); match cli.command { - Commands::Check { node_data, genesis_time, layer_duration } => { + Commands::Check { + node_data, + genesis_time, + layer_duration, + } => { let dir_path = node_data.clone(); let db_file_path = dir_path.join("state.sql"); let db_file_str = db_file_path.to_str().expect("Cannot compose path"); @@ -85,13 +93,18 @@ async fn main() -> Result<(), Box> { println!("Latest layer in db: {}", db_layer); println!("Latest calculated layer: {}", time_layer); if time_layer - db_layer > 100 { - println!("Too far behind"); + println!("Too far behind"); } else { - println!("OK!"); + println!("OK!"); } Ok(()) } - Commands::Download { node_data, go_spacemesh_path, download_url, max_retries } => { + Commands::Download { + node_data, + go_spacemesh_path, + download_url, + max_retries, + } => { let dir_path = node_data; let temp_file_path = dir_path.join("state.download"); let redirect_file_path = dir_path.join("state.url"); @@ -106,14 +119,21 @@ async fn main() -> Result<(), Box> { std::fs::read_to_string(&redirect_file_path)? } else { let go_path = resolve_path(&go_spacemesh_path).unwrap(); - let go_path_str = go_path.to_str().expect("Cannot resolve path to go-spacemesh"); + let go_path_str = go_path + .to_str() + .expect("Cannot resolve path to go-spacemesh"); let path = format!("{}/state.zip", &get_version(&go_path_str)?); let url = build_url(&download_url, &path)?; url.to_string() }; - if let Err(e) = download_with_retries(&url, &temp_file_path, &redirect_file_path, max_retries).await { - eprintln!("Failed to download a file after {} attempts: {}", max_retries, e); + if let Err(e) = + download_with_retries(&url, &temp_file_path, &redirect_file_path, max_retries).await + { + eprintln!( + "Failed to download a file after {} attempts: {}", + max_retries, e + ); process::exit(1); } @@ -121,12 +141,12 @@ async fn main() -> Result<(), Box> { std::fs::rename(&temp_file_path, &archive_file_path)?; println!("Archive downloaded!"); } - + // Unzip match unpack(&archive_file_path, &unpacked_file_path).await { Ok(_) => { println!("Archive unpacked successfully"); - }, + } Err(e) if e.raw_os_error() == Some(28) => { println!("Cannot unpack archive: not enough disk space"); std::fs::remove_file(&unpacked_file_path)?; @@ -170,7 +190,8 @@ async fn main() -> Result<(), Box> { } } } - std::fs::rename(&unpacked_file_path, &final_file_path).expect("Cannot rename downloaded file into state.sql"); + std::fs::rename(&unpacked_file_path, &final_file_path) + .expect("Cannot rename downloaded file into state.sql"); std::fs::remove_file(&redirect_file_path)?; std::fs::remove_file(&archive_file_path)?; diff --git a/src/parsers.rs b/src/parsers.rs index 0002508..c35c1d0 100644 --- a/src/parsers.rs +++ b/src/parsers.rs @@ -1,17 +1,11 @@ use std::io::{Error, ErrorKind}; pub fn parse_duration(v: &str) -> Result { - let ds = v.parse::().map_err( - |e| Error::new( - ErrorKind::InvalidInput, - e.to_string() - ) - )?; + let ds = v + .parse::() + .map_err(|e| Error::new(ErrorKind::InvalidInput, e.to_string()))?; let res = chrono::Duration::from_std(ds.into()) - .map_err(|e| Error::new( - ErrorKind::InvalidInput, - e.to_string() - ))?; - + .map_err(|e| Error::new(ErrorKind::InvalidInput, e.to_string()))?; + Ok(res) } diff --git a/src/sql.rs b/src/sql.rs index c5cb0d3..3418d05 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -1,18 +1,16 @@ +use rusqlite::{params, Connection}; use std::error::Error; -use rusqlite::{Connection, params}; pub fn get_last_layer_from_db(db_path: &str) -> Result> { let conn = Connection::open(db_path)?; let mut stmt = conn.prepare("SELECT * FROM layers ORDER BY id DESC LIMIT 1")?; - let mut layer_iter = stmt.query_map(params![], |row| { - Ok(row.get::<_, i32>(0)?) - })?; + let mut layer_iter = stmt.query_map(params![], |row| Ok(row.get::<_, i32>(0)?))?; if let Some(result) = layer_iter.next() { - let last_id = result?; - Ok(last_id) + let last_id = result?; + Ok(last_id) } else { - Ok(0) + Ok(0) } -} \ No newline at end of file +} diff --git a/src/utils.rs b/src/utils.rs index f873df9..350bbc1 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,12 +1,15 @@ -use chrono::{DateTime, Utc, Duration}; -use url::{Url, ParseError}; -use std::{error::Error, path::PathBuf, env}; +use chrono::{DateTime, Duration, Utc}; +use std::{env, error::Error, path::PathBuf}; +use url::{ParseError, Url}; pub fn strip_trailing_newline(input: &str) -> &str { input.trim_end() } -pub fn calculate_latest_layer(genesis_time: DateTime, layer_duration: Duration) -> Result> { +pub fn calculate_latest_layer( + genesis_time: DateTime, + layer_duration: Duration, +) -> Result> { let delta = Utc::now() - genesis_time; Ok(delta.num_milliseconds() / layer_duration.num_milliseconds()) } @@ -23,30 +26,31 @@ pub fn trim_version(version: &str) -> &str { pub fn build_url(base: &Url, path: &str) -> Result { let mut url = base.clone(); - url.path_segments_mut().expect("cannot be base").extend(path.split('/')); + url + .path_segments_mut() + .expect("cannot be base") + .extend(path.split('/')); Ok(url) } pub fn backup_file(original_path: &PathBuf) -> Result { if !original_path.exists() { - return Err( - std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "No file to make a backup" - ) - ); + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "No file to make a backup", + )); } let mut backup_path = original_path.with_extension("sql.bak"); let mut counter = 1; while backup_path.exists() { - let new_name = format!("state.sql.bak.{}", counter); - backup_path = original_path.with_file_name(new_name); - counter += 1; + let new_name = format!("state.sql.bak.{}", counter); + backup_path = original_path.with_file_name(new_name); + counter += 1; } std::fs::rename(original_path, &backup_path)?; Ok(backup_path) -} \ No newline at end of file +} diff --git a/src/zip.rs b/src/zip.rs index 1826979..ba235d4 100644 --- a/src/zip.rs +++ b/src/zip.rs @@ -1,5 +1,5 @@ use std::fs::File; -use std::io::{Write, Read, Error}; +use std::io::{Error, Read, Write}; use std::path::Path; use zip::ZipArchive; @@ -7,12 +7,13 @@ pub async fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Error let file = File::open(archive_path)?; let mut zip = ZipArchive::new(file)?; - let mut state_sql = zip.by_name("state.sql") + let mut state_sql = zip + .by_name("state.sql") .map_err(|e| Error::new(std::io::ErrorKind::NotFound, e.to_string()))?; let outpath = Path::new(output_path); if let Some(p) = outpath.parent() { - std::fs::create_dir_all(&p)?; + std::fs::create_dir_all(&p)?; } let mut outfile = File::create(&outpath)?; @@ -24,33 +25,33 @@ pub async fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Error loop { match state_sql.read(&mut buffer) { - Ok(0) => { - if last_reported_progress != 100 { - last_reported_progress = 100; - println!("Unzipping... {}%", last_reported_progress); - } - break; - }, - Ok(bytes_read) => { - outfile.write_all(&buffer[..bytes_read])?; - extracted_size += bytes_read as u64; - - let progress = (extracted_size as f64 / total_size as f64 * 100.0).round() as i64; - if last_reported_progress != progress { - last_reported_progress = progress; - println!("Unzipping... {}%", progress); - } + Ok(0) => { + if last_reported_progress != 100 { + last_reported_progress = 100; + println!("Unzipping... {}%", last_reported_progress); } - Err(e) => return Err(e), + break; + } + Ok(bytes_read) => { + outfile.write_all(&buffer[..bytes_read])?; + extracted_size += bytes_read as u64; + + let progress = (extracted_size as f64 / total_size as f64 * 100.0).round() as i64; + if last_reported_progress != progress { + last_reported_progress = progress; + println!("Unzipping... {}%", progress); + } + } + Err(e) => return Err(e), } } if last_reported_progress < 100 { return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Archive was not fully unpacked", + std::io::ErrorKind::InvalidData, + "Archive was not fully unpacked", )); } Ok(()) -} \ No newline at end of file +} From 5da260e876d6238c14bfe0270c0f148fdea37e89 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Thu, 1 Feb 2024 15:15:16 +0700 Subject: [PATCH 14/32] refactor: clippy --- src/checksum.rs | 4 ++-- src/main.rs | 6 +++--- src/sql.rs | 2 +- src/zip.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/checksum.rs b/src/checksum.rs index c325f44..7f92ebb 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -62,14 +62,14 @@ pub async fn verify( redirect_file_path: &Path, unpacked_file_path: &Path, ) -> Result { - let archive_url_str = String::from_utf8(std::fs::read(&redirect_file_path)?) + let archive_url_str = String::from_utf8(std::fs::read(redirect_file_path)?) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; let archive_url = Url::parse(&archive_url_str) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; let md5_expected = download_checksum(&archive_url).await?; - let md5_actual = calculate_checksum(&unpacked_file_path)?; + let md5_actual = calculate_checksum(unpacked_file_path)?; Ok(md5_actual == md5_expected) } diff --git a/src/main.rs b/src/main.rs index 7ff4d62..c4dd457 100644 --- a/src/main.rs +++ b/src/main.rs @@ -122,7 +122,7 @@ async fn main() -> Result<(), Box> { let go_path_str = go_path .to_str() .expect("Cannot resolve path to go-spacemesh"); - let path = format!("{}/state.zip", &get_version(&go_path_str)?); + let path = format!("{}/state.zip", &get_version(go_path_str)?); let url = build_url(&download_url, &path)?; url.to_string() }; @@ -173,7 +173,7 @@ async fn main() -> Result<(), Box> { process::exit(4); } Err(e) => { - println!("Cannot verify checksum: {}", e.to_string()); + println!("Cannot verify checksum: {}", e); process::exit(5); } } @@ -186,7 +186,7 @@ async fn main() -> Result<(), Box> { println!("File backed up to: {}", backup_name); } Err(e) => { - println!("Cannot create a backup file: {}", e.to_string()) + println!("Cannot create a backup file: {}", e) } } } diff --git a/src/sql.rs b/src/sql.rs index 3418d05..3234cc4 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -5,7 +5,7 @@ pub fn get_last_layer_from_db(db_path: &str) -> Result> { let conn = Connection::open(db_path)?; let mut stmt = conn.prepare("SELECT * FROM layers ORDER BY id DESC LIMIT 1")?; - let mut layer_iter = stmt.query_map(params![], |row| Ok(row.get::<_, i32>(0)?))?; + let mut layer_iter = stmt.query_map(params![], |row| row.get::<_, i32>(0))?; if let Some(result) = layer_iter.next() { let last_id = result?; diff --git a/src/zip.rs b/src/zip.rs index ba235d4..c785cf3 100644 --- a/src/zip.rs +++ b/src/zip.rs @@ -13,9 +13,9 @@ pub async fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Error let outpath = Path::new(output_path); if let Some(p) = outpath.parent() { - std::fs::create_dir_all(&p)?; + std::fs::create_dir_all(p)?; } - let mut outfile = File::create(&outpath)?; + let mut outfile = File::create(outpath)?; let total_size = state_sql.size(); let mut extracted_size: u64 = 0; From debec7a8a09a8cb3c1f0b74f90fd949a97cda1b1 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Thu, 1 Feb 2024 15:21:55 +0700 Subject: [PATCH 15/32] feat: add CI --- .github/workflows/ci.yml | 142 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ba5443b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,142 @@ +on: + pull_request: + push: + tags: + - 'v*' + +name: CI + +jobs: + check: + name: Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - uses: dtolnay/rust-toolchain@1.74.1 + - uses: Swatinem/rust-cache@v2 + - run: cargo check --workspace --all-features + + fmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@1.74.1 + with: + components: rustfmt + - uses: Swatinem/rust-cache@v2 + - run: cargo fmt --all -- --check + + clippy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@master + with: + components: clippy + toolchain: stable + - uses: Swatinem/rust-cache@v2 + - name: Annotate commit with clippy warnings + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + args: --all-features --workspace + + build: + name: Build library + runs-on: ${{ matrix.os }} + needs: + - fmt + - clippy + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + artifact-name: linux + + - os: [self-hosted, linux, arm64] + artifact-name: linux-arm64 + + - os: [self-hosted, macos, arm64] + artifact-name: macos-m1 + + - os: macos-latest + artifact-name: macos + + - os: windows-2019 + artifact-name: windows + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@1.74.1 + - uses: Swatinem/rust-cache@v2 + with: + key: ${{ join( matrix.os, '-' ) }} + + - name: Version suffix (for release only) + id: version + run: echo "suffix=${{ github.ref_type == 'tag' && '-' || ''}}${{ github.ref_type == 'tag' && github.ref || ''}}" >> $GITHUB_OUTPUT + + - name: Install SQLite3 (Linux Arm64) + if: matrix.artifact-name == 'linux-arm64' + run: sudo apt-get install libsqlite3-dev + + - name: Install SQLite3 (Windows) + if: matrix.artifact-name == 'windows' + run: | + choco install -y wget + cd C:\ + mkdir lib + cd lib + wget https://github.com/buggins/ddbc/raw/master/libs/win64/sqlite3.lib + echo "LIB=C:\lib" >> $env:GITHUB_ENV + + - name: Build + run: cargo build --release + + - name: Archive artifacts + uses: actions/upload-artifact@v3 + with: + name: quicksync-${{ matrix.artifact-name }}${{ steps.version.output.suffix }} + path: | + target/release/quicksync${{ matrix.os == 'windows-2019' && '.exe' || '' }} + if-no-files-found: error + + release: + name: Publish release + if: github.event_name == 'push' && github.ref_type == 'tag' + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + path: ./artifacts + - name: List artifacts + run: ls -R ./artifacts + - name: Create a draft release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + draft: true + prerelease: false + - name: Pack artifacts + run: > + mkdir ./assets; + for dir in ./artifacts/*/; do + zip -o -j -r "./assets/$(basename "$dir")-$TAG.zip" "$dir"; + done + env: + TAG: ${{ github.ref_name }} + - name: Upload Release Assets + run: gh release upload $TAG ./assets/*.zip + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} \ No newline at end of file From 511a9c92c5d4f137a4c24c4397ca9b97f1acede5 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Wed, 7 Feb 2024 21:22:17 +0700 Subject: [PATCH 16/32] refactor: use blocking API instead of async --- Cargo.toml | 2 +- src/checksum.rs | 67 ++++++++-------- src/download.rs | 208 ++++++++++++++++++++++++------------------------ src/main.rs | 48 ++++++----- src/sql.rs | 4 +- src/zip.rs | 2 +- 6 files changed, 167 insertions(+), 164 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 15f1b43..eef2595 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ clap = { version="4.4.14", features=["derive"] } duration-string = "0.3.0" futures-util = "0.3.30" md5 = "0.7.0" -reqwest = { version = "0.11.23", features = ["json", "stream"] } +reqwest = { version = "0.11.23", features = ["json", "stream", "blocking"] } rusqlite = "0.30.0" tokio = { version = "1.35.1", features = ["full"] } url = "2.5.0" diff --git a/src/checksum.rs b/src/checksum.rs index 7f92ebb..360d6a9 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -1,42 +1,39 @@ -use reqwest::Client; +use reqwest::blocking::Client; use std::{ - fs::File, - io::{self, BufReader}, - io::{BufRead, Error}, - path::Path, + fs::File, + io::{self, BufReader, BufRead, Error}, + path::Path, }; use url::Url; use crate::utils::strip_trailing_newline; -pub async fn download_checksum(url: &Url) -> Result { - let mut u = url.clone(); - u.path_segments_mut() - .expect("Wrong URL") - .pop() - .push("state.sql.md5"); - let md5_url = u.to_string(); +pub fn download_checksum(url: &Url) -> Result { + let mut u = url.clone(); + u.path_segments_mut() + .expect("Wrong URL") + .pop() + .push("state.sql.md5"); + let md5_url = u.to_string(); - let client = Client::new(); - let response = client - .get(md5_url) - .send() - .await - .map_err(|e| Error::new(std::io::ErrorKind::Other, e.to_string()))?; + let client = Client::new(); + let response = client + .get(md5_url) + .send() + .map_err(|e| Error::new(std::io::ErrorKind::Other, e.to_string()))?; - if response.status().is_success() { - let md5 = response - .text() - .await - .map_err(|e| Error::new(std::io::ErrorKind::Other, e.to_string()))?; - let stripped = strip_trailing_newline(&md5); - Ok(stripped.to_string()) - } else { - Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "Cannot download MD5 checksum", - )) - } + if response.status().is_success() { + let md5 = response + .text() + .map_err(|e| Error::new(std::io::ErrorKind::Other, e.to_string()))?; + let stripped = strip_trailing_newline(&md5); + Ok(stripped.to_string()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "Cannot download MD5 checksum", + )) + } } pub fn calculate_checksum(file_path: &Path) -> io::Result { @@ -58,17 +55,17 @@ pub fn calculate_checksum(file_path: &Path) -> io::Result { Ok(format!("{:x}", hash)) } -pub async fn verify( +pub fn verify( redirect_file_path: &Path, unpacked_file_path: &Path, ) -> Result { let archive_url_str = String::from_utf8(std::fs::read(redirect_file_path)?) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; let archive_url = Url::parse(&archive_url_str) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; - let md5_expected = download_checksum(&archive_url).await?; + let md5_expected = download_checksum(&archive_url)?; let md5_actual = calculate_checksum(unpacked_file_path)?; Ok(md5_actual == md5_expected) diff --git a/src/download.rs b/src/download.rs index d3f3753..f26d8da 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,126 +1,122 @@ -use futures_util::StreamExt; -use reqwest::{header, Client}; +use reqwest::blocking::Client; use std::collections::VecDeque; use std::error::Error; -use std::fs::{create_dir_all, OpenOptions}; -use std::io::{Seek, SeekFrom, Write}; +use std::fs::{self, OpenOptions}; +use std::io::{self, Read, Seek, SeekFrom, Write}; use std::path::Path; use std::time::Instant; -use tokio::time::sleep; -pub async fn download_file( - url: &str, - file_path: &Path, - redirect_path: &Path, -) -> Result<(), Box> { - if let Some(dir) = file_path.parent() { - create_dir_all(dir)?; - } +pub fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> Result<(), Box> { + if let Some(dir) = file_path.parent() { + fs::create_dir_all(dir)?; + } - let mut file = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(file_path)?; - - let file_size = file.metadata()?.len(); - - let client = Client::new(); - let response = client - .get(url) - .header("Range", format!("bytes={}-", file_size)) - .send() - .await?; - let final_url = response.url().clone(); - - std::fs::write(redirect_path, final_url.as_str())?; - - if !response.status().is_success() { - let err_message = format!("Cannot download: {:?}", response.status()); - - std::fs::remove_file(redirect_path)?; - std::fs::remove_file(file_path)?; - return Err(Box::new(std::io::Error::new( - std::io::ErrorKind::NotFound, - err_message, - ))); - } + let mut file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(file_path)?; + + let file_size = file.metadata()?.len(); + + let client = Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build()?; + let mut response = client + .get(url) + .header("Range", format!("bytes={}-", file_size)) + .send()?; - let total_size = response - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|ct_len| ct_len.to_str().ok()) - .and_then(|ct_len| ct_len.parse::().ok()) - .unwrap_or(0) - + file_size; - - file.seek(SeekFrom::End(0))?; - let mut stream = response.bytes_stream(); - - let start = Instant::now(); - let mut downloaded: u64 = file_size; - let mut newly_downloaded: u64 = 0; - let mut last_reported_progress: i64 = -1; - - let mut measurements = VecDeque::with_capacity(10); - - while let Some(item) = stream.next().await { - let chunk = item?; - file.write_all(&chunk)?; - newly_downloaded += chunk.len() as u64; - downloaded += chunk.len() as u64; - - let elapsed = start.elapsed().as_secs_f64(); - let speed = if elapsed > 0.0 { - newly_downloaded as f64 / elapsed - } else { - 0.0 - }; - measurements.push_back(speed); - let avg_speed = measurements.iter().sum::() / measurements.len() as f64; - let eta = if avg_speed > 0.0 { - (total_size as f64 - downloaded as f64) / avg_speed - } else { - 0.0 - }; - - let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; - if progress > last_reported_progress { - println!( - "Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", - progress, - downloaded as f64 / 1_024_000.00, - total_size as f64 / 1_024_000.00, - eta - ); - last_reported_progress = progress; + let final_url = response.url().clone(); + + fs::write(redirect_path, final_url.as_str())?; + + if !response.status().is_success() { + let err_message = format!("Cannot download: {:?}", response.status()); + fs::remove_file(redirect_path)?; + fs::remove_file(file_path)?; + return Err(Box::new(io::Error::new(io::ErrorKind::NotFound, err_message))); } - } - Ok(()) + + let total_size = response + .headers() + .get(reqwest::header::CONTENT_LENGTH) + .and_then(|ct_len| ct_len.to_str().ok()) + .and_then(|ct_len| ct_len.parse::().ok()) + .unwrap_or(0) + + file_size; + + file.seek(SeekFrom::End(0))?; + let mut downloaded: u64 = file_size; + let mut last_reported_progress: i64 = -1; + let start = Instant::now(); + let mut measurements = VecDeque::with_capacity(10); + + let mut buffer = [0; 16 * 1024]; + while let Ok(bytes_read) = response.read(&mut buffer) { + if bytes_read == 0 { + break; + } + file.write_all(&buffer[..bytes_read])?; + downloaded += bytes_read as u64; + + let elapsed = start.elapsed().as_secs_f64(); + let speed = if elapsed > 0.0 { + downloaded as f64 / elapsed + } else { + 0.0 + }; + measurements.push_back(speed); + if measurements.len() > 10 { + measurements.pop_front(); + } + let avg_speed = measurements.iter().sum::() / measurements.len() as f64; + let eta = if avg_speed > 0.0 { + (total_size as f64 - downloaded as f64) / avg_speed + } else { + 0.0 + }; + + let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; + if progress > last_reported_progress { + println!( + "Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", + progress, + downloaded as f64 / 1_024_000.00, + total_size as f64 / 1_024_000.00, + eta + ); + last_reported_progress = progress; + } + } + + println!("Download finished"); + + Ok(()) } -pub async fn download_with_retries( +pub fn download_with_retries( url: &str, file_path: &Path, redirect_path: &Path, max_retries: u32, -) -> Result<(), Box> { +) -> Result<(), Box> { let mut attempts = 0; loop { - match download_file(url, file_path, redirect_path).await { - Ok(()) => return Ok(()), - Err(e) if attempts < max_retries => { - eprintln!( - "Download error: {}. Attemmpt {} / {}", - e, - attempts + 1, - max_retries - ); - attempts += 1; - sleep(std::time::Duration::from_secs(5)).await; + match download_file(url, file_path, redirect_path) { + Ok(()) => return Ok(()), + Err(e) if attempts < max_retries => { + eprintln!( + "Download error: {}. Attempt {} / {}", + e, + attempts + 1, + max_retries + ); + attempts += 1; + std::thread::sleep(std::time::Duration::from_secs(5)); + } + Err(e) => return Err(e), } - Err(e) => return Err(e), - } } } diff --git a/src/main.rs b/src/main.rs index c4dd457..85496ae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,9 +73,7 @@ fn go_spacemesh_default_path() -> &'static str { "./go-spacemesh" } } - -#[tokio::main] -async fn main() -> Result<(), Box> { +fn main() -> Result<(), Box> { let cli = Cli::parse(); match cli.command { @@ -84,20 +82,32 @@ async fn main() -> Result<(), Box> { genesis_time, layer_duration, } => { - let dir_path = node_data.clone(); - let db_file_path = dir_path.join("state.sql"); - let db_file_str = db_file_path.to_str().expect("Cannot compose path"); - println!("Checking database: {}", &db_file_str); - let db_layer = i64::from(get_last_layer_from_db(db_file_str)?); - let time_layer = calculate_latest_layer(genesis_time, layer_duration)?; - println!("Latest layer in db: {}", db_layer); - println!("Latest calculated layer: {}", time_layer); - if time_layer - db_layer > 100 { - println!("Too far behind"); - } else { - println!("OK!"); + let result = { + let dir_path = node_data.clone(); + let db_file_path = dir_path.join("state.sql"); + let db_file_str = db_file_path.to_str().expect("Cannot compose path"); + println!("Checking database: {}", db_file_str); + let db_layer = if db_file_path.exists() { + i64::from(get_last_layer_from_db(&db_file_path)?) + } else { + 0 + }; + let time_layer = calculate_latest_layer(genesis_time, layer_duration)?; + println!("Latest layer in db: {}", db_layer); + println!("Latest calculated layer: {}", time_layer); + if db_layer == 0 { + println!("Database file is not found"); + } else if time_layer - db_layer > 100 { + println!("Too far behind"); + } else { + println!("OK!"); + } + Ok(()) + }; + if result.is_err() { + process::exit(1); } - Ok(()) + result } Commands::Download { node_data, @@ -128,7 +138,7 @@ async fn main() -> Result<(), Box> { }; if let Err(e) = - download_with_retries(&url, &temp_file_path, &redirect_file_path, max_retries).await + download_with_retries(&url, &temp_file_path, &redirect_file_path, max_retries) { eprintln!( "Failed to download a file after {} attempts: {}", @@ -143,7 +153,7 @@ async fn main() -> Result<(), Box> { } // Unzip - match unpack(&archive_file_path, &unpacked_file_path).await { + match unpack(&archive_file_path, &unpacked_file_path) { Ok(_) => { println!("Archive unpacked successfully"); } @@ -162,7 +172,7 @@ async fn main() -> Result<(), Box> { // Verify checksum println!("Verifying MD5 checksum..."); - match verify(&redirect_file_path, &unpacked_file_path).await { + match verify(&redirect_file_path, &unpacked_file_path) { Ok(true) => { println!("Checksum is valid"); } diff --git a/src/sql.rs b/src/sql.rs index 3234cc4..0ae5813 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -1,7 +1,7 @@ use rusqlite::{params, Connection}; -use std::error::Error; +use std::{error::Error, path::PathBuf}; -pub fn get_last_layer_from_db(db_path: &str) -> Result> { +pub fn get_last_layer_from_db(db_path: &PathBuf) -> Result> { let conn = Connection::open(db_path)?; let mut stmt = conn.prepare("SELECT * FROM layers ORDER BY id DESC LIMIT 1")?; diff --git a/src/zip.rs b/src/zip.rs index c785cf3..7d4d29c 100644 --- a/src/zip.rs +++ b/src/zip.rs @@ -3,7 +3,7 @@ use std::io::{Error, Read, Write}; use std::path::Path; use zip::ZipArchive; -pub async fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Error> { +pub fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Error> { let file = File::open(archive_path)?; let mut zip = ZipArchive::new(file)?; From d26cb7233ffdefa7d5da9f86b8b1915a493bbcaf Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Wed, 7 Feb 2024 22:35:31 +0700 Subject: [PATCH 17/32] refactor: use anyhow::Result and don't mess with Errors --- Cargo.lock | 7 ++ Cargo.toml | 1 + src/checksum.rs | 69 +++++++--------- src/download.rs | 189 ++++++++++++++++++++++---------------------- src/go_spacemesh.rs | 4 +- src/main.rs | 15 ++-- src/sql.rs | 7 +- src/utils.rs | 14 ++-- src/zip.rs | 10 +-- 9 files changed, 158 insertions(+), 158 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b620164..f31a0bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,6 +109,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "anyhow" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" + [[package]] name = "autocfg" version = "1.1.0" @@ -1009,6 +1015,7 @@ dependencies = [ name = "quicksync" version = "0.1.0" dependencies = [ + "anyhow", "chrono", "clap", "duration-string", diff --git a/Cargo.toml b/Cargo.toml index eef2595..a1101be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = "1.0.79" chrono = "0.4.31" clap = { version="4.4.14", features=["derive"] } duration-string = "0.3.0" diff --git a/src/checksum.rs b/src/checksum.rs index 360d6a9..f1d8b1c 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -1,42 +1,38 @@ -use reqwest::blocking::Client; +use anyhow::Result; +use reqwest::blocking::{Client, Response}; use std::{ - fs::File, - io::{self, BufReader, BufRead, Error}, - path::Path, + fs::File, + io::{BufRead, BufReader}, + path::Path, }; use url::Url; use crate::utils::strip_trailing_newline; -pub fn download_checksum(url: &Url) -> Result { - let mut u = url.clone(); - u.path_segments_mut() - .expect("Wrong URL") - .pop() - .push("state.sql.md5"); - let md5_url = u.to_string(); - - let client = Client::new(); - let response = client - .get(md5_url) - .send() - .map_err(|e| Error::new(std::io::ErrorKind::Other, e.to_string()))?; - - if response.status().is_success() { - let md5 = response - .text() - .map_err(|e| Error::new(std::io::ErrorKind::Other, e.to_string()))?; - let stripped = strip_trailing_newline(&md5); - Ok(stripped.to_string()) - } else { - Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "Cannot download MD5 checksum", - )) - } +pub fn download_checksum(url: &Url) -> Result { + let mut u = url.clone(); + u.path_segments_mut() + .expect("Wrong URL") + .pop() + .push("state.sql.md5"); + let md5_url = u.to_string(); + + let client = Client::new(); + let response: Response = client.get(md5_url).send()?; + + if response.status().is_success() { + let md5 = response.text()?; + let stripped = strip_trailing_newline(&md5); + Ok(stripped.to_string()) + } else { + anyhow::bail!( + "Cannot download MD5 checksum: status code is {:?}", + response.status() + ); + } } -pub fn calculate_checksum(file_path: &Path) -> io::Result { +pub fn calculate_checksum(file_path: &Path) -> Result { let file = File::open(file_path)?; let mut reader = BufReader::with_capacity(16 * 1024 * 1024, file); let mut hasher = md5::Context::new(); @@ -55,15 +51,10 @@ pub fn calculate_checksum(file_path: &Path) -> io::Result { Ok(format!("{:x}", hash)) } -pub fn verify( - redirect_file_path: &Path, - unpacked_file_path: &Path, -) -> Result { - let archive_url_str = String::from_utf8(std::fs::read(redirect_file_path)?) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; +pub fn verify(redirect_file_path: &Path, unpacked_file_path: &Path) -> Result { + let archive_url_str = String::from_utf8(std::fs::read(redirect_file_path)?)?; - let archive_url = Url::parse(&archive_url_str) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + let archive_url = Url::parse(&archive_url_str)?; let md5_expected = download_checksum(&archive_url)?; let md5_actual = calculate_checksum(unpacked_file_path)?; diff --git a/src/download.rs b/src/download.rs index f26d8da..e21626e 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,98 +1,101 @@ +use anyhow::Result; use reqwest::blocking::Client; use std::collections::VecDeque; -use std::error::Error; use std::fs::{self, OpenOptions}; -use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::io::{Read, Seek, SeekFrom, Write}; use std::path::Path; use std::time::Instant; -pub fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> Result<(), Box> { - if let Some(dir) = file_path.parent() { - fs::create_dir_all(dir)?; - } +pub fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> Result<()> { + if let Some(dir) = file_path.parent() { + fs::create_dir_all(dir)?; + } - let mut file = OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(file_path)?; + let mut file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(file_path)?; - let file_size = file.metadata()?.len(); + let file_size = file.metadata()?.len(); - let client = Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .build()?; - let mut response = client - .get(url) - .header("Range", format!("bytes={}-", file_size)) - .send()?; + let client = Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build()?; + let mut response = client + .get(url) + .header("Range", format!("bytes={}-", file_size)) + .send()?; - let final_url = response.url().clone(); + let final_url = response.url().clone(); - fs::write(redirect_path, final_url.as_str())?; + fs::write(redirect_path, final_url.as_str())?; - if !response.status().is_success() { - let err_message = format!("Cannot download: {:?}", response.status()); - fs::remove_file(redirect_path)?; - fs::remove_file(file_path)?; - return Err(Box::new(io::Error::new(io::ErrorKind::NotFound, err_message))); - } + if !response.status().is_success() { + fs::remove_file(redirect_path)?; + fs::remove_file(file_path)?; + + anyhow::bail!( + "Failed to download: Response status code is {:?}", + response.status() + ); + } - let total_size = response - .headers() - .get(reqwest::header::CONTENT_LENGTH) - .and_then(|ct_len| ct_len.to_str().ok()) - .and_then(|ct_len| ct_len.parse::().ok()) - .unwrap_or(0) - + file_size; - - file.seek(SeekFrom::End(0))?; - let mut downloaded: u64 = file_size; - let mut last_reported_progress: i64 = -1; - let start = Instant::now(); - let mut measurements = VecDeque::with_capacity(10); - - let mut buffer = [0; 16 * 1024]; - while let Ok(bytes_read) = response.read(&mut buffer) { - if bytes_read == 0 { - break; - } - file.write_all(&buffer[..bytes_read])?; - downloaded += bytes_read as u64; - - let elapsed = start.elapsed().as_secs_f64(); - let speed = if elapsed > 0.0 { - downloaded as f64 / elapsed - } else { - 0.0 - }; - measurements.push_back(speed); - if measurements.len() > 10 { - measurements.pop_front(); - } - let avg_speed = measurements.iter().sum::() / measurements.len() as f64; - let eta = if avg_speed > 0.0 { - (total_size as f64 - downloaded as f64) / avg_speed - } else { - 0.0 - }; - - let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; - if progress > last_reported_progress { - println!( - "Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", - progress, - downloaded as f64 / 1_024_000.00, - total_size as f64 / 1_024_000.00, - eta - ); - last_reported_progress = progress; - } + let total_size = response + .headers() + .get(reqwest::header::CONTENT_LENGTH) + .and_then(|ct_len| ct_len.to_str().ok()) + .and_then(|ct_len| ct_len.parse::().ok()) + .unwrap_or(0) + + file_size; + + file.seek(SeekFrom::End(0))?; + let mut downloaded: u64 = file_size; + let mut last_reported_progress: i64 = -1; + let start = Instant::now(); + let mut measurements = VecDeque::with_capacity(10); + + let mut buffer = [0; 16 * 1024]; + while let Ok(bytes_read) = response.read(&mut buffer) { + if bytes_read == 0 { + break; } + file.write_all(&buffer[..bytes_read])?; + downloaded += bytes_read as u64; + + let elapsed = start.elapsed().as_secs_f64(); + let speed = if elapsed > 0.0 { + downloaded as f64 / elapsed + } else { + 0.0 + }; + measurements.push_back(speed); + if measurements.len() > 10 { + measurements.pop_front(); + } + let avg_speed = measurements.iter().sum::() / measurements.len() as f64; + let eta = if avg_speed > 0.0 { + (total_size as f64 - downloaded as f64) / avg_speed + } else { + 0.0 + }; + + let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; + if progress > last_reported_progress { + println!( + "Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", + progress, + downloaded as f64 / 1_024_000.00, + total_size as f64 / 1_024_000.00, + eta + ); + last_reported_progress = progress; + } + } - println!("Download finished"); + println!("Download finished"); - Ok(()) + Ok(()) } pub fn download_with_retries( @@ -100,23 +103,23 @@ pub fn download_with_retries( file_path: &Path, redirect_path: &Path, max_retries: u32, -) -> Result<(), Box> { +) -> Result<()> { let mut attempts = 0; loop { - match download_file(url, file_path, redirect_path) { - Ok(()) => return Ok(()), - Err(e) if attempts < max_retries => { - eprintln!( - "Download error: {}. Attempt {} / {}", - e, - attempts + 1, - max_retries - ); - attempts += 1; - std::thread::sleep(std::time::Duration::from_secs(5)); - } - Err(e) => return Err(e), + match download_file(url, file_path, redirect_path) { + Ok(()) => return Ok(()), + Err(e) if attempts < max_retries => { + eprintln!( + "Download error: {}. Attempt {} / {}", + e, + attempts + 1, + max_retries + ); + attempts += 1; + std::thread::sleep(std::time::Duration::from_secs(5)); } + Err(e) => return Err(e), + } } } diff --git a/src/go_spacemesh.rs b/src/go_spacemesh.rs index 703706f..f2bc38f 100644 --- a/src/go_spacemesh.rs +++ b/src/go_spacemesh.rs @@ -1,9 +1,9 @@ -use std::error::Error; +use anyhow::Result; use std::process::Command; use crate::utils::trim_version; -pub fn get_version(path: &str) -> Result> { +pub fn get_version(path: &str) -> Result { let output = Command::new(path).arg("version").output()?; let version = String::from_utf8(output.stdout)?; diff --git a/src/main.rs b/src/main.rs index 85496ae..2b57e89 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,5 @@ use chrono::Duration; use clap::{Parser, Subcommand}; -use std::error::Error; use std::path::PathBuf; use std::process; use url::Url; @@ -73,7 +72,7 @@ fn go_spacemesh_default_path() -> &'static str { "./go-spacemesh" } } -fn main() -> Result<(), Box> { +fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match cli.command { @@ -157,12 +156,14 @@ fn main() -> Result<(), Box> { Ok(_) => { println!("Archive unpacked successfully"); } - Err(e) if e.raw_os_error() == Some(28) => { - println!("Cannot unpack archive: not enough disk space"); - std::fs::remove_file(&unpacked_file_path)?; - process::exit(2); - } Err(e) => { + if let Some(io_err) = e.downcast_ref::() { + if io_err.raw_os_error() == Some(28) { + println!("Cannot unpack archive: not enough disk space"); + std::fs::remove_file(&unpacked_file_path)?; + process::exit(2); + } + } println!("Cannot unpack archive: {}", e); std::fs::remove_file(&unpacked_file_path)?; std::fs::remove_file(&archive_file_path)?; diff --git a/src/sql.rs b/src/sql.rs index 0ae5813..2072889 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -1,8 +1,9 @@ +use anyhow::{Context, Result}; use rusqlite::{params, Connection}; -use std::{error::Error, path::PathBuf}; +use std::path::PathBuf; -pub fn get_last_layer_from_db(db_path: &PathBuf) -> Result> { - let conn = Connection::open(db_path)?; +pub fn get_last_layer_from_db(db_path: &PathBuf) -> Result { + let conn = Connection::open(db_path).context("Failed to connect to db")?; let mut stmt = conn.prepare("SELECT * FROM layers ORDER BY id DESC LIMIT 1")?; let mut layer_iter = stmt.query_map(params![], |row| row.get::<_, i32>(0))?; diff --git a/src/utils.rs b/src/utils.rs index 350bbc1..d456b36 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,5 +1,6 @@ +use anyhow::Result; use chrono::{DateTime, Duration, Utc}; -use std::{env, error::Error, path::PathBuf}; +use std::{env, path::PathBuf}; use url::{ParseError, Url}; pub fn strip_trailing_newline(input: &str) -> &str { @@ -9,12 +10,12 @@ pub fn strip_trailing_newline(input: &str) -> &str { pub fn calculate_latest_layer( genesis_time: DateTime, layer_duration: Duration, -) -> Result> { +) -> Result { let delta = Utc::now() - genesis_time; Ok(delta.num_milliseconds() / layer_duration.num_milliseconds()) } -pub fn resolve_path(relative_path: &PathBuf) -> Result> { +pub fn resolve_path(relative_path: &PathBuf) -> Result { let current_dir = env::current_dir()?; let resolved_path = current_dir.join(relative_path); Ok(resolved_path) @@ -33,12 +34,9 @@ pub fn build_url(base: &Url, path: &str) -> Result { Ok(url) } -pub fn backup_file(original_path: &PathBuf) -> Result { +pub fn backup_file(original_path: &PathBuf) -> Result { if !original_path.exists() { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "No file to make a backup", - )); + anyhow::bail!("No file to make a backup"); } let mut backup_path = original_path.with_extension("sql.bak"); diff --git a/src/zip.rs b/src/zip.rs index 7d4d29c..c8eb5b5 100644 --- a/src/zip.rs +++ b/src/zip.rs @@ -1,9 +1,10 @@ +use anyhow::Result; use std::fs::File; use std::io::{Error, Read, Write}; use std::path::Path; use zip::ZipArchive; -pub fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Error> { +pub fn unpack(archive_path: &Path, output_path: &Path) -> Result<()> { let file = File::open(archive_path)?; let mut zip = ZipArchive::new(file)?; @@ -42,15 +43,12 @@ pub fn unpack(archive_path: &Path, output_path: &Path) -> Result<(), Error> { println!("Unzipping... {}%", progress); } } - Err(e) => return Err(e), + Err(e) => anyhow::bail!(e), } } if last_reported_progress < 100 { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "Archive was not fully unpacked", - )); + anyhow::bail!("Archive was not fully unpacked"); } Ok(()) From 1b71ad687a75cfb9e42eaa7c9f62fa7044775dc2 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Fri, 9 Feb 2024 15:22:08 +0700 Subject: [PATCH 18/32] tweak: get rid of useless check --- src/main.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2b57e89..826e706 100644 --- a/src/main.rs +++ b/src/main.rs @@ -96,10 +96,6 @@ fn main() -> anyhow::Result<()> { println!("Latest calculated layer: {}", time_layer); if db_layer == 0 { println!("Database file is not found"); - } else if time_layer - db_layer > 100 { - println!("Too far behind"); - } else { - println!("OK!"); } Ok(()) }; From 5ae2127f41c530bc02bf5c0fb1d718c7fbf678f5 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Mon, 12 Feb 2024 18:01:52 +0700 Subject: [PATCH 19/32] feat: check latest layer available in cloud --- Cargo.lock | 39 +++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/main.rs | 33 +++++++++++++++++++++++++++----- src/utils.rs | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 121 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f31a0bd..0000c7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,6 +40,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.16" @@ -1021,6 +1030,7 @@ dependencies = [ "duration-string", "futures-util", "md5", + "regex", "reqwest", "rusqlite", "tokio", @@ -1052,6 +1062,35 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "regex" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "reqwest" version = "0.11.23" diff --git a/Cargo.toml b/Cargo.toml index a1101be..ad23f43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ clap = { version="4.4.14", features=["derive"] } duration-string = "0.3.0" futures-util = "0.3.30" md5 = "0.7.0" +regex = "1.10.3" reqwest = { version = "0.11.23", features = ["json", "stream", "blocking"] } rusqlite = "0.30.0" tokio = { version = "1.35.1", features = ["full"] } diff --git a/src/main.rs b/src/main.rs index 826e706..774c5f2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,6 +27,8 @@ struct Cli { command: Commands, } +const DEFAULT_DOWNLOAD_URL: &str = "https://quicksync.spacemesh.network/"; + #[derive(Subcommand, Debug)] enum Commands { /// Checks if quicksync is recommended @@ -35,11 +37,21 @@ enum Commands { #[clap(short = 'd', long)] node_data: PathBuf, /// Genesis time in ISO format - #[clap(short = 'g', long, default_value = "2023-07-14T08:00:00Z")] + #[clap(short = 't', long, default_value = "2023-07-14T08:00:00Z")] genesis_time: chrono::DateTime, /// Layer duration #[clap(short = 'l', long, default_value = "5m", value_parser = parse_duration)] layer_duration: Duration, + /// Path to go-spacemesh binary + #[clap(short = 'g', long, default_value = go_spacemesh_default_path())] + go_spacemesh_path: PathBuf, + /// URL to download database from. Node version will be appended at the end + #[clap( + short = 'u', + long, + default_value = DEFAULT_DOWNLOAD_URL + )] + download_url: Url, }, /// Downloads latest db from official website Download { @@ -53,7 +65,7 @@ enum Commands { #[clap( short = 'u', long, - default_value = "https://quicksync.spacemesh.network/" + default_value = DEFAULT_DOWNLOAD_URL )] download_url: Url, /// Maximum retries amount for downloading (or resuming download) if something went wrong @@ -80,6 +92,8 @@ fn main() -> anyhow::Result<()> { node_data, genesis_time, layer_duration, + go_spacemesh_path, + download_url, } => { let result = { let dir_path = node_data.clone(); @@ -91,12 +105,21 @@ fn main() -> anyhow::Result<()> { } else { 0 }; - let time_layer = calculate_latest_layer(genesis_time, layer_duration)?; - println!("Latest layer in db: {}", db_layer); - println!("Latest calculated layer: {}", time_layer); if db_layer == 0 { println!("Database file is not found"); } + println!("Latest layer in db: {}", db_layer); + + let time_layer = calculate_latest_layer(genesis_time, layer_duration)?; + println!("Current network layer: {}", time_layer); + + let go_path = resolve_path(&go_spacemesh_path).unwrap(); + let go_path_str = go_path + .to_str() + .expect("Cannot resolve path to go-spacemesh"); + let go_version = get_version(&go_path_str)?; + let quicksync_layer = fetch_latest_available_layer(&download_url, &go_version)?; + println!("Latest layer in cloud: {}", quicksync_layer); Ok(()) }; if result.is_err() { diff --git a/src/utils.rs b/src/utils.rs index d456b36..7b9b2d9 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,5 +1,7 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; use chrono::{DateTime, Duration, Utc}; +use regex::Regex; +use reqwest::blocking::Client; use std::{env, path::PathBuf}; use url::{ParseError, Url}; @@ -52,3 +54,53 @@ pub fn backup_file(original_path: &PathBuf) -> Result { Ok(backup_path) } + +fn extract_number_from_url(url: &Url) -> Result { + let re = Regex::new(r"/(\d+)\.sql\.zip$")?; + let path = url.path(); + let caps = re + .captures(path) + .ok_or_else(|| anyhow!("No numeric value found in URL: {}", url))?; + + let number_str = caps + .get(1) + .ok_or_else(|| anyhow!("No numeric value captured"))? + .as_str(); + let number = number_str.parse::()?; + + Ok(number) +} + +pub fn fetch_latest_available_layer(download_url: &Url, go_version: &str) -> Result { + let client = Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build()?; + + let path = format!("{}/state.zip", go_version); + let url = build_url(&download_url, &path)?; + + let response = client.head(url).send()?; + + let final_url = response.url(); + let num = extract_number_from_url(final_url)?; + + Ok(num) +} + +#[cfg(test)] +mod tests { + use super::*; + use url::Url; + + #[test] + fn test_extract_number_valid() { + let url = Url::parse("https://quicksync-downloads.spacemesh.network/10/61579.sql.zip").unwrap(); + assert_eq!(extract_number_from_url(&url).unwrap(), 61579); + } + + #[test] + fn test_extract_number_invalid() { + let url = Url::parse("https://quicksync.spacemesh.network/state.zip").unwrap(); + assert!(extract_number_from_url(&url).is_err()); + } +} From d070de153f31484db893b5edf48442233d45883a Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Tue, 13 Feb 2024 13:42:56 +0700 Subject: [PATCH 20/32] chore: bump & release version to v0.1.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0000c7f..8021827 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1022,7 +1022,7 @@ dependencies = [ [[package]] name = "quicksync" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index ad23f43..aaec582 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quicksync" -version = "0.1.0" +version = "0.1.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 967fe5939e2e1cfc5fb131c5e01746e597922b7a Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Tue, 13 Feb 2024 14:15:02 +0700 Subject: [PATCH 21/32] feat: bundle sqlite3.dll --- Cargo.lock | 3 ++- Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8021827..fecc9c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -775,6 +775,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" dependencies = [ + "cc", "pkg-config", "vcpkg", ] @@ -1022,7 +1023,7 @@ dependencies = [ [[package]] name = "quicksync" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index aaec582..1c73796 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quicksync" -version = "0.1.1" +version = "0.1.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -14,7 +14,7 @@ futures-util = "0.3.30" md5 = "0.7.0" regex = "1.10.3" reqwest = { version = "0.11.23", features = ["json", "stream", "blocking"] } -rusqlite = "0.30.0" +rusqlite = { version = "0.30.0", features = ["bundled"] } tokio = { version = "1.35.1", features = ["full"] } url = "2.5.0" zip = "0.6.6" From 13ddb75e344064f8baedb00c4716fbdd2c92a307 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Wed, 14 Feb 2024 23:37:55 +0700 Subject: [PATCH 22/32] feat: unpack state.zip despite of hierarchy in archive --- src/zip.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/zip.rs b/src/zip.rs index c8eb5b5..cd2ad38 100644 --- a/src/zip.rs +++ b/src/zip.rs @@ -4,13 +4,28 @@ use std::io::{Error, Read, Write}; use std::path::Path; use zip::ZipArchive; +fn find_file_index_in_archive(archive: &mut ZipArchive, file_name: &str) -> Result { + for i in 0..archive.len() { + let file = archive.by_index(i)?; + if file.name().ends_with(file_name) { + return Ok(i); + } + } + + Err(Error::new( + std::io::ErrorKind::NotFound, + format!("File '{}' not found in archive", file_name), + )) +} + + + pub fn unpack(archive_path: &Path, output_path: &Path) -> Result<()> { let file = File::open(archive_path)?; let mut zip = ZipArchive::new(file)?; - let mut state_sql = zip - .by_name("state.sql") - .map_err(|e| Error::new(std::io::ErrorKind::NotFound, e.to_string()))?; + let file_index = find_file_index_in_archive(&mut zip, "state.sql")?; + let mut state_sql = zip.by_index(file_index)?; let outpath = Path::new(output_path); if let Some(p) = outpath.parent() { From 7f892d8d9e28b436829e8bea28c5c9141a7953b1 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Wed, 14 Feb 2024 23:38:15 +0700 Subject: [PATCH 23/32] feat: display error messages in stderr --- src/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 774c5f2..bcc51b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -178,12 +178,12 @@ fn main() -> anyhow::Result<()> { Err(e) => { if let Some(io_err) = e.downcast_ref::() { if io_err.raw_os_error() == Some(28) { - println!("Cannot unpack archive: not enough disk space"); + eprintln!("Cannot unpack archive: not enough disk space"); std::fs::remove_file(&unpacked_file_path)?; process::exit(2); } } - println!("Cannot unpack archive: {}", e); + eprintln!("Cannot unpack archive: {}", e); std::fs::remove_file(&unpacked_file_path)?; std::fs::remove_file(&archive_file_path)?; process::exit(3); @@ -197,13 +197,13 @@ fn main() -> anyhow::Result<()> { println!("Checksum is valid"); } Ok(false) => { - println!("MD5 checksums are not equal. Deleting archive and unpacked state.sql"); + eprintln!("MD5 checksums are not equal. Deleting archive and unpacked state.sql"); std::fs::remove_file(&unpacked_file_path)?; std::fs::remove_file(&archive_file_path)?; process::exit(4); } Err(e) => { - println!("Cannot verify checksum: {}", e); + eprintln!("Cannot verify checksum: {}", e); process::exit(5); } } @@ -216,7 +216,7 @@ fn main() -> anyhow::Result<()> { println!("File backed up to: {}", backup_name); } Err(e) => { - println!("Cannot create a backup file: {}", e) + eprintln!("Cannot create a backup file: {}", e) } } } From 2190d7d26cc8e64ffc70f3c111eb652de1037230 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Wed, 14 Feb 2024 23:38:47 +0700 Subject: [PATCH 24/32] Release v0.1.3 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fecc9c5..8e52448 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1023,7 +1023,7 @@ dependencies = [ [[package]] name = "quicksync" -version = "0.1.2" +version = "0.1.3" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 1c73796..5b7649b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quicksync" -version = "0.1.2" +version = "0.1.3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 4f8ec39b9147e78561009143ce3e3a94a7910402 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Thu, 15 Feb 2024 03:39:31 +0700 Subject: [PATCH 25/32] fix: download checksum from correct url --- src/checksum.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/checksum.rs b/src/checksum.rs index f1d8b1c..43186e9 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -9,13 +9,18 @@ use url::Url; use crate::utils::strip_trailing_newline; +fn replace_sql_zip_with_md5(url: &Url) -> Result { + let url_str = url.as_str(); + if url_str.ends_with(".sql.zip") { + let new_url_str = url_str.replace(".sql.zip", ".sql.md5"); + Ok(Url::parse(&new_url_str)?) + } else { + anyhow::bail!("URL does not end with .sql.zip") + } +} + pub fn download_checksum(url: &Url) -> Result { - let mut u = url.clone(); - u.path_segments_mut() - .expect("Wrong URL") - .pop() - .push("state.sql.md5"); - let md5_url = u.to_string(); + let md5_url = replace_sql_zip_with_md5(&url)?; let client = Client::new(); let response: Response = client.get(md5_url).send()?; From 53cf403f0c0b45cb37a7241bcfa262595f5f64d6 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Thu, 15 Feb 2024 03:39:46 +0700 Subject: [PATCH 26/32] fix: handle download errors properly --- src/download.rs | 79 +++++++++++++++++++++++++++---------------------- src/main.rs | 4 ++- 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/src/download.rs b/src/download.rs index e21626e..c5afd73 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; use reqwest::blocking::Client; use std::collections::VecDeque; use std::fs::{self, OpenOptions}; @@ -56,40 +56,47 @@ pub fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> Resul let mut measurements = VecDeque::with_capacity(10); let mut buffer = [0; 16 * 1024]; - while let Ok(bytes_read) = response.read(&mut buffer) { - if bytes_read == 0 { - break; - } - file.write_all(&buffer[..bytes_read])?; - downloaded += bytes_read as u64; - - let elapsed = start.elapsed().as_secs_f64(); - let speed = if elapsed > 0.0 { - downloaded as f64 / elapsed - } else { - 0.0 - }; - measurements.push_back(speed); - if measurements.len() > 10 { - measurements.pop_front(); - } - let avg_speed = measurements.iter().sum::() / measurements.len() as f64; - let eta = if avg_speed > 0.0 { - (total_size as f64 - downloaded as f64) / avg_speed - } else { - 0.0 - }; - - let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; - if progress > last_reported_progress { - println!( - "Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", - progress, - downloaded as f64 / 1_024_000.00, - total_size as f64 / 1_024_000.00, - eta - ); - last_reported_progress = progress; + loop { + match response.read(&mut buffer) { + Ok(0) => { + break; + } + Ok(bytes_read) => { + file.write_all(&buffer[..bytes_read])?; + downloaded += bytes_read as u64; + + let elapsed = start.elapsed().as_secs_f64(); + let speed = if elapsed > 0.0 { + downloaded as f64 / elapsed + } else { + 0.0 + }; + measurements.push_back(speed); + if measurements.len() > 10 { + measurements.pop_front(); + } + let avg_speed = measurements.iter().sum::() / measurements.len() as f64; + let eta = if avg_speed > 0.0 { + (total_size as f64 - downloaded as f64) / avg_speed + } else { + 0.0 + }; + + let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; + if progress > last_reported_progress { + println!( + "Downloading... {:.2}% ({:.2} MB/{:.2} MB) ETA: {:.0} sec", + progress, + downloaded as f64 / 1_024_000.00, + total_size as f64 / 1_024_000.00, + eta + ); + last_reported_progress = progress; + } + } + Err(e) => { + return Err(anyhow!(e)); + } } } @@ -119,7 +126,7 @@ pub fn download_with_retries( attempts += 1; std::thread::sleep(std::time::Duration::from_secs(5)); } - Err(e) => return Err(e), + Err(e) => return Err(anyhow!(e)), } } } diff --git a/src/main.rs b/src/main.rs index bcc51b3..1e241af 100644 --- a/src/main.rs +++ b/src/main.rs @@ -69,7 +69,7 @@ enum Commands { )] download_url: Url, /// Maximum retries amount for downloading (or resuming download) if something went wrong - #[clap(short = 'r', long, default_value = "5")] + #[clap(short = 'r', long, default_value = "10")] max_retries: u32, }, } @@ -186,6 +186,7 @@ fn main() -> anyhow::Result<()> { eprintln!("Cannot unpack archive: {}", e); std::fs::remove_file(&unpacked_file_path)?; std::fs::remove_file(&archive_file_path)?; + std::fs::remove_file(&redirect_file_path)?; process::exit(3); } } @@ -200,6 +201,7 @@ fn main() -> anyhow::Result<()> { eprintln!("MD5 checksums are not equal. Deleting archive and unpacked state.sql"); std::fs::remove_file(&unpacked_file_path)?; std::fs::remove_file(&archive_file_path)?; + std::fs::remove_file(&redirect_file_path)?; process::exit(4); } Err(e) => { From f5953adf054106b4463f09e9f80bcbb884eda5e2 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Thu, 15 Feb 2024 14:32:24 +0700 Subject: [PATCH 27/32] chore: rustfmt --- src/checksum.rs | 6 +++--- src/download.rs | 4 ++-- src/zip.rs | 19 ++++++++++--------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/checksum.rs b/src/checksum.rs index 43186e9..d603db9 100644 --- a/src/checksum.rs +++ b/src/checksum.rs @@ -12,10 +12,10 @@ use crate::utils::strip_trailing_newline; fn replace_sql_zip_with_md5(url: &Url) -> Result { let url_str = url.as_str(); if url_str.ends_with(".sql.zip") { - let new_url_str = url_str.replace(".sql.zip", ".sql.md5"); - Ok(Url::parse(&new_url_str)?) + let new_url_str = url_str.replace(".sql.zip", ".sql.md5"); + Ok(Url::parse(&new_url_str)?) } else { - anyhow::bail!("URL does not end with .sql.zip") + anyhow::bail!("URL does not end with .sql.zip") } } diff --git a/src/download.rs b/src/download.rs index c5afd73..dba38cc 100644 --- a/src/download.rs +++ b/src/download.rs @@ -64,7 +64,7 @@ pub fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> Resul Ok(bytes_read) => { file.write_all(&buffer[..bytes_read])?; downloaded += bytes_read as u64; - + let elapsed = start.elapsed().as_secs_f64(); let speed = if elapsed > 0.0 { downloaded as f64 / elapsed @@ -81,7 +81,7 @@ pub fn download_file(url: &str, file_path: &Path, redirect_path: &Path) -> Resul } else { 0.0 }; - + let progress = (downloaded as f64 / total_size as f64 * 100.0).round() as i64; if progress > last_reported_progress { println!( diff --git a/src/zip.rs b/src/zip.rs index cd2ad38..5dcef0a 100644 --- a/src/zip.rs +++ b/src/zip.rs @@ -4,22 +4,23 @@ use std::io::{Error, Read, Write}; use std::path::Path; use zip::ZipArchive; -fn find_file_index_in_archive(archive: &mut ZipArchive, file_name: &str) -> Result { +fn find_file_index_in_archive( + archive: &mut ZipArchive, + file_name: &str, +) -> Result { for i in 0..archive.len() { - let file = archive.by_index(i)?; - if file.name().ends_with(file_name) { - return Ok(i); - } + let file = archive.by_index(i)?; + if file.name().ends_with(file_name) { + return Ok(i); + } } Err(Error::new( - std::io::ErrorKind::NotFound, - format!("File '{}' not found in archive", file_name), + std::io::ErrorKind::NotFound, + format!("File '{}' not found in archive", file_name), )) } - - pub fn unpack(archive_path: &Path, output_path: &Path) -> Result<()> { let file = File::open(archive_path)?; let mut zip = ZipArchive::new(file)?; From 33d16e6ba37a97dae75e306218060507a4d44f09 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Thu, 15 Feb 2024 14:18:26 +0700 Subject: [PATCH 28/32] release v0.1.4 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e52448..2b3ee70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1023,7 +1023,7 @@ dependencies = [ [[package]] name = "quicksync" -version = "0.1.3" +version = "0.1.4" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 5b7649b..9b45cfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quicksync" -version = "0.1.3" +version = "0.1.4" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From afa04c82c2ae29c3301d05ce58335c3da7d69c5c Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Sat, 17 Feb 2024 00:02:45 +0700 Subject: [PATCH 29/32] fix: handle properly when db is corrupted --- src/main.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 1e241af..656b11e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -101,13 +101,17 @@ fn main() -> anyhow::Result<()> { let db_file_str = db_file_path.to_str().expect("Cannot compose path"); println!("Checking database: {}", db_file_str); let db_layer = if db_file_path.exists() { - i64::from(get_last_layer_from_db(&db_file_path)?) + i64::from(get_last_layer_from_db(&db_file_path).or_else( + |err| { + eprintln!("{}", err); + println!("Cannot read database, trating it as empty database"); + return Ok::(0); + } + )?) } else { + println!("Database file is not found"); 0 }; - if db_layer == 0 { - println!("Database file is not found"); - } println!("Latest layer in db: {}", db_layer); let time_layer = calculate_latest_layer(genesis_time, layer_duration)?; From 850f9d42714ba4c4d27a2b4e15a1fb806fcf24a1 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Sat, 17 Feb 2024 00:54:16 +0700 Subject: [PATCH 30/32] Release v0.1.5 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b3ee70..d62e976 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1023,7 +1023,7 @@ dependencies = [ [[package]] name = "quicksync" -version = "0.1.4" +version = "0.1.5" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 9b45cfd..1495f20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quicksync" -version = "0.1.4" +version = "0.1.5" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 60050ba2a8f4eaf3c81f9105e6733e62bcdb9ccc Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Tue, 20 Feb 2024 13:51:54 +0700 Subject: [PATCH 31/32] refactor: follow recommendations and back up wal file --- Cargo.lock | 64 ------------------------------------- Cargo.toml | 2 -- README.md | 3 +- src/download.rs | 2 +- src/main.rs | 51 +++++++++++++++++------------ src/reader_with_progress.rs | 34 ++++++++++++++++++++ src/utils.rs | 8 ++--- src/zip.rs | 55 +++++++++++-------------------- 8 files changed, 90 insertions(+), 129 deletions(-) create mode 100644 src/reader_with_progress.rs diff --git a/Cargo.lock b/Cargo.lock index d62e976..391c6d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -786,16 +786,6 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" -[[package]] -name = "lock_api" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.20" @@ -936,29 +926,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.48.5", -] - [[package]] name = "password-hash" version = "0.4.2" @@ -1029,12 +996,10 @@ dependencies = [ "chrono", "clap", "duration-string", - "futures-util", "md5", "regex", "reqwest", "rusqlite", - "tokio", "url", "zip", ] @@ -1180,12 +1145,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "security-framework" version = "2.9.2" @@ -1274,15 +1233,6 @@ dependencies = [ "digest", ] -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - [[package]] name = "slab" version = "0.4.9" @@ -1409,25 +1359,11 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot", "pin-project-lite", - "signal-hook-registry", "socket2", - "tokio-macros", "windows-sys 0.48.0", ] -[[package]] -name = "tokio-macros" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tokio-native-tls" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 1495f20..822d13e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,11 +10,9 @@ anyhow = "1.0.79" chrono = "0.4.31" clap = { version="4.4.14", features=["derive"] } duration-string = "0.3.0" -futures-util = "0.3.30" md5 = "0.7.0" regex = "1.10.3" reqwest = { version = "0.11.23", features = ["json", "stream", "blocking"] } rusqlite = { version = "0.30.0", features = ["bundled"] } -tokio = { version = "1.35.1", features = ["full"] } url = "2.5.0" zip = "0.6.6" diff --git a/README.md b/README.md index f8b3fa9..8368494 100644 --- a/README.md +++ b/README.md @@ -16,4 +16,5 @@ cargo run -- help - `2` - cannot unpack archive: not enough disk space - `3` - cannot unpack archive: any other reason - `4` - invalid checksum -- `5` - cannot verify checksum for some reason \ No newline at end of file +- `5` - cannot verify checksum for some reason +- `6` - cannot create a backup file \ No newline at end of file diff --git a/src/download.rs b/src/download.rs index dba38cc..8d45e3f 100644 --- a/src/download.rs +++ b/src/download.rs @@ -117,7 +117,7 @@ pub fn download_with_retries( match download_file(url, file_path, redirect_path) { Ok(()) => return Ok(()), Err(e) if attempts < max_retries => { - eprintln!( + println!( "Download error: {}. Attempt {} / {}", e, attempts + 1, diff --git a/src/main.rs b/src/main.rs index 656b11e..bb34c10 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod checksum; mod download; mod go_spacemesh; mod parsers; +mod reader_with_progress; mod sql; mod utils; mod zip; @@ -84,6 +85,26 @@ fn go_spacemesh_default_path() -> &'static str { "./go-spacemesh" } } + +fn backup_or_fail(file_path: &PathBuf) -> () { + if file_path.exists() { + println!( + "Backing up file: {}", + file_path.file_name().unwrap().to_str().unwrap() + ); + match backup_file(&file_path) { + Ok(b) => { + let backup_name = b.to_str().expect("Cannot get a path of backed up file"); + println!("File backed up to: {}", backup_name); + } + Err(e) => { + eprintln!("Cannot create a backup file: {}", e); + process::exit(6); + } + } + } +} + fn main() -> anyhow::Result<()> { let cli = Cli::parse(); @@ -101,13 +122,11 @@ fn main() -> anyhow::Result<()> { let db_file_str = db_file_path.to_str().expect("Cannot compose path"); println!("Checking database: {}", db_file_str); let db_layer = if db_file_path.exists() { - i64::from(get_last_layer_from_db(&db_file_path).or_else( - |err| { - eprintln!("{}", err); - println!("Cannot read database, trating it as empty database"); - return Ok::(0); - } - )?) + i64::from(get_last_layer_from_db(&db_file_path).or_else(|err| { + eprintln!("{}", err); + println!("Cannot read database, trating it as empty database"); + return Ok::(0); + })?) } else { println!("Database file is not found"); 0 @@ -143,6 +162,7 @@ fn main() -> anyhow::Result<()> { let archive_file_path = dir_path.join("state.zip"); let unpacked_file_path = dir_path.join("state_downloaded.sql"); let final_file_path = dir_path.join("state.sql"); + let wal_file_path = dir_path.join("state.sql-wal"); // Download archive if needed if !archive_file_path.exists() { @@ -155,7 +175,7 @@ fn main() -> anyhow::Result<()> { .to_str() .expect("Cannot resolve path to go-spacemesh"); let path = format!("{}/state.zip", &get_version(go_path_str)?); - let url = build_url(&download_url, &path)?; + let url = build_url(&download_url, &path); url.to_string() }; @@ -214,18 +234,9 @@ fn main() -> anyhow::Result<()> { } } - if final_file_path.exists() { - println!("Backing up current state.sql file"); - match backup_file(&final_file_path) { - Ok(b) => { - let backup_name = b.to_str().expect("Cannot get a path of backed up file"); - println!("File backed up to: {}", backup_name); - } - Err(e) => { - eprintln!("Cannot create a backup file: {}", e) - } - } - } + backup_or_fail(&final_file_path); + backup_or_fail(&wal_file_path); + std::fs::rename(&unpacked_file_path, &final_file_path) .expect("Cannot rename downloaded file into state.sql"); diff --git a/src/reader_with_progress.rs b/src/reader_with_progress.rs new file mode 100644 index 0000000..765cb2e --- /dev/null +++ b/src/reader_with_progress.rs @@ -0,0 +1,34 @@ +use std::io::Read; + +pub struct ReaderWithProgress { + reader: R, + total: u64, + extracted: u64, + last_reported_progress: u64, +} + +impl ReaderWithProgress { + pub fn new(reader: R, total_size: u64) -> ReaderWithProgress { + ReaderWithProgress { + reader, + total: total_size, + extracted: 0, + last_reported_progress: 0, + } + } +} + +impl Read for ReaderWithProgress { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let bytes_read = self.reader.read(buf)?; + self.extracted += bytes_read as u64; + + let progress = (self.extracted as f64 / self.total as f64 * 100.0).round() as u64; + if self.last_reported_progress != progress { + self.last_reported_progress = progress; + println!("Unzipping... {}%", progress); + } + + Ok(bytes_read) + } +} diff --git a/src/utils.rs b/src/utils.rs index 7b9b2d9..c00969f 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -3,7 +3,7 @@ use chrono::{DateTime, Duration, Utc}; use regex::Regex; use reqwest::blocking::Client; use std::{env, path::PathBuf}; -use url::{ParseError, Url}; +use url::Url; pub fn strip_trailing_newline(input: &str) -> &str { input.trim_end() @@ -27,13 +27,13 @@ pub fn trim_version(version: &str) -> &str { version.split('+').next().unwrap_or(version) } -pub fn build_url(base: &Url, path: &str) -> Result { +pub fn build_url(base: &Url, path: &str) -> Url { let mut url = base.clone(); url .path_segments_mut() .expect("cannot be base") .extend(path.split('/')); - Ok(url) + url } pub fn backup_file(original_path: &PathBuf) -> Result { @@ -77,7 +77,7 @@ pub fn fetch_latest_available_layer(download_url: &Url, go_version: &str) -> Res .build()?; let path = format!("{}/state.zip", go_version); - let url = build_url(&download_url, &path)?; + let url = build_url(&download_url, &path); let response = client.head(url).send()?; diff --git a/src/zip.rs b/src/zip.rs index 5dcef0a..18462f4 100644 --- a/src/zip.rs +++ b/src/zip.rs @@ -1,19 +1,27 @@ use anyhow::Result; use std::fs::File; -use std::io::{Error, Read, Write}; +use std::io::{BufReader, Error}; use std::path::Path; +use zip::read::ZipFile; use zip::ZipArchive; -fn find_file_index_in_archive( - archive: &mut ZipArchive, +use crate::reader_with_progress::ReaderWithProgress; + +fn find_file_in_archive<'a>( + archive: &'a mut ZipArchive, file_name: &str, -) -> Result { +) -> Result, Error> { + let mut found_idx = None; for i in 0..archive.len() { let file = archive.by_index(i)?; if file.name().ends_with(file_name) { - return Ok(i); + found_idx = Some(i); + break; } } + if let Some(idx) = found_idx { + return Ok(archive.by_index(idx)?); + } Err(Error::new( std::io::ErrorKind::NotFound, @@ -25,8 +33,7 @@ pub fn unpack(archive_path: &Path, output_path: &Path) -> Result<()> { let file = File::open(archive_path)?; let mut zip = ZipArchive::new(file)?; - let file_index = find_file_index_in_archive(&mut zip, "state.sql")?; - let mut state_sql = zip.by_index(file_index)?; + let state_sql: ZipFile = find_file_in_archive(&mut zip, "state.sql")?; let outpath = Path::new(output_path); if let Some(p) = outpath.parent() { @@ -35,37 +42,11 @@ pub fn unpack(archive_path: &Path, output_path: &Path) -> Result<()> { let mut outfile = File::create(outpath)?; let total_size = state_sql.size(); - let mut extracted_size: u64 = 0; - let mut buffer = [0; 4096]; - - let mut last_reported_progress: i64 = -1; - - loop { - match state_sql.read(&mut buffer) { - Ok(0) => { - if last_reported_progress != 100 { - last_reported_progress = 100; - println!("Unzipping... {}%", last_reported_progress); - } - break; - } - Ok(bytes_read) => { - outfile.write_all(&buffer[..bytes_read])?; - extracted_size += bytes_read as u64; + let mut reader = + ReaderWithProgress::new(BufReader::with_capacity(1024 * 1024, state_sql), total_size); - let progress = (extracted_size as f64 / total_size as f64 * 100.0).round() as i64; - if last_reported_progress != progress { - last_reported_progress = progress; - println!("Unzipping... {}%", progress); - } - } - Err(e) => anyhow::bail!(e), - } - } - - if last_reported_progress < 100 { - anyhow::bail!("Archive was not fully unpacked"); - } + std::io::copy(&mut reader, &mut outfile)?; + println!("Unzipping... 100%"); Ok(()) } From 28aafad479bb22cb96b0e9108d4571e64f202389 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Tue, 20 Feb 2024 14:35:35 +0700 Subject: [PATCH 32/32] Release v0.1.6 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 391c6d5..b05cf1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -990,7 +990,7 @@ dependencies = [ [[package]] name = "quicksync" -version = "0.1.5" +version = "0.1.6" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 822d13e..babebd0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quicksync" -version = "0.1.5" +version = "0.1.6" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html