diff --git a/Cargo.lock b/Cargo.lock index 4f56ad7..6e7ca51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,9 +97,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" @@ -146,9 +146,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.91" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" [[package]] name = "arrayref" @@ -186,7 +186,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "synstructure", ] @@ -198,7 +198,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -207,25 +207,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "155a5a185e42c6b77ac7b88a15143d930a9e9727a5b7b77eed417404ab15c247" -[[package]] -name = "assert-json-diff" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "async-convert" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d416feee97712e43152cd42874de162b8f9b77295b1c85e5d92725cc8310bae" -dependencies = [ - "async-trait", -] - [[package]] name = "async-io" version = "2.3.4" @@ -256,43 +237,6 @@ dependencies = [ "pin-project-lite 0.2.15", ] -[[package]] -name = "async-openai" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6db3286b4f52b6556ac5208fb575d035eca61a2bf40d7e75d1db2733ffc599f" -dependencies = [ - "async-convert", - "backoff", - "base64 0.22.1", - "bytes 1.8.0", - "derive_builder", - "eventsource-stream", - "futures", - "rand 0.8.5", - "reqwest 0.12.9", - "reqwest-eventsource", - "secrecy", - "serde", - "serde_json", - "thiserror", - "tokio 1.41.0", - "tokio-stream", - "tokio-util 0.7.12", - "tracing", -] - -[[package]] -name = "async-recursion" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.85", -] - [[package]] name = "async-stream" version = "0.3.6" @@ -312,7 +256,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -323,7 +267,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -376,7 +320,7 @@ dependencies = [ "derive_utils", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -385,20 +329,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "backoff" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" -dependencies = [ - "futures-core", - "getrandom 0.2.15", - "instant", - "pin-project-lite 0.2.15", - "rand 0.8.5", - "tokio 1.41.0", -] - [[package]] name = "backtrace" version = "0.3.74" @@ -426,12 +356,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -507,17 +431,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "bstr" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" -dependencies = [ - "memchr", - "regex-automata", - "serde", -] - [[package]] name = "bumpalo" version = "3.16.0" @@ -553,9 +466,9 @@ checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad" [[package]] name = "cc" -version = "1.1.31" +version = "1.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "baee610e9452a8f6f0a1b6194ec09ff9e2d85dea54432acdae41aa0761c95d70" dependencies = [ "shlex", ] @@ -768,28 +681,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.85", -] - -[[package]] -name = "csv" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" -dependencies = [ - "csv-core", - "itoa 1.0.11", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" -dependencies = [ - "memchr", + "syn 2.0.87", ] [[package]] @@ -836,7 +728,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -860,7 +752,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -871,7 +763,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -951,7 +843,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -961,7 +853,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -974,7 +866,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -985,7 +877,7 @@ checksum = "65f152f4b8559c4da5d574bafc7af85454d706b4c5fe8b530d508cacbb6807ea" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1016,12 +908,12 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] name = "dkn-compute" -version = "0.2.18" +version = "0.2.19" dependencies = [ "async-trait", "base64 0.22.1", @@ -1053,7 +945,7 @@ dependencies = [ [[package]] name = "dkn-p2p" -version = "0.2.18" +version = "0.2.19" dependencies = [ "env_logger 0.11.5", "eyre", @@ -1065,7 +957,7 @@ dependencies = [ [[package]] name = "dkn-workflows" -version = "0.2.18" +version = "0.2.19" dependencies = [ "dotenvy", "env_logger 0.11.5", @@ -1180,7 +1072,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1269,17 +1161,6 @@ dependencies = [ "pin-project-lite 0.2.15", ] -[[package]] -name = "eventsource-stream" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" -dependencies = [ - "futures-core", - "nom", - "pin-project-lite 0.2.15", -] - [[package]] name = "eyre" version = "0.6.12" @@ -1290,16 +1171,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "fancy-regex" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7493d4c459da9f84325ad297371a6b2b8a162800873a22e3b6b6512e61d18c05" -dependencies = [ - "bit-set", - "regex", -] - [[package]] name = "fastbloom-rs" version = "0.5.9" @@ -1469,7 +1340,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1617,12 +1488,6 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - [[package]] name = "h2" version = "0.2.7" @@ -1695,9 +1560,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" dependencies = [ "allocator-api2", "equivalent", @@ -1845,15 +1710,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "html-escape" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" -dependencies = [ - "utf8-width", -] - [[package]] name = "html2text" version = "0.12.6" @@ -1881,20 +1737,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "html5ever" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" -dependencies = [ - "log", - "mac", - "markup5ever 0.11.0", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "html5ever" version = "0.27.0" @@ -1906,7 +1748,7 @@ dependencies = [ "markup5ever 0.12.1", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2060,7 +1902,6 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "httparse", - "httpdate 1.0.3", "itoa 1.0.11", "pin-project-lite 0.2.15", "smallvec", @@ -2079,7 +1920,6 @@ dependencies = [ "hyper 1.5.0", "hyper-util", "rustls", - "rustls-native-certs", "rustls-pki-types", "tokio 1.41.0", "tokio-rustls", @@ -2100,19 +1940,6 @@ dependencies = [ "tokio-tls", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes 1.8.0", - "hyper 0.14.31", - "native-tls", - "tokio 1.41.0", - "tokio-native-tls", -] - [[package]] name = "hyper-tls" version = "0.6.0" @@ -2171,6 +1998,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2189,12 +2134,23 @@ dependencies = [ [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -2268,7 +2224,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.1", ] [[package]] @@ -2382,41 +2338,6 @@ dependencies = [ "winapi-build", ] -[[package]] -name = "langchain-rust" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e85dc2101f68748bf3618320e5e980cf5da00e7d7dd9ade07c9d16f34f85a50" -dependencies = [ - "async-openai", - "async-recursion", - "async-stream", - "async-trait", - "csv", - "futures", - "futures-util", - "glob", - "html-escape", - "log", - "mockito", - "readability", - "regex", - "reqwest 0.12.9", - "reqwest-eventsource", - "scraper 0.20.0", - "secrecy", - "serde", - "serde_json", - "strum_macros", - "text-splitter 0.16.1", - "thiserror", - "tiktoken-rs", - "tokio 1.41.0", - "tokio-stream", - "url", - "urlencoding", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -2857,7 +2778,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2983,6 +2904,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -3005,7 +2932,7 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.15.0", + "hashbrown 0.15.1", ] [[package]] @@ -3043,20 +2970,6 @@ dependencies = [ "tendril", ] -[[package]] -name = "markup5ever" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" -dependencies = [ - "log", - "phf 0.10.1", - "phf_codegen 0.10.0", - "string_cache", - "string_cache_codegen", - "tendril", -] - [[package]] name = "markup5ever" version = "0.12.1" @@ -3080,19 +2993,7 @@ dependencies = [ "html5ever 0.25.2", "markup5ever 0.10.1", "tendril", - "xml5ever 0.16.2", -] - -[[package]] -name = "markup5ever_rcdom" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9521dd6750f8e80ee6c53d65e2e4656d7de37064f3a7a5d2d11d05df93839c2" -dependencies = [ - "html5ever 0.26.0", - "markup5ever 0.11.0", - "tendril", - "xml5ever 0.17.0", + "xml5ever", ] [[package]] @@ -3187,30 +3088,6 @@ dependencies = [ "ws2_32-sys", ] -[[package]] -name = "mockito" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b34bd91b9e5c5b06338d392463e1318d683cf82ec3d3af4014609be6e2108d" -dependencies = [ - "assert-json-diff", - "bytes 1.8.0", - "colored", - "futures-util", - "http 1.1.0", - "http-body 1.0.1", - "http-body-util", - "hyper 1.5.0", - "hyper-util", - "log", - "rand 0.8.5", - "regex", - "serde_json", - "serde_urlencoded", - "similar", - "tokio 1.41.0", -] - [[package]] name = "multiaddr" version = "0.18.2" @@ -3472,22 +3349,22 @@ dependencies = [ "scraper 0.19.1", "serde", "serde_json", - "text-splitter 0.13.3", + "text-splitter", "url", ] [[package]] name = "ollama-workflows" version = "0.1.0" -source = "git+https://github.com/andthattoo/ollama-workflows#a0b5efb03d61c3a7c30382be19568980c65a8b52" +source = "git+https://github.com/andthattoo/ollama-workflows#47df3be4853a0b35c172d13787f7d24fe6e0ba41" dependencies = [ "async-trait", + "base64 0.22.1", "colored", "dotenv", "env_logger 0.9.3", "gem-rs", "html2text", - "langchain-rust", "log", "ollama-rs", "openai_dive", @@ -3500,7 +3377,7 @@ dependencies = [ "serde", "serde_json", "simsimd", - "text-splitter 0.13.3", + "text-splitter", "tokio 1.41.0", "tokio-util 0.7.12", ] @@ -3519,9 +3396,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openai_dive" -version = "0.5.7" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3d3fe8284533a78c1ab1cf7da0a682f6738c621a3639e55a6e8041901b151a" +checksum = "311f110be1c6910150a9dbc2d0f98722a1c4aabcf9a35a5e55255e7a2adeb656" dependencies = [ "bytes 1.8.0", "derive_builder", @@ -3555,7 +3432,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3751,7 +3628,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3798,7 +3675,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3945,18 +3822,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", -] - -[[package]] -name = "pulldown-cmark" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" -dependencies = [ - "bitflags 2.6.0", - "memchr", - "unicase", + "syn 2.0.87", ] [[package]] @@ -3997,7 +3863,7 @@ dependencies = [ "pin-project-lite 0.2.15", "quinn-proto", "quinn-udp", - "rustc-hash 2.0.0", + "rustc-hash", "rustls", "socket2 0.5.7", "thiserror", @@ -4014,7 +3880,7 @@ dependencies = [ "bytes 1.8.0", "rand 0.8.5", "ring 0.17.8", - "rustc-hash 2.0.0", + "rustc-hash", "rustls", "slab", "thiserror", @@ -4024,9 +3890,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e346e016eacfff12233c243718197ca12f148c84e1e84268a896699b41c71780" +checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" dependencies = [ "cfg_aliases", "libc", @@ -4138,20 +4004,6 @@ dependencies = [ "yasna", ] -[[package]] -name = "readability" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56596e20a6d3cf715182d9b6829220621e6e985cec04d00410cee29821b4220" -dependencies = [ - "html5ever 0.26.0", - "lazy_static", - "markup5ever_rcdom 0.2.0", - "regex", - "reqwest 0.11.27", - "url", -] - [[package]] name = "redox_syscall" version = "0.5.7" @@ -4225,46 +4077,6 @@ dependencies = [ "winreg 0.7.0", ] -[[package]] -name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64 0.21.7", - "bytes 1.8.0", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.31", - "hyper-tls 0.5.0", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite 0.2.15", - "rustls-pemfile 1.0.4", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper 0.1.2", - "system-configuration 0.5.1", - "tokio 1.41.0", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg 0.50.0", -] - [[package]] name = "reqwest" version = "0.12.9" @@ -4295,13 +4107,12 @@ dependencies = [ "pin-project-lite 0.2.15", "quinn", "rustls", - "rustls-native-certs", - "rustls-pemfile 2.2.0", + "rustls-pemfile", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper", "system-configuration 0.6.1", "tokio 1.41.0", "tokio-native-tls", @@ -4317,22 +4128,6 @@ dependencies = [ "windows-registry", ] -[[package]] -name = "reqwest-eventsource" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632c55746dbb44275691640e7b40c907c16a2dc1a5842aa98aaec90da6ec6bde" -dependencies = [ - "eventsource-stream", - "futures-core", - "futures-timer", - "mime", - "nom", - "pin-project-lite 0.2.15", - "reqwest 0.12.9", - "thiserror", -] - [[package]] name = "reqwest-streams" version = "0.7.0" @@ -4411,12 +4206,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.0.0" @@ -4443,9 +4232,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags 2.6.0", "errno", @@ -4468,28 +4257,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" -dependencies = [ - "openssl-probe", - "rustls-pemfile 2.2.0", - "rustls-pki-types", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -4595,22 +4362,6 @@ dependencies = [ "tendril", ] -[[package]] -name = "scraper" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90460b31bfe1fc07be8262e42c665ad97118d4585869de9345a84d501a9eaf0" -dependencies = [ - "ahash", - "cssparser 0.31.2", - "ego-tree", - "getopts", - "html5ever 0.27.0", - "once_cell", - "selectors 0.25.0", - "tendril", -] - [[package]] name = "search_with_google" version = "0.5.0" @@ -4624,16 +4375,6 @@ dependencies = [ "tokio 0.2.25", ] -[[package]] -name = "secrecy" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" -dependencies = [ - "serde", - "zeroize", -] - [[package]] name = "security-framework" version = "2.11.1" @@ -4665,7 +4406,7 @@ checksum = "8ee061f90afcc8678bef7a78d0d121683f0ba753f740ff7005f833ec445876b7" dependencies = [ "bit-set", "html5ever 0.25.2", - "markup5ever_rcdom 0.1.0", + "markup5ever_rcdom", ] [[package]] @@ -4730,7 +4471,7 @@ checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4847,12 +4588,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "similar" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" - [[package]] name = "simsimd" version = "4.4.0" @@ -5006,7 +4741,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5028,21 +4763,15 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.85" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "sync_wrapper" version = "1.0.1" @@ -5060,7 +4789,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5155,25 +4884,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "text-splitter" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f280573deec490e745c503ecc1d0e17104e98936eaefd7b0aa4b1422c74b317" -dependencies = [ - "ahash", - "auto_enums", - "either", - "itertools", - "once_cell", - "pulldown-cmark", - "regex", - "strum", - "thiserror", - "tiktoken-rs", - "unicode-segmentation", -] - [[package]] name = "thin-slice" version = "0.1.1" @@ -5182,37 +4892,22 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", -] - -[[package]] -name = "tiktoken-rs" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c314e7ce51440f9e8f5a497394682a57b7c323d0f4d0a6b1b13c429056e0e234" -dependencies = [ - "anyhow", - "base64 0.21.7", - "bstr", - "fancy-regex", - "lazy_static", - "parking_lot", - "rustc-hash 1.1.0", + "syn 2.0.87", ] [[package]] @@ -5257,6 +4952,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -5315,7 +5020,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5339,17 +5044,6 @@ dependencies = [ "tokio 1.41.0", ] -[[package]] -name = "tokio-stream" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" -dependencies = [ - "futures-core", - "pin-project-lite 0.2.15", - "tokio 1.41.0", -] - [[package]] name = "tokio-tls" version = "0.3.1" @@ -5415,7 +5109,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -5530,12 +5224,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.3", "percent-encoding", ] @@ -5552,10 +5246,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] -name = "utf8-width" -version = "0.1.7" +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "utf8parse" @@ -5641,7 +5341,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -5675,7 +5375,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6002,6 +5702,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "ws2_32-sys" version = "0.2.1" @@ -6043,9 +5755,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af4e2e2f7cba5a093896c1e150fbfe177d1883e7448200efb81d40b9d339ef26" +checksum = "af310deaae937e48a26602b730250b4949e125f468f11e6990be3e5304ddd96f" [[package]] name = "xml5ever" @@ -6059,17 +5771,6 @@ dependencies = [ "time 0.1.45", ] -[[package]] -name = "xml5ever" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4034e1d05af98b51ad7214527730626f019682d797ba38b51689212118d8e650" -dependencies = [ - "log", - "mac", - "markup5ever 0.11.0", -] - [[package]] name = "xmltree" version = "0.10.3" @@ -6131,6 +5832,30 @@ dependencies = [ "time 0.3.36", ] +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -6149,7 +5874,28 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", ] [[package]] @@ -6169,5 +5915,27 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", ] diff --git a/Cargo.toml b/Cargo.toml index 6286d33..7f4233b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ default-members = ["compute"] [workspace.package] edition = "2021" -version = "0.2.18" +version = "0.2.19" license = "Apache-2.0" readme = "README.md" diff --git a/compute/src/config.rs b/compute/src/config.rs index 8fcfa79..4c9f1fd 100644 --- a/compute/src/config.rs +++ b/compute/src/config.rs @@ -1,4 +1,7 @@ -use crate::utils::{address_in_use, crypto::to_address}; +use crate::utils::{ + address_in_use, + crypto::{secret_to_keypair, to_address}, +}; use dkn_p2p::libp2p::Multiaddr; use dkn_workflows::DriaWorkflowsConfig; use eyre::{eyre, Result}; @@ -71,14 +74,21 @@ impl DriaComputeNodeConfig { panic!("Please provide an admin public key."); } }; + + let address = to_address(&public_key); + log::info!("Node Address: 0x{}", hex::encode(address)); + + // to this here to log the peer id at start + log::info!( + "Node PeerID: {}", + secret_to_keypair(&secret_key).public().to_peer_id() + ); + log::info!( "Admin Public Key: 0x{}", hex::encode(admin_public_key.serialize_compressed()) ); - let address = to_address(&public_key); - log::info!("Node Address: 0x{}", hex::encode(address)); - let workflows = DriaWorkflowsConfig::new_from_csv(&env::var("DKN_MODELS").unwrap_or_default()); #[cfg(not(test))] diff --git a/compute/src/handlers/workflow.rs b/compute/src/handlers/workflow.rs index 33d3b86..5941b8e 100644 --- a/compute/src/handlers/workflow.rs +++ b/compute/src/handlers/workflow.rs @@ -35,7 +35,6 @@ impl ComputeHandler for WorkflowHandler { node: &mut DriaComputeNode, message: DKNMessage, ) -> Result { - let config = &node.config; let task = message .parse_payload::>(true) .wrap_err("Could not parse workflow task")?; @@ -55,7 +54,7 @@ impl ComputeHandler for WorkflowHandler { } // check task inclusion via the bloom filter - if !task.filter.contains(&config.address)? { + if !task.filter.contains(&node.config.address)? { log::info!( "Task {} does not include this node within the filter.", task.task_id @@ -66,7 +65,10 @@ impl ComputeHandler for WorkflowHandler { } // read model / provider from the task - let (model_provider, model) = config.workflows.get_any_matching_model(task.input.model)?; + let (model_provider, model) = node + .config + .workflows + .get_any_matching_model(task.input.model)?; let model_name = model.to_string(); // get model name, we will pass it in payload log::info!("Using model {} for task {}", model_name, task.task_id); @@ -74,8 +76,8 @@ impl ComputeHandler for WorkflowHandler { let executor = if model_provider == ModelProvider::Ollama { Executor::new_at( model, - &config.workflows.ollama.host, - config.workflows.ollama.port, + &node.config.workflows.ollama.host, + node.config.workflows.ollama.port, ) } else { Executor::new(model) @@ -93,13 +95,14 @@ impl ComputeHandler for WorkflowHandler { log::info!("Received cancellation, quitting all tasks."); return Ok(MessageAcceptance::Accept); }, - exec_result_inner = executor.execute(entry.as_ref(), task.input.workflow, &mut memory) => { + exec_result_inner = executor.execute(entry.as_ref(), &task.input.workflow, &mut memory) => { exec_result = exec_result_inner.map_err(|e| eyre!("Execution error: {}", e.to_string())); } } - match exec_result { + let (publish_result, acceptance) = match exec_result { Ok(result) => { + log::warn!("Task {} result:", result); // obtain public key from the payload let task_public_key_bytes = hex::decode(&task.public_key).wrap_err("Could not decode public key")?; @@ -110,18 +113,16 @@ impl ComputeHandler for WorkflowHandler { result, &task.task_id, &task_public_key, - &config.secret_key, + &node.config.secret_key, model_name, )?; let payload_str = serde_json::to_string(&payload) .wrap_err("Could not serialize response payload")?; // publish the result - let message = DKNMessage::new(payload_str, Self::RESPONSE_TOPIC); - node.publish(message)?; - // accept so that if there are others included in filter they can do the task - Ok(MessageAcceptance::Accept) + let message = DKNMessage::new(payload_str, Self::RESPONSE_TOPIC); + (node.publish(message), MessageAcceptance::Accept) } Err(err) => { // use pretty display string for error logging with causes @@ -129,17 +130,38 @@ impl ComputeHandler for WorkflowHandler { log::error!("Task {} failed: {}", task.task_id, err_string); // prepare error payload - let error_payload = TaskErrorPayload::new(task.task_id, err_string, model_name); + let error_payload = + TaskErrorPayload::new(task.task_id.clone(), err_string, model_name); let error_payload_str = serde_json::to_string(&error_payload) .wrap_err("Could not serialize error payload")?; // publish the error result for diagnostics - let message = DKNMessage::new(error_payload_str, Self::RESPONSE_TOPIC); - node.publish(message)?; - // ignore just in case, workflow may be bugged - Ok(MessageAcceptance::Ignore) + let message = DKNMessage::new_signed( + error_payload_str, + Self::RESPONSE_TOPIC, + &node.config.secret_key, + ); + (node.publish(message), MessageAcceptance::Ignore) } + }; + + // if for some reason we couldnt publish the result, publish the error itself so that RPC doesnt hang + if let Err(publish_err) = publish_result { + let err_msg = format!("Could not publish result: {:?}", publish_err); + log::error!("{}", err_msg); + let payload = serde_json::json!({ + "taskId": task.task_id, + "error": err_msg + }); + let message = DKNMessage::new_signed( + payload.to_string(), + Self::RESPONSE_TOPIC, + &node.config.secret_key, + ); + node.publish(message)?; } + + Ok(acceptance) } } diff --git a/compute/src/node.rs b/compute/src/node.rs index e89a961..2fbcb25 100644 --- a/compute/src/node.rs +++ b/compute/src/node.rs @@ -60,7 +60,7 @@ impl DriaComputeNode { ); // create p2p client - let p2p = DriaP2PClient::new( + let mut p2p = DriaP2PClient::new( keypair, config.p2p_listen_addr.clone(), &available_nodes.bootstrap_nodes, @@ -68,6 +68,16 @@ impl DriaComputeNode { P2P_VERSION, )?; + // dial rpc nodes + if available_nodes.rpc_addrs.is_empty() { + log::warn!("No RPC nodes found to be dialled!"); + } else { + for rpc_addr in &available_nodes.rpc_addrs { + log::info!("Dialing RPC node: {}", rpc_addr); + p2p.dial(rpc_addr.clone())?; + } + } + Ok(DriaComputeNode { p2p, config, @@ -136,6 +146,7 @@ impl DriaComputeNode { event = self.p2p.process_events() => { // refresh admin rpc peer ids if self.available_nodes_last_refreshed.elapsed() > Duration::from_secs(RPC_PEER_ID_REFRESH_INTERVAL_SECS) { + log::info!("Refreshing available nodes."); self.available_nodes = AvailableNodes::get_available_nodes().await.unwrap_or_default().join(self.available_nodes.clone()).sort_dedup(); self.available_nodes_last_refreshed = tokio::time::Instant::now(); } @@ -156,12 +167,17 @@ impl DriaComputeNode { } }; + // log::info!( + // "Received {} message ({})\nFrom: {}\nSource: {}", + // topic_str, + // message_id, + // peer_id, + // ); log::info!( - "Received {} message ({})\nFrom: {}\nSource: {}", + "Received {} message ({}) from {}", topic_str, message_id, peer_id, - source_peer_id ); // ensure that message is from the static RPCs diff --git a/compute/src/utils/available_nodes.rs b/compute/src/utils/available_nodes.rs index 45a6f73..c3ea0de 100644 --- a/compute/src/utils/available_nodes.rs +++ b/compute/src/utils/available_nodes.rs @@ -25,18 +25,20 @@ const STATIC_RPC_PEER_IDS: [&str; 0] = []; /// API URL for refreshing the Admin RPC PeerIDs from Dria server. const RPC_PEER_ID_REFRESH_API_URL: &str = "https://dkn.dria.co/available-nodes"; -#[derive(serde::Deserialize, Debug)] -pub struct AvailableNodesApiResponse { - pub bootstraps: Vec, - pub relays: Vec, - pub rpcs: Vec, -} - +/// Available nodes within the hybrid P2P network. +/// +/// - Bootstrap: used for Kademlia DHT bootstrap. +/// - Relay: used for DCutR relay protocol. +/// - RPC: used for RPC nodes for task & ping messages. +/// +/// Note that while bootstrap & relay nodes are `Multiaddr`, RPC nodes are `PeerId` because we communicate +/// with them via GossipSub only. #[derive(Debug, Default, Clone)] pub struct AvailableNodes { pub bootstrap_nodes: Vec, pub relay_nodes: Vec, pub rpc_nodes: Vec, + pub rpc_addrs: Vec, } impl AvailableNodes { @@ -66,6 +68,7 @@ impl AvailableNodes { bootstrap_nodes: parse_vec(bootstrap_nodes), relay_nodes: parse_vec(relay_nodes), rpc_nodes: vec![], + rpc_addrs: vec![], } } @@ -75,6 +78,7 @@ impl AvailableNodes { bootstrap_nodes: parse_vec(STATIC_BOOTSTRAP_NODES.to_vec()), relay_nodes: parse_vec(STATIC_RELAY_NODES.to_vec()), rpc_nodes: parse_vec(STATIC_RPC_PEER_IDS.to_vec()), + rpc_addrs: vec![], } } @@ -83,7 +87,7 @@ impl AvailableNodes { self.bootstrap_nodes.extend(other.bootstrap_nodes); self.relay_nodes.extend(other.relay_nodes); self.rpc_nodes.extend(other.rpc_nodes); - + self.rpc_addrs.extend(other.rpc_addrs); self } @@ -98,11 +102,23 @@ impl AvailableNodes { self.rpc_nodes.sort_unstable(); self.rpc_nodes.dedup(); + self.rpc_addrs.sort_unstable(); + self.rpc_addrs.dedup(); + self } /// Refreshes the available nodes for Bootstrap, Relay and RPC nodes. pub async fn get_available_nodes() -> Result { + #[derive(serde::Deserialize, Debug)] + struct AvailableNodesApiResponse { + pub bootstraps: Vec, + pub relays: Vec, + pub rpcs: Vec, + #[serde(rename = "rpcAddrs")] + pub rpc_addrs: Vec, + } + let response = reqwest::get(RPC_PEER_ID_REFRESH_API_URL).await?; let response_body = response.json::().await?; @@ -110,6 +126,7 @@ impl AvailableNodes { bootstrap_nodes: parse_vec(response_body.bootstraps), relay_nodes: parse_vec(response_body.relays), rpc_nodes: parse_vec(response_body.rpcs), + rpc_addrs: parse_vec(response_body.rpc_addrs), }) } } @@ -137,9 +154,6 @@ mod tests { #[tokio::test] #[ignore = "run this manually"] async fn test_get_available_nodes() { - std::env::set_var("RUST_LOG", "info"); - let _ = env_logger::try_init(); - let available_nodes = AvailableNodes::get_available_nodes().await.unwrap(); println!("{:#?}", available_nodes); } diff --git a/p2p/src/behaviour.rs b/p2p/src/behaviour.rs index 6bfbc75..7100907 100644 --- a/p2p/src/behaviour.rs +++ b/p2p/src/behaviour.rs @@ -10,12 +10,12 @@ use libp2p::{autonat, dcutr, gossipsub, identify, kad, relay}; #[derive(libp2p::swarm::NetworkBehaviour)] pub struct DriaBehaviour { - pub(crate) relay: relay::client::Behaviour, - pub(crate) gossipsub: gossipsub::Behaviour, - pub(crate) kademlia: kad::Behaviour, - pub(crate) identify: identify::Behaviour, - pub(crate) autonat: autonat::Behaviour, - pub(crate) dcutr: dcutr::Behaviour, + pub relay: relay::client::Behaviour, + pub gossipsub: gossipsub::Behaviour, + pub kademlia: kad::Behaviour, + pub identify: identify::Behaviour, + pub autonat: autonat::Behaviour, + pub dcutr: dcutr::Behaviour, } impl DriaBehaviour { diff --git a/p2p/src/client.rs b/p2p/src/client.rs index 6af488f..8cc6740 100644 --- a/p2p/src/client.rs +++ b/p2p/src/client.rs @@ -1,13 +1,12 @@ use super::*; -use eyre::Result; +use eyre::{Context, Result}; use libp2p::futures::StreamExt; use libp2p::gossipsub::{ Message, MessageAcceptance, MessageId, PublishError, SubscriptionError, TopicHash, }; use libp2p::kad::{GetClosestPeersError, GetClosestPeersOk, QueryResult}; -use libp2p::{ - autonat, gossipsub, identify, kad, multiaddr::Protocol, noise, swarm::SwarmEvent, tcp, yamux, -}; +use libp2p::swarm::{dial_opts::DialOpts, NetworkInfo, SwarmEvent}; +use libp2p::{autonat, gossipsub, identify, kad, multiaddr::Protocol, noise, tcp, yamux}; use libp2p::{Multiaddr, PeerId, StreamProtocol, Swarm, SwarmBuilder}; use libp2p_identity::Keypair; use std::time::{Duration, Instant}; @@ -137,6 +136,12 @@ impl DriaP2PClient { }) } + /// Returns the network information, such as the number of + /// incoming and outgoing connections. + pub fn network_info(&self) -> NetworkInfo { + self.swarm.network_info() + } + /// Subscribe to a topic. pub fn subscribe(&mut self, topic_name: &str) -> Result { log::debug!("Subscribing to {}", topic_name); @@ -206,6 +211,11 @@ impl DriaP2PClient { self.swarm.behaviour().gossipsub.all_peers().collect() } + /// Dials a given peer. + pub fn dial(&mut self, peer_id: impl Into) -> Result<()> { + self.swarm.dial(peer_id).wrap_err("could not dial") + } + /// Listens to the Swarm for incoming messages. /// /// This method should be called in a loop to keep the client running. diff --git a/workflows/src/apis/serper.rs b/workflows/src/apis/serper.rs index 0612160..76340b0 100644 --- a/workflows/src/apis/serper.rs +++ b/workflows/src/apis/serper.rs @@ -48,7 +48,6 @@ impl SerperConfig { log::debug!("Serper API key not found, skipping Serper check"); return Ok(()); }; - println!("API KEY: {}", api_key); log::info!("Serper API key found, checking Serper service"); // make a dummy request diff --git a/workflows/src/providers/gemini.rs b/workflows/src/providers/gemini.rs index 660bf65..54f131d 100644 --- a/workflows/src/providers/gemini.rs +++ b/workflows/src/providers/gemini.rs @@ -55,7 +55,7 @@ impl GeminiConfig { } // make a dummy request - if let Err(err) = self.dummy_request(&api_key, &requested_model).await { + if let Err(err) = self.dummy_request(api_key, &requested_model).await { log::warn!( "Model {} failed dummy request, ignoring it: {}", requested_model, @@ -139,7 +139,7 @@ impl GeminiConfig { let request = client .post(format!( "https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent", - model.to_string() + model )) .query(&[("key", api_key)]) .header("Content-Type", "application/json") diff --git a/workflows/src/providers/ollama.rs b/workflows/src/providers/ollama.rs index 57c1789..905fea2 100644 --- a/workflows/src/providers/ollama.rs +++ b/workflows/src/providers/ollama.rs @@ -308,7 +308,7 @@ mod tests { let exe = Executor::new(Model::default()); let mut memory = ProgramMemory::new(); - let result = exe.execute(None, workflow, &mut memory).await; + let result = exe.execute(None, &workflow, &mut memory).await; println!("Result: {}", result.unwrap()); } } diff --git a/workflows/src/providers/openai.rs b/workflows/src/providers/openai.rs index 5491a60..b31c5f2 100644 --- a/workflows/src/providers/openai.rs +++ b/workflows/src/providers/openai.rs @@ -39,7 +39,7 @@ impl OpenAIConfig { }; // check if models exist within the account and select those that are available - let openai_model_names = self.fetch_models(&api_key).await?; + let openai_model_names = self.fetch_models(api_key).await?; let mut available_models = Vec::new(); for requested_model in models { // check if model exists @@ -52,7 +52,7 @@ impl OpenAIConfig { } // make a dummy request - if let Err(err) = self.dummy_request(&api_key, &requested_model).await { + if let Err(err) = self.dummy_request(api_key, &requested_model).await { log::warn!( "Model {} failed dummy request, ignoring it: {}", requested_model, @@ -105,13 +105,13 @@ impl OpenAIConfig { // parse response if !response.status().is_success() { - return Err(eyre!( + Err(eyre!( "Failed to fetch OpenAI models:\n{}", response .text() .await .unwrap_or("Could not get error text as well".to_string()) - )); + )) } else { let openai_models = response.json::().await?; Ok(openai_models.data.into_iter().map(|m| m.id).collect())