From 94ba337c46d9e5bd4b666b7b4f497061af3ecae2 Mon Sep 17 00:00:00 2001 From: rmn-boiko Date: Mon, 14 Oct 2024 13:15:13 +0200 Subject: [PATCH] Revert "Merge branch 'master' into chore/push-and-pul-command-e2e-tests" This reverts commit a4ec39c4d5d891aed60b93bb813083a72c958e0c, reversing changes made to 1130fa5bc1f8816d0b2968b1fdb96e135dea55bf. --- CHANGELOG.md | 5 +- Cargo.lock | 146 +++++----- Cargo.toml | 138 ++++----- LICENSE.txt | 2 +- examples/query-commitments/README.md | 17 -- examples/query-commitments/example.py | 266 ------------------ examples/query-commitments/requirements.in | 4 - examples/reth-vs-snp500/init-odf-all.sh | 23 -- examples/reth-vs-snp500/init-odf.sh | 17 -- .../kamu-base-with-data-mt/extra/.kamuconfig | 3 - .../kamu-base-with-data-mt/init-workspace.py | 21 +- 11 files changed, 151 insertions(+), 491 deletions(-) delete mode 100644 examples/query-commitments/README.md delete mode 100755 examples/query-commitments/example.py delete mode 100644 examples/query-commitments/requirements.in delete mode 100755 examples/reth-vs-snp500/init-odf-all.sh delete mode 100755 examples/reth-vs-snp500/init-odf.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c944f697..4b95638ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Recommendation: for ease of reading, use the following order: - Fixed --> -## [0.204.5] - 2024-10-08 +## [Unreleased] ### Added - Postgres implementation for dataset entry and account Re-BAC repositories - Added (or expanded) E2E tests for: @@ -42,9 +42,6 @@ Recommendation: for ease of reading, use the following order: - Simplified error handling code in repositories - Hidden part of the test code behind the feature gate - Updated our crate dependencies so they can be built in isolation -### Fixed -- `--yes / -y` flag: fixed when working from a TTY -- CI: Fixes `kamu-base-with-data-mt` image builds ## [0.204.4] - 2024-09-30 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 19d059dde..79f4fb3fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "async-utils" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", ] @@ -2448,7 +2448,7 @@ checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "container-runtime" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "cfg-if", @@ -2878,7 +2878,7 @@ checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "database-common" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "aws-config", @@ -2902,7 +2902,7 @@ dependencies = [ [[package]] name = "database-common-macros" -version = "0.204.5" +version = "0.204.4" dependencies = [ "quote", "syn 2.0.79", @@ -3746,7 +3746,7 @@ dependencies = [ [[package]] name = "enum-variants" -version = "0.204.5" +version = "0.204.4" [[package]] name = "env_filter" @@ -3815,7 +3815,7 @@ dependencies = [ [[package]] name = "event-sourcing" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -3831,7 +3831,7 @@ dependencies = [ [[package]] name = "event-sourcing-macros" -version = "0.204.5" +version = "0.204.4" dependencies = [ "quote", "syn 2.0.79", @@ -4499,7 +4499,7 @@ dependencies = [ [[package]] name = "http-common" -version = "0.204.5" +version = "0.204.4" dependencies = [ "axum", "http 1.1.0", @@ -4798,7 +4798,7 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "init-on-startup" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "database-common", @@ -4841,7 +4841,7 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "internal-error" -version = "0.204.5" +version = "0.204.4" dependencies = [ "thiserror", ] @@ -4989,7 +4989,7 @@ dependencies = [ [[package]] name = "kamu" -version = "0.204.5" +version = "0.204.4" dependencies = [ "alloy", "async-recursion", @@ -5077,7 +5077,7 @@ dependencies = [ [[package]] name = "kamu-accounts" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "base32", @@ -5103,7 +5103,7 @@ dependencies = [ [[package]] name = "kamu-accounts-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5124,7 +5124,7 @@ dependencies = [ [[package]] name = "kamu-accounts-mysql" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5145,7 +5145,7 @@ dependencies = [ [[package]] name = "kamu-accounts-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5166,7 +5166,7 @@ dependencies = [ [[package]] name = "kamu-accounts-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "argon2", "chrono", @@ -5182,7 +5182,7 @@ dependencies = [ [[package]] name = "kamu-accounts-services" -version = "0.204.5" +version = "0.204.4" dependencies = [ "argon2", "async-trait", @@ -5209,7 +5209,7 @@ dependencies = [ [[package]] name = "kamu-accounts-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5230,7 +5230,7 @@ dependencies = [ [[package]] name = "kamu-adapter-auth-oso" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "dill", @@ -5252,7 +5252,7 @@ dependencies = [ [[package]] name = "kamu-adapter-flight-sql" -version = "0.204.5" +version = "0.204.4" dependencies = [ "arrow-flight", "async-trait", @@ -5275,7 +5275,7 @@ dependencies = [ [[package]] name = "kamu-adapter-graphql" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-graphql", "async-trait", @@ -5326,7 +5326,7 @@ dependencies = [ [[package]] name = "kamu-adapter-http" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "aws-sdk-s3", @@ -5392,7 +5392,7 @@ dependencies = [ [[package]] name = "kamu-adapter-oauth" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5411,7 +5411,7 @@ dependencies = [ [[package]] name = "kamu-adapter-odata" -version = "0.204.5" +version = "0.204.4" dependencies = [ "axum", "chrono", @@ -5447,7 +5447,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "internal-error", @@ -5459,7 +5459,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "database-common-macros", @@ -5473,7 +5473,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "database-common", @@ -5490,7 +5490,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "dill", "kamu-auth-rebac", @@ -5499,7 +5499,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-services" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "dill", @@ -5518,7 +5518,7 @@ dependencies = [ [[package]] name = "kamu-auth-rebac-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "database-common", @@ -5535,7 +5535,7 @@ dependencies = [ [[package]] name = "kamu-cli" -version = "0.204.5" +version = "0.204.4" dependencies = [ "arrow-flight", "async-graphql", @@ -5656,7 +5656,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5678,7 +5678,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-common-macros" -version = "0.204.5" +version = "0.204.4" dependencies = [ "quote", "syn 2.0.79", @@ -5686,7 +5686,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5699,7 +5699,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-mysql" -version = "0.204.5" +version = "0.204.4" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5713,7 +5713,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5727,7 +5727,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "indoc 2.0.5", @@ -5745,7 +5745,7 @@ dependencies = [ [[package]] name = "kamu-cli-e2e-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "indoc 2.0.5", "kamu-cli-e2e-common", @@ -5759,7 +5759,7 @@ dependencies = [ [[package]] name = "kamu-cli-puppet" -version = "0.204.5" +version = "0.204.4" dependencies = [ "assert_cmd", "async-trait", @@ -5775,7 +5775,7 @@ dependencies = [ [[package]] name = "kamu-core" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -5806,7 +5806,7 @@ dependencies = [ [[package]] name = "kamu-data-utils" -version = "0.204.5" +version = "0.204.4" dependencies = [ "arrow", "arrow-digest", @@ -5831,7 +5831,7 @@ dependencies = [ [[package]] name = "kamu-datafusion-cli" -version = "0.204.5" +version = "0.204.4" dependencies = [ "arrow", "async-trait", @@ -5855,7 +5855,7 @@ dependencies = [ [[package]] name = "kamu-datasets" -version = "0.204.5" +version = "0.204.4" dependencies = [ "aes-gcm", "async-trait", @@ -5875,7 +5875,7 @@ dependencies = [ [[package]] name = "kamu-datasets-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5898,7 +5898,7 @@ dependencies = [ [[package]] name = "kamu-datasets-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5921,7 +5921,7 @@ dependencies = [ [[package]] name = "kamu-datasets-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "database-common", @@ -5935,7 +5935,7 @@ dependencies = [ [[package]] name = "kamu-datasets-services" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5966,7 +5966,7 @@ dependencies = [ [[package]] name = "kamu-datasets-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -5989,7 +5989,7 @@ dependencies = [ [[package]] name = "kamu-flow-system" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6018,7 +6018,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6048,7 +6048,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6073,7 +6073,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "database-common", @@ -6086,7 +6086,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-services" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6130,7 +6130,7 @@ dependencies = [ [[package]] name = "kamu-flow-system-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6155,7 +6155,7 @@ dependencies = [ [[package]] name = "kamu-ingest-datafusion" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6191,7 +6191,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6210,7 +6210,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6233,7 +6233,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "database-common", @@ -6247,7 +6247,7 @@ dependencies = [ [[package]] name = "kamu-messaging-outbox-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6269,7 +6269,7 @@ dependencies = [ [[package]] name = "kamu-repo-tools" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "clap", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "kamu-task-system" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6302,7 +6302,7 @@ dependencies = [ [[package]] name = "kamu-task-system-inmem" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6321,7 +6321,7 @@ dependencies = [ [[package]] name = "kamu-task-system-postgres" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6344,7 +6344,7 @@ dependencies = [ [[package]] name = "kamu-task-system-repo-tests" -version = "0.204.5" +version = "0.204.4" dependencies = [ "chrono", "database-common", @@ -6356,7 +6356,7 @@ dependencies = [ [[package]] name = "kamu-task-system-services" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6384,7 +6384,7 @@ dependencies = [ [[package]] name = "kamu-task-system-sqlite" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-stream", "async-trait", @@ -6781,7 +6781,7 @@ dependencies = [ [[package]] name = "messaging-outbox" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -6910,7 +6910,7 @@ dependencies = [ [[package]] name = "multiformats" -version = "0.204.5" +version = "0.204.4" dependencies = [ "base64 0.22.1", "bs58", @@ -7235,7 +7235,7 @@ dependencies = [ [[package]] name = "observability" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "axum", @@ -7294,7 +7294,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendatafabric" -version = "0.204.5" +version = "0.204.4" dependencies = [ "arrow", "base64 0.22.1", @@ -8254,7 +8254,7 @@ dependencies = [ [[package]] name = "random-names" -version = "0.204.5" +version = "0.204.4" dependencies = [ "rand", ] @@ -9825,7 +9825,7 @@ dependencies = [ [[package]] name = "time-source" -version = "0.204.5" +version = "0.204.4" dependencies = [ "async-trait", "chrono", @@ -10239,7 +10239,7 @@ dependencies = [ [[package]] name = "tracing-perfetto" -version = "0.204.5" +version = "0.204.4" dependencies = [ "conv", "serde", diff --git a/Cargo.toml b/Cargo.toml index 0aabea069..9e5404939 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,95 +92,95 @@ resolver = "2" [workspace.dependencies] # Apps -kamu-cli = { version = "0.204.5", path = "src/app/cli", default-features = false } +kamu-cli = { version = "0.204.4", path = "src/app/cli", default-features = false } # Utils -async-utils = { version = "0.204.5", path = "src/utils/async-utils", default-features = false } -container-runtime = { version = "0.204.5", path = "src/utils/container-runtime", default-features = false } -database-common = { version = "0.204.5", path = "src/utils/database-common", default-features = false } -database-common-macros = { version = "0.204.5", path = "src/utils/database-common-macros", default-features = false } -enum-variants = { version = "0.204.5", path = "src/utils/enum-variants", default-features = false } -event-sourcing = { version = "0.204.5", path = "src/utils/event-sourcing", default-features = false } -event-sourcing-macros = { version = "0.204.5", path = "src/utils/event-sourcing-macros", default-features = false } -http-common = { version = "0.204.5", path = "src/utils/http-common", default-features = false } -init-on-startup = { version = "0.204.5", path = "src/utils/init-on-startup", default-features = false } -internal-error = { version = "0.204.5", path = "src/utils/internal-error", default-features = false } -kamu-cli-puppet = { version = "0.204.5", path = "src/utils/kamu-cli-puppet", default-features = false } -kamu-data-utils = { version = "0.204.5", path = "src/utils/data-utils", default-features = false } -kamu-datafusion-cli = { version = "0.204.5", path = "src/utils/datafusion-cli", default-features = false } -messaging-outbox = { version = "0.204.5", path = "src/utils/messaging-outbox", default-features = false } -multiformats = { version = "0.204.5", path = "src/utils/multiformats", default-features = false } -observability = { version = "0.204.5", path = "src/utils/observability", default-features = false } -random-names = { version = "0.204.5", path = "src/utils/random-names", default-features = false } -time-source = { version = "0.204.5", path = "src/utils/time-source", default-features = false } -tracing-perfetto = { version = "0.204.5", path = "src/utils/tracing-perfetto", default-features = false } +async-utils = { version = "0.204.4", path = "src/utils/async-utils", default-features = false } +container-runtime = { version = "0.204.4", path = "src/utils/container-runtime", default-features = false } +database-common = { version = "0.204.4", path = "src/utils/database-common", default-features = false } +database-common-macros = { version = "0.204.4", path = "src/utils/database-common-macros", default-features = false } +enum-variants = { version = "0.204.4", path = "src/utils/enum-variants", default-features = false } +event-sourcing = { version = "0.204.4", path = "src/utils/event-sourcing", default-features = false } +event-sourcing-macros = { version = "0.204.4", path = "src/utils/event-sourcing-macros", default-features = false } +http-common = { version = "0.204.4", path = "src/utils/http-common", default-features = false } +init-on-startup = { version = "0.204.4", path = "src/utils/init-on-startup", default-features = false } +internal-error = { version = "0.204.4", path = "src/utils/internal-error", default-features = false } +kamu-cli-puppet = { version = "0.204.4", path = "src/utils/kamu-cli-puppet", default-features = false } +kamu-data-utils = { version = "0.204.4", path = "src/utils/data-utils", default-features = false } +kamu-datafusion-cli = { version = "0.204.4", path = "src/utils/datafusion-cli", default-features = false } +messaging-outbox = { version = "0.204.4", path = "src/utils/messaging-outbox", default-features = false } +multiformats = { version = "0.204.4", path = "src/utils/multiformats", default-features = false } +observability = { version = "0.204.4", path = "src/utils/observability", default-features = false } +random-names = { version = "0.204.4", path = "src/utils/random-names", default-features = false } +time-source = { version = "0.204.4", path = "src/utils/time-source", default-features = false } +tracing-perfetto = { version = "0.204.4", path = "src/utils/tracing-perfetto", default-features = false } # Domain -kamu-accounts = { version = "0.204.5", path = "src/domain/accounts/domain", default-features = false } -kamu-auth-rebac = { version = "0.204.5", path = "src/domain/auth-rebac/domain", default-features = false } -kamu-core = { version = "0.204.5", path = "src/domain/core", default-features = false } -kamu-datasets = { version = "0.204.5", path = "src/domain/datasets/domain", default-features = false } -kamu-flow-system = { version = "0.204.5", path = "src/domain/flow-system/domain", default-features = false } -kamu-task-system = { version = "0.204.5", path = "src/domain/task-system/domain", default-features = false } -opendatafabric = { version = "0.204.5", path = "src/domain/opendatafabric", default-features = false } +kamu-accounts = { version = "0.204.4", path = "src/domain/accounts/domain", default-features = false } +kamu-auth-rebac = { version = "0.204.4", path = "src/domain/auth-rebac/domain", default-features = false } +kamu-core = { version = "0.204.4", path = "src/domain/core", default-features = false } +kamu-datasets = { version = "0.204.4", path = "src/domain/datasets/domain", default-features = false } +kamu-flow-system = { version = "0.204.4", path = "src/domain/flow-system/domain", default-features = false } +kamu-task-system = { version = "0.204.4", path = "src/domain/task-system/domain", default-features = false } +opendatafabric = { version = "0.204.4", path = "src/domain/opendatafabric", default-features = false } # Domain service layer -kamu-accounts-services = { version = "0.204.5", path = "src/domain/accounts/services", default-features = false } -kamu-auth-rebac-services = { version = "0.204.5", path = "src/domain/auth-rebac/services", default-features = false } -kamu-datasets-services = { version = "0.204.5", path = "src/domain/datasets/services", default-features = false } -kamu-flow-system-services = { version = "0.204.5", path = "src/domain/flow-system/services", default-features = false } -kamu-task-system-services = { version = "0.204.5", path = "src/domain/task-system/services", default-features = false } +kamu-accounts-services = { version = "0.204.4", path = "src/domain/accounts/services", default-features = false } +kamu-auth-rebac-services = { version = "0.204.4", path = "src/domain/auth-rebac/services", default-features = false } +kamu-datasets-services = { version = "0.204.4", path = "src/domain/datasets/services", default-features = false } +kamu-flow-system-services = { version = "0.204.4", path = "src/domain/flow-system/services", default-features = false } +kamu-task-system-services = { version = "0.204.4", path = "src/domain/task-system/services", default-features = false } # Infra -kamu = { version = "0.204.5", path = "src/infra/core", default-features = false } -kamu-ingest-datafusion = { version = "0.204.5", path = "src/infra/ingest-datafusion", default-features = false } +kamu = { version = "0.204.4", path = "src/infra/core", default-features = false } +kamu-ingest-datafusion = { version = "0.204.4", path = "src/infra/ingest-datafusion", default-features = false } ## Flow System -kamu-flow-system-repo-tests = { version = "0.204.5", path = "src/infra/flow-system/repo-tests", default-features = false } -kamu-flow-system-inmem = { version = "0.204.5", path = "src/infra/flow-system/inmem", default-features = false } -kamu-flow-system-postgres = { version = "0.204.5", path = "src/infra/flow-system/postgres", default-features = false } -kamu-flow-system-sqlite = { version = "0.204.5", path = "src/infra/flow-system/sqlite", default-features = false } +kamu-flow-system-repo-tests = { version = "0.204.4", path = "src/infra/flow-system/repo-tests", default-features = false } +kamu-flow-system-inmem = { version = "0.204.4", path = "src/infra/flow-system/inmem", default-features = false } +kamu-flow-system-postgres = { version = "0.204.4", path = "src/infra/flow-system/postgres", default-features = false } +kamu-flow-system-sqlite = { version = "0.204.4", path = "src/infra/flow-system/sqlite", default-features = false } ## Accounts -kamu-accounts-inmem = { version = "0.204.5", path = "src/infra/accounts/inmem", default-features = false } -kamu-accounts-mysql = { version = "0.204.5", path = "src/infra/accounts/mysql", default-features = false } -kamu-accounts-postgres = { version = "0.204.5", path = "src/infra/accounts/postgres", default-features = false } -kamu-accounts-sqlite = { version = "0.204.5", path = "src/infra/accounts/sqlite", default-features = false } -kamu-accounts-repo-tests = { version = "0.204.5", path = "src/infra/accounts/repo-tests", default-features = false } +kamu-accounts-inmem = { version = "0.204.4", path = "src/infra/accounts/inmem", default-features = false } +kamu-accounts-mysql = { version = "0.204.4", path = "src/infra/accounts/mysql", default-features = false } +kamu-accounts-postgres = { version = "0.204.4", path = "src/infra/accounts/postgres", default-features = false } +kamu-accounts-sqlite = { version = "0.204.4", path = "src/infra/accounts/sqlite", default-features = false } +kamu-accounts-repo-tests = { version = "0.204.4", path = "src/infra/accounts/repo-tests", default-features = false } ## Datasets -kamu-datasets-inmem = { version = "0.204.5", path = "src/infra/datasets/inmem", default-features = false } -kamu-datasets-postgres = { version = "0.204.5", path = "src/infra/datasets/postgres", default-features = false } -kamu-datasets-sqlite = { version = "0.204.5", path = "src/infra/datasets/sqlite", default-features = false } -kamu-datasets-repo-tests = { version = "0.204.5", path = "src/infra/datasets/repo-tests", default-features = false } +kamu-datasets-inmem = { version = "0.204.4", path = "src/infra/datasets/inmem", default-features = false } +kamu-datasets-postgres = { version = "0.204.4", path = "src/infra/datasets/postgres", default-features = false } +kamu-datasets-sqlite = { version = "0.204.4", path = "src/infra/datasets/sqlite", default-features = false } +kamu-datasets-repo-tests = { version = "0.204.4", path = "src/infra/datasets/repo-tests", default-features = false } ## Task System -kamu-task-system-inmem = { version = "0.204.5", path = "src/infra/task-system/inmem", default-features = false } -kamu-task-system-postgres = { version = "0.204.5", path = "src/infra/task-system/postgres", default-features = false } -kamu-task-system-sqlite = { version = "0.204.5", path = "src/infra/task-system/sqlite", default-features = false } -kamu-task-system-repo-tests = { version = "0.204.5", path = "src/infra/task-system/repo-tests", default-features = false } +kamu-task-system-inmem = { version = "0.204.4", path = "src/infra/task-system/inmem", default-features = false } +kamu-task-system-postgres = { version = "0.204.4", path = "src/infra/task-system/postgres", default-features = false } +kamu-task-system-sqlite = { version = "0.204.4", path = "src/infra/task-system/sqlite", default-features = false } +kamu-task-system-repo-tests = { version = "0.204.4", path = "src/infra/task-system/repo-tests", default-features = false } ## ReBAC -kamu-auth-rebac-inmem = { version = "0.204.5", path = "src/infra/auth-rebac/inmem", default-features = false } -kamu-auth-rebac-repo-tests = { version = "0.204.5", path = "src/infra/auth-rebac/repo-tests", default-features = false } -kamu-auth-rebac-postgres = { version = "0.204.5", path = "src/infra/auth-rebac/postgres", default-features = false } -kamu-auth-rebac-sqlite = { version = "0.204.5", path = "src/infra/auth-rebac/sqlite", default-features = false } +kamu-auth-rebac-inmem = { version = "0.204.4", path = "src/infra/auth-rebac/inmem", default-features = false } +kamu-auth-rebac-repo-tests = { version = "0.204.4", path = "src/infra/auth-rebac/repo-tests", default-features = false } +kamu-auth-rebac-postgres = { version = "0.204.4", path = "src/infra/auth-rebac/postgres", default-features = false } +kamu-auth-rebac-sqlite = { version = "0.204.4", path = "src/infra/auth-rebac/sqlite", default-features = false } ## Outbox -kamu-messaging-outbox-inmem = { version = "0.204.5", path = "src/infra/messaging-outbox/inmem", default-features = false } -kamu-messaging-outbox-postgres = { version = "0.204.5", path = "src/infra/messaging-outbox/postgres", default-features = false } -kamu-messaging-outbox-sqlite = { version = "0.204.5", path = "src/infra/messaging-outbox/sqlite", default-features = false } -kamu-messaging-outbox-repo-tests = { version = "0.204.5", path = "src/infra/messaging-outbox/repo-tests", default-features = false } +kamu-messaging-outbox-inmem = { version = "0.204.4", path = "src/infra/messaging-outbox/inmem", default-features = false } +kamu-messaging-outbox-postgres = { version = "0.204.4", path = "src/infra/messaging-outbox/postgres", default-features = false } +kamu-messaging-outbox-sqlite = { version = "0.204.4", path = "src/infra/messaging-outbox/sqlite", default-features = false } +kamu-messaging-outbox-repo-tests = { version = "0.204.4", path = "src/infra/messaging-outbox/repo-tests", default-features = false } # Adapters -kamu-adapter-auth-oso = { version = "0.204.5", path = "src/adapter/auth-oso", default-features = false } -kamu-adapter-flight-sql = { version = "0.204.5", path = "src/adapter/flight-sql", default-features = false } -kamu-adapter-graphql = { version = "0.204.5", path = "src/adapter/graphql", default-features = false } -kamu-adapter-http = { version = "0.204.5", path = "src/adapter/http", default-features = false } -kamu-adapter-odata = { version = "0.204.5", path = "src/adapter/odata", default-features = false } -kamu-adapter-oauth = { version = "0.204.5", path = "src/adapter/oauth", default-features = false } +kamu-adapter-auth-oso = { version = "0.204.4", path = "src/adapter/auth-oso", default-features = false } +kamu-adapter-flight-sql = { version = "0.204.4", path = "src/adapter/flight-sql", default-features = false } +kamu-adapter-graphql = { version = "0.204.4", path = "src/adapter/graphql", default-features = false } +kamu-adapter-http = { version = "0.204.4", path = "src/adapter/http", default-features = false } +kamu-adapter-odata = { version = "0.204.4", path = "src/adapter/odata", default-features = false } +kamu-adapter-oauth = { version = "0.204.4", path = "src/adapter/oauth", default-features = false } # E2E -kamu-cli-e2e-common = { version = "0.204.5", path = "src/e2e/app/cli/common", default-features = false } -kamu-cli-e2e-common-macros = { version = "0.204.5", path = "src/e2e/app/cli/common-macros", default-features = false } -kamu-cli-e2e-repo-tests = { version = "0.204.5", path = "src/e2e/app/cli/repo-tests", default-features = false } +kamu-cli-e2e-common = { version = "0.204.4", path = "src/e2e/app/cli/common", default-features = false } +kamu-cli-e2e-common-macros = { version = "0.204.4", path = "src/e2e/app/cli/common-macros", default-features = false } +kamu-cli-e2e-repo-tests = { version = "0.204.4", path = "src/e2e/app/cli/repo-tests", default-features = false } [workspace.package] -version = "0.204.5" +version = "0.204.4" edition = "2021" homepage = "https://github.com/kamu-data/kamu-cli" repository = "https://github.com/kamu-data/kamu-cli" diff --git a/LICENSE.txt b/LICENSE.txt index 7f5413248..5a0c90820 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -11,7 +11,7 @@ Business Source License 1.1 Licensor: Kamu Data, Inc. -Licensed Work: Kamu CLI Version 0.204.5 +Licensed Work: Kamu CLI Version 0.204.4 The Licensed Work is © 2023 Kamu Data, Inc. Additional Use Grant: You may use the Licensed Work for any purpose, diff --git a/examples/query-commitments/README.md b/examples/query-commitments/README.md deleted file mode 100644 index 1b83763ae..000000000 --- a/examples/query-commitments/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Examples of Verifiable Batch Queries -Query proofs allow you to hold any ODF node forever accountable for the result it provided you, no matter how much time had passed. - -See [batch query commitments documentation](https://docs.kamu.dev/node/commitments/) and [REST API reference](https://docs.kamu.dev/node/protocols/rest-api/) for the overview of this mechanism. - -The included script illustrates: -- how to query data and receive a cryptographic proof -- how to validate commitment consistency on the client side -- and how to ask another node to verify the commitment by reproducing the query. - -Running: -```sh -pip install -r requirements.in -python ./example.py --node-url https://node.example.com --private-key -``` - -Private key is used to show examples of failed verification (by forging a signature). If you don't provide it - those cases will be skipped. diff --git a/examples/query-commitments/example.py b/examples/query-commitments/example.py deleted file mode 100755 index 5a2b78e6e..000000000 --- a/examples/query-commitments/example.py +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env python -import argparse -import copy -import canonicaljson -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey -import hashlib -import requests -import json -import base64 -import base58 - - -def decode_multibase(multibase): - prefix, value = multibase[0], multibase[1:] - if prefix == 'u': - b64_urlsafe_nopad = value - b64_urlsafe = b64_urlsafe_nopad + "=" * ((4 - len(b64_urlsafe_nopad) % 4) % 4) - return base64.urlsafe_b64decode(b64_urlsafe) - elif prefix == 'f': - return bytes.fromhex(value) - elif prefix == 'z': - return base58.b58decode(value) - else: - raise("Malformed multibase value") - - -def encode_multibase_base64_urlsafe_nopad(bin): - return 'u' + base64.urlsafe_b64encode(bin).decode('utf-8').rstrip("=") - - -def multihash_sha3_256_multibase_base16(bin): - return 'f1620' + hashlib.sha3_256(bin).hexdigest() - - -def public_key_from_did(did): - assert did.startswith('did:key:') - multicodec = decode_multibase(did[len('did:key:'):]) - assert multicodec[0:1].hex() == 'ed' # 0xed is multicodec value for Ed25519Pub - return Ed25519PublicKey.from_public_bytes(multicodec[2:]) - -def main(args): - # Query data - resp = requests.get( - args.node_url + "/query", - params=dict( - query=args.query, - include="proof", - ) - ) - resp.raise_for_status() - resp_data = resp.json() - - print(">>> Node's response:") - print(json.dumps(resp_data, indent=2)) - print() - print() - - - # Verify commitment consistency - # This should always be done by the client after receiving a proof to ensure its disputable - assert resp_data["commitment"]["inputHash"] == multihash_sha3_256_multibase_base16( - canonicaljson.encode_canonical_json(resp_data["input"]) - ) - assert resp_data["commitment"]["outputHash"] == multihash_sha3_256_multibase_base16( - canonicaljson.encode_canonical_json(resp_data["output"]) - ) - assert resp_data["commitment"]["subQueriesHash"] == multihash_sha3_256_multibase_base16( - canonicaljson.encode_canonical_json(resp_data["subQueries"]) - ) - - signature = decode_multibase(resp_data["proof"]["proofValue"]) - public_key = public_key_from_did(resp_data["proof"]["verificationMethod"]) - public_key.verify(signature, canonicaljson.encode_canonical_json(resp_data["commitment"])) - print("Commitment is consistent!") - - - print(">>> Commitment:") - commitment = resp_data.copy() - del commitment["output"] - print(json.dumps(commitment, indent=2)) - print() - print() - - - # Remote validation through reproducibility - print(">>> Verifying original commitment:") - resp = requests.post( - args.node_url + "/verify", - json=commitment - ) - resp.raise_for_status() - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == True - - - # Invalid request: input hash - print(">>> Simulating invalid request (input hash):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["commitment"]["inputHash"] = "f1620bd01de1b46f8afe08e128ddd225acdb4457c09919d7c50c2054859a178de51a6" - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "InvalidRequest::InputHash" - - - # Invalid request: subQueries hash - print(">>> Simulating invalid request (subQueries hash):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["commitment"]["subQueriesHash"] = "f1620bd01de1b46f8afe08e128ddd225acdb4457c09919d7c50c2054859a178de51a6" - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "InvalidRequest::SubQueriesHash" - - - # Invalid request: bad signature - print(">>> Simulating invalid request (bad signature):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["proof"]["proofValue"] = "uZbm7fFcWc4l6iyvaKe_txdKntL3h3kvsGHOaKIbPV6c42PH1VnSmpYHMopv4TU68syzgoEdcS26AvpkSQb9dBQ" - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "InvalidRequest::BadSignature" - - - if args.private_key is None: - print("Private key is not provided - skipping tests that require signature forging") - return - - private_key = Ed25519PrivateKey.from_private_bytes(decode_multibase(args.private_key)) - - - # Cannot reproduce the query: output mismatch - # Dataset stays the same but we fake the output hash and the signature - print(">>> Simulating invalid request (output mismatch):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["commitment"]["outputHash"] = "f1620ff7f5beaf16900218a3ac4aae82cdccf764816986c7c739c716cf7dc03112a2d" - - canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) - signature = private_key.sign(canonical_commitment) - invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) - - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "VerificationFailed::OutputMismatch" - - - # Cannot reproduce the query: dataset is missing - # Dataset stays the same but we fake the output hash and the signature - print(">>> Simulating invalid request (dataset is missing):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["input"]["datasets"][0]["id"] = invalid_commitment["input"]["datasets"][0]["id"][:-4] + "beef" - invalid_commitment["commitment"]["inputHash"] = multihash_sha3_256_multibase_base16( - canonicaljson.encode_canonical_json( - invalid_commitment["input"] - ) - ) - - canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) - signature = private_key.sign(canonical_commitment) - invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) - - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "VerificationFailed::DatasetNotFound" - - - # Cannot reproduce the query: block is missing - # Dataset stays the same but we fake the output hash and the signature - print(">>> Simulating invalid request (block is missing):") - invalid_commitment = copy.deepcopy(commitment) - invalid_commitment["input"]["datasets"][0]["blockHash"] = invalid_commitment["input"]["datasets"][0]["blockHash"][:-4] + "beef" - invalid_commitment["commitment"]["inputHash"] = multihash_sha3_256_multibase_base16( - canonicaljson.encode_canonical_json( - invalid_commitment["input"] - ) - ) - - canonical_commitment = canonicaljson.encode_canonical_json(invalid_commitment["commitment"]) - signature = private_key.sign(canonical_commitment) - invalid_commitment["proof"]["proofValue"] = encode_multibase_base64_urlsafe_nopad(signature) - - print(json.dumps(invalid_commitment, indent=2)) - print() - print() - - print("Verification result:") - resp = requests.post( - args.node_url + "/verify", - json=invalid_commitment - ) - resp_data = resp.json() - print(json.dumps(resp_data, indent=2)) - print() - print() - assert resp_data["ok"] == False - assert resp_data["error"]["kind"] == "VerificationFailed::DatasetBlockNotFound" - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--node-url', required=True) - parser.add_argument('--private-key', required=False) - parser.add_argument('--query', default='select block_hash, to from "kamu/net.rocketpool.reth.tokens-minted" order by offset desc limit 1') - args = parser.parse_args() - main(args) \ No newline at end of file diff --git a/examples/query-commitments/requirements.in b/examples/query-commitments/requirements.in deleted file mode 100644 index ab209766b..000000000 --- a/examples/query-commitments/requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -base58 -canonicaljson -cryptography -requests \ No newline at end of file diff --git a/examples/reth-vs-snp500/init-odf-all.sh b/examples/reth-vs-snp500/init-odf-all.sh deleted file mode 100755 index 1739ff387..000000000 --- a/examples/reth-vs-snp500/init-odf-all.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -set -e - -NODE_URL="odf+https://node.demo.kamu.dev/kamu/" - -kamu init || true - -# Root -kamu pull "${NODE_URL}net.rocketpool.reth.tokens-minted" -kamu pull "${NODE_URL}net.rocketpool.reth.tokens-burned" -kamu pull "${NODE_URL}com.cryptocompare.ohlcv.eth-usd" -kamu pull "${NODE_URL}co.alphavantage.tickers.daily.spy" - -kamu pull "${NODE_URL}account.transactions" -kamu pull "${NODE_URL}account.tokens.transfers" - -# Deriv -kamu pull "${NODE_URL}net.rocketpool.reth.mint-burn" -kamu pull "${NODE_URL}account.tokens.portfolio" -kamu pull "${NODE_URL}account.tokens.portfolio.market-value" -kamu pull "${NODE_URL}account.tokens.portfolio.usd" -kamu pull "${NODE_URL}account.whatif.reth-vs-snp500.market-value" -kamu pull "${NODE_URL}account.whatif.reth-vs-snp500.portfolio" diff --git a/examples/reth-vs-snp500/init-odf.sh b/examples/reth-vs-snp500/init-odf.sh deleted file mode 100755 index abec551cc..000000000 --- a/examples/reth-vs-snp500/init-odf.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -set -e - -NODE_URL="odf+https://node.demo.kamu.dev/kamu/" - -kamu init || true - -# Root -kamu pull "${NODE_URL}net.rocketpool.reth.tokens-minted" -kamu pull "${NODE_URL}net.rocketpool.reth.tokens-burned" -kamu pull "${NODE_URL}com.cryptocompare.ohlcv.eth-usd" -kamu pull "${NODE_URL}co.alphavantage.tickers.daily.spy" - -kamu pull "${NODE_URL}account.transactions" -kamu pull "${NODE_URL}account.tokens.transfers" - -kamu add -r . diff --git a/images/kamu-base-with-data-mt/extra/.kamuconfig b/images/kamu-base-with-data-mt/extra/.kamuconfig index 68000e8f8..046242971 100644 --- a/images/kamu-base-with-data-mt/extra/.kamuconfig +++ b/images/kamu-base-with-data-mt/extra/.kamuconfig @@ -3,9 +3,6 @@ version: 1 content: users: predefined: - - accountName: kamu - isAdmin: true - avatarUrl: https://avatars.githubusercontent.com/u/50896974?s=200&v=4 - accountName: sh101-bowen avatarUrl: https://cdn-icons-png.flaticon.com/512/3118/3118054.png - accountName: sh102-gambier diff --git a/images/kamu-base-with-data-mt/init-workspace.py b/images/kamu-base-with-data-mt/init-workspace.py index 286c24ce4..1d2b50a36 100755 --- a/images/kamu-base-with-data-mt/init-workspace.py +++ b/images/kamu-base-with-data-mt/init-workspace.py @@ -1,55 +1,48 @@ #!/usr/bin/env python -import shutil +import os +import sys import subprocess -from pathlib import Path ############################################################################### -CURRENT_PATH = Path(__file__).resolve().parent S3_REPO_URL = "s3://datasets.kamu.dev/odf/v2/example-mt/" - ############################################################################### def s3_listdir(url): return [ line.strip().split(' ')[1] for line in subprocess.run( - f"aws s3 ls {url}", - shell=True, + f"aws s3 ls {url}", + shell=True, text=True, check=True, capture_output=True, ).stdout.splitlines() ] - def s3_cat(url): return subprocess.run( - f"aws s3 cp {url} -", - shell=True, + f"aws s3 cp {url} -", + shell=True, text=True, check=True, capture_output=True, ).stdout.strip() - ############################################################################### subprocess.run( - "kamu init --multi-tenant --exists-ok", + "kamu init --multi-tenant --exists-ok", shell=True, check=True, ) -shutil.copy(CURRENT_PATH / "extra/.kamuconfig", ".kamuconfig") - for did in s3_listdir(S3_REPO_URL): url = S3_REPO_URL + did alias = s3_cat(f"{S3_REPO_URL}{did}info/alias") account, name = alias.split('/', 1) - subprocess.run( f"kamu --account {account} pull --no-alias {url} --as {name}", shell=True,