diff --git a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs index bee28b2b5..408f9ac53 100644 --- a/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs +++ b/src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs @@ -260,11 +260,6 @@ impl KamuApiServerClientExt for KamuApiServerClient { account_name_maybe: Option, ) -> DatasetId { let dataset_id = self.create_player_scores_dataset(token).await; - - // TODO: Use the alias from the reply, after fixing the bug: - // https://github.com/kamu-data/kamu-cli/issues/891 - - // At the moment, only single-tenant let dataset_alias = DatasetAlias::new( account_name_maybe, DatasetName::new_unchecked("player-scores"), diff --git a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs index c99f906c0..abb5120d3 100644 --- a/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs @@ -83,6 +83,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark, + options = Options::default().with_frozen_system_time(), ); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -90,6 +91,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative, + options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); @@ -109,6 +111,7 @@ kamu_cli_run_api_server_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3, + options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, datafusion" ); @@ -117,6 +120,7 @@ kamu_cli_execute_command_e2e_test!( kamu_cli_execute_command_e2e_test!( storage = inmem, fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative, + options = Options::default().with_frozen_system_time(), extra_test_groups = "containerized, engine, ingest, transform, datafusion" ); diff --git a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs index 09cb6009e..5deee2f3e 100644 --- a/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs +++ b/src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs @@ -452,7 +452,7 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien // 2. Pushing datasets to the API server { kamu_in_push_workspace - .set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); // 2.1. Add datasets { @@ -566,8 +566,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien +--------+----+----------------------+--------------------------+----------+-----------+-------+ | offset | op | system_time | match_time | match_id | player_id | score | +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); @@ -590,8 +590,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -659,8 +659,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien +--------+----+----------------------+--------------------------+----------+-----------+-------+ | offset | op | system_time | match_time | match_id | player_id | score | +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | - | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | + | 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); @@ -669,8 +669,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | - | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | + | 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -707,7 +707,7 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe // 2. Pushing datasets to the API server { kamu_in_push_workspace - .set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); // 2.1. Add datasets { @@ -764,7 +764,7 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe { let mut kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await; kamu_in_pull_workspace - .set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); + .set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap())); // Pull datasets one by one and check data run_and_assert_command_success( @@ -807,8 +807,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe +--------+----+----------------------+--------------------------+----------+-----------+-------+ | offset | op | system_time | match_time | match_id | player_id | score | +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); @@ -831,8 +831,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -882,8 +882,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe +--------+----+----------------------+--------------------------+----------+-----------+-------+ | offset | op | system_time | match_time | match_id | player_id | score | +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | - | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | + | 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 | +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); @@ -892,8 +892,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | - | 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | + | 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -925,7 +925,7 @@ pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) { "pull", dataset_name.as_str(), "--set-watermark", - "2000-01-01T00:00:00Z", + "2051-01-02T03:04:05Z", ]) .await .success(); @@ -940,8 +940,7 @@ pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { - kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); +pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) { let dataset_name = DatasetName::new_unchecked("player-scores"); let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); @@ -988,8 +987,8 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -1043,8 +1042,8 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) { +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 1 | 2 | Charlie | 90 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 1 | 2 | Charlie | 90 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); @@ -1123,8 +1122,7 @@ pub async fn test_smart_push_visibility(kamu_api_server_client: KamuApiServerCli //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { - kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); +pub async fn test_smart_push_pull_s3(kamu: KamuCliPuppet) { let dataset_name = DatasetName::new_unchecked("player-scores"); kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR) @@ -1176,8 +1174,8 @@ pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { +--------+----+----------------------+--------------------------+----------+-----------+-------+ | offset | op | system_time | match_time | match_id | player_id | score | +--------+----+----------------------+--------------------------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+----------+-----------+-------+ "# ); @@ -1188,8 +1186,7 @@ pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -pub async fn test_smart_pull_derivative(mut kamu: KamuCliPuppet) { - kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap())); +pub async fn test_smart_pull_derivative(kamu: KamuCliPuppet) { let dataset_name = DatasetName::new_unchecked("player-scores"); let dataset_derivative_name = DatasetName::new_unchecked("leaderboard"); @@ -1249,8 +1246,8 @@ pub async fn test_smart_pull_derivative(mut kamu: KamuCliPuppet) { +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ | offset | op | system_time | match_time | place | match_id | player_id | score | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ - | 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | - | 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | + | 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 | + | 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 | +--------+----+----------------------+--------------------------+-------+----------+-----------+-------+ "# ); diff --git a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs index 438ef28a5..bebf25b1e 100644 --- a/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs +++ b/src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs @@ -253,6 +253,8 @@ pub struct DatasetRecord { #[serde(rename = "ID")] pub id: DatasetID, pub name: DatasetName, + // CLI returns regular ENUM DatasetKind(Root/Derivative) for local datasets + // but for remote it is Remote(DatasetKind) type pub kind: String, pub head: Multihash, pub pulled: Option>,