Skip to content

Commit

Permalink
Fix review comments. Iter 2
Browse files Browse the repository at this point in the history
  • Loading branch information
@rmn-boiko
rmn-boiko committed Oct 14, 2024
1 parent ec01c6f commit 159ff46
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 39 deletions.
5 changes: 0 additions & 5 deletions src/e2e/app/cli/common/src/kamu_api_server_client_ext.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,11 +260,6 @@ impl KamuApiServerClientExt for KamuApiServerClient {
account_name_maybe: Option<AccountName>,
) -> DatasetId {
let dataset_id = self.create_player_scores_dataset(token).await;

// TODO: Use the alias from the reply, after fixing the bug:
// https://github.com/kamu-data/kamu-cli/issues/891

// At the moment, only single-tenant
let dataset_alias = DatasetAlias::new(
account_name_maybe,
DatasetName::new_unchecked("player-scores"),
Expand Down
4 changes: 4 additions & 0 deletions src/e2e/app/cli/inmem/tests/tests/test_smart_transfer_protocol.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,15 @@ kamu_cli_run_api_server_e2e_test!(
kamu_cli_execute_command_e2e_test!(
storage = inmem,
fixture = kamu_cli_e2e_repo_tests::test_smart_pull_set_watermark,
options = Options::default().with_frozen_system_time(),
);

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

kamu_cli_execute_command_e2e_test!(
storage = inmem,
fixture = kamu_cli_e2e_repo_tests::test_smart_pull_reset_derivative,
options = Options::default().with_frozen_system_time(),
extra_test_groups = "containerized, engine, ingest, transform, datafusion"
);

Expand All @@ -109,6 +111,7 @@ kamu_cli_run_api_server_e2e_test!(
kamu_cli_execute_command_e2e_test!(
storage = inmem,
fixture = kamu_cli_e2e_repo_tests::test_smart_push_pull_s3,
options = Options::default().with_frozen_system_time(),
extra_test_groups = "containerized, engine, ingest, datafusion"
);

Expand All @@ -117,6 +120,7 @@ kamu_cli_execute_command_e2e_test!(
kamu_cli_execute_command_e2e_test!(
storage = inmem,
fixture = kamu_cli_e2e_repo_tests::test_smart_pull_derivative,
options = Options::default().with_frozen_system_time(),
extra_test_groups = "containerized, engine, ingest, transform, datafusion"
);

Expand Down
65 changes: 31 additions & 34 deletions src/e2e/app/cli/repo-tests/src/test_smart_transfer_protocol.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -452,7 +452,7 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien
// 2. Pushing datasets to the API server
{
kamu_in_push_workspace
.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap()));
.set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap()));

// 2.1. Add datasets
{
Expand Down Expand Up @@ -566,8 +566,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien
+--------+----+----------------------+--------------------------+----------+-----------+-------+
| offset | op | system_time | match_time | match_id | player_id | score |
+--------+----+----------------------+--------------------------+----------+-----------+-------+
| 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 |
| 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 |
| 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 |
| 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 |
+--------+----+----------------------+--------------------------+----------+-----------+-------+
"#
);
Expand All @@ -590,8 +590,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| offset | op | system_time | match_time | place | match_id | player_id | score |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 |
| 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
| 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 |
| 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
"#
);
Expand Down Expand Up @@ -659,8 +659,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien
+--------+----+----------------------+--------------------------+----------+-----------+-------+
| offset | op | system_time | match_time | match_id | player_id | score |
+--------+----+----------------------+--------------------------+----------+-----------+-------+
| 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 |
| 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 |
| 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 |
| 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 |
+--------+----+----------------------+--------------------------+----------+-----------+-------+
"#
);
Expand All @@ -669,8 +669,8 @@ pub async fn test_smart_push_pull_all(kamu_api_server_client: KamuApiServerClien
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| offset | op | system_time | match_time | place | match_id | player_id | score |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
| 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 |
| 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
| 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
"#
);
Expand Down Expand Up @@ -707,7 +707,7 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe
// 2. Pushing datasets to the API server
{
kamu_in_push_workspace
.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap()));
.set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap()));

// 2.1. Add datasets
{
Expand Down Expand Up @@ -764,7 +764,7 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe
{
let mut kamu_in_pull_workspace = KamuCliPuppet::new_workspace_tmp().await;
kamu_in_pull_workspace
.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap()));
.set_system_time(Some(DateTime::from_str("2050-01-02T03:04:05Z").unwrap()));

// Pull datasets one by one and check data
run_and_assert_command_success(
Expand Down Expand Up @@ -807,8 +807,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe
+--------+----+----------------------+--------------------------+----------+-----------+-------+
| offset | op | system_time | match_time | match_id | player_id | score |
+--------+----+----------------------+--------------------------+----------+-----------+-------+
| 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 |
| 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 |
| 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 |
| 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 |
+--------+----+----------------------+--------------------------+----------+-----------+-------+
"#
);
Expand All @@ -831,8 +831,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| offset | op | system_time | match_time | place | match_id | player_id | score |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 |
| 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
| 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 |
| 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
"#
);
Expand Down Expand Up @@ -882,8 +882,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe
+--------+----+----------------------+--------------------------+----------+-----------+-------+
| offset | op | system_time | match_time | match_id | player_id | score |
+--------+----+----------------------+--------------------------+----------+-----------+-------+
| 2 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 |
| 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 |
| 2 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | Alice | 70 |
| 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | Charlie | 90 |
+--------+----+----------------------+--------------------------+----------+-----------+-------+
"#
);
Expand All @@ -892,8 +892,8 @@ pub async fn test_smart_push_pull_recursive(kamu_api_server_client: KamuApiServe
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| offset | op | system_time | match_time | place | match_id | player_id | score |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| 2 | 1 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
| 3 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 |
| 2 | 1 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
| 3 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 2 | 2 | Charlie | 90 |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
"#
);
Expand Down Expand Up @@ -925,7 +925,7 @@ pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) {
"pull",
dataset_name.as_str(),
"--set-watermark",
"2000-01-01T00:00:00Z",
"2051-01-02T03:04:05Z",
])
.await
.success();
Expand All @@ -940,8 +940,7 @@ pub async fn test_smart_pull_set_watermark(kamu: KamuCliPuppet) {

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) {
kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap()));
pub async fn test_smart_pull_reset_derivative(kamu: KamuCliPuppet) {
let dataset_name = DatasetName::new_unchecked("player-scores");
let dataset_derivative_name = DatasetName::new_unchecked("leaderboard");

Expand Down Expand Up @@ -988,8 +987,8 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) {
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| offset | op | system_time | match_time | place | match_id | player_id | score |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 |
| 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
| 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 |
| 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
"#
);
Expand Down Expand Up @@ -1043,8 +1042,8 @@ pub async fn test_smart_pull_reset_derivative(mut kamu: KamuCliPuppet) {
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| offset | op | system_time | match_time | place | match_id | player_id | score |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 |
| 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-02T00:00:00.001Z | 1 | 2 | Charlie | 90 |
| 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00Z | 2 | 2 | Alice | 70 |
| 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-02T00:00:00.001Z | 1 | 2 | Charlie | 90 |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
"#
);
Expand Down Expand Up @@ -1123,8 +1122,7 @@ pub async fn test_smart_push_visibility(kamu_api_server_client: KamuApiServerCli

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) {
kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap()));
pub async fn test_smart_push_pull_s3(kamu: KamuCliPuppet) {
let dataset_name = DatasetName::new_unchecked("player-scores");

kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR)
Expand Down Expand Up @@ -1176,8 +1174,8 @@ pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) {
+--------+----+----------------------+--------------------------+----------+-----------+-------+
| offset | op | system_time | match_time | match_id | player_id | score |
+--------+----+----------------------+--------------------------+----------+-----------+-------+
| 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 |
| 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 |
| 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | Alice | 100 |
| 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 1 | Bob | 80 |
+--------+----+----------------------+--------------------------+----------+-----------+-------+
"#
);
Expand All @@ -1188,8 +1186,7 @@ pub async fn test_smart_push_pull_s3(mut kamu: KamuCliPuppet) {

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

pub async fn test_smart_pull_derivative(mut kamu: KamuCliPuppet) {
kamu.set_system_time(Some(DateTime::from_str("2000-01-01T00:00:00Z").unwrap()));
pub async fn test_smart_pull_derivative(kamu: KamuCliPuppet) {
let dataset_name = DatasetName::new_unchecked("player-scores");
let dataset_derivative_name = DatasetName::new_unchecked("leaderboard");

Expand Down Expand Up @@ -1249,8 +1246,8 @@ pub async fn test_smart_pull_derivative(mut kamu: KamuCliPuppet) {
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| offset | op | system_time | match_time | place | match_id | player_id | score |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
| 0 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 |
| 1 | 0 | 2000-01-01T00:00:00Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
| 0 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00Z | 1 | 1 | Alice | 100 |
| 1 | 0 | 2050-01-02T03:04:05Z | 2000-01-01T00:00:00.001Z | 2 | 1 | Bob | 80 |
+--------+----+----------------------+--------------------------+-------+----------+-----------+-------+
"#
);
Expand Down
2 changes: 2 additions & 0 deletions src/utils/kamu-cli-puppet/src/kamu_cli_puppet_ext.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,8 @@ pub struct DatasetRecord {
#[serde(rename = "ID")]
pub id: DatasetID,
pub name: DatasetName,
// CLI returns regular ENUM DatasetKind(Root/Derivative) for local datasets
// but for remote it is Remote(DatasetKind) type
pub kind: String,
pub head: Multihash,
pub pulled: Option<DateTime<Utc>>,
Expand Down

0 comments on commit 159ff46

Please sign in to comment.