Add verify and compact commands

src/app/cli/src/commands/verify_command.rs

@@ -152,7 +152,6 @@ impl VerifyCommand {
 
         Ok(self
             .verification_svc
-            .clone()
             .verify_multi(filtered_requests, options, listener)
             .await)
     }

src/e2e/app/cli/inmem/tests/tests/commands/mod.rs

@@ -8,6 +8,7 @@
 // by the Apache License, Version 2.0.
 
 mod test_add_command;
+mod test_compact_command;
 mod test_complete_command;
 mod test_config_command;
 mod test_delete_command;
@@ -18,3 +19,4 @@ mod test_repo_alias_command;
 mod test_sql_command;
 mod test_system_api_server_gql_query;
 mod test_system_generate_token_command;
+mod test_verify_command;

src/e2e/app/cli/inmem/tests/tests/commands/test_compact_command.rs

@@ -0,0 +1,36 @@
+// Copyright Kamu Data, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+use kamu_cli_e2e_common::prelude::*;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_compact_hard
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_compact_keep_metadata_only
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_compact_verify
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

src/e2e/app/cli/inmem/tests/tests/commands/test_verify_command.rs

@@ -0,0 +1,36 @@
+// Copyright Kamu Data, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+use kamu_cli_e2e_common::prelude::*;
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_verify_regular_dataset
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_verify_recursive
+    extra_test_groups = "containerized, engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+kamu_cli_execute_command_e2e_test!(
+    storage = inmem,
+    fixture = kamu_cli_e2e_repo_tests::test_verify_integrity
+    extra_test_groups = "engine, ingest, datafusion"
+);
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

src/e2e/app/cli/repo-tests/src/commands/mod.rs

@@ -8,6 +8,7 @@
 // by the Apache License, Version 2.0.
 
 mod test_add_command;
+mod test_compact_command;
 mod test_complete_command;
 mod test_config_command;
 mod test_delete_command;
@@ -18,8 +19,10 @@ mod test_repo_alias_command;
 mod test_sql_command;
 mod test_system_api_server_gql_query;
 mod test_system_generate_token_command;
+mod test_verify_command;
 
 pub use test_add_command::*;
+pub use test_compact_command::*;
 pub use test_complete_command::*;
 pub use test_config_command::*;
 pub use test_delete_command::*;
@@ -30,3 +33,4 @@ pub use test_repo_alias_command::*;
 pub use test_sql_command::*;
 pub use test_system_api_server_gql_query::*;
 pub use test_system_generate_token_command::*;
+pub use test_verify_command::*;

src/e2e/app/cli/repo-tests/src/commands/test_compact_command.rs

@@ -0,0 +1,208 @@
+// Copyright Kamu Data, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+use std::assert_matches::assert_matches;
+
+use kamu_cli_e2e_common::DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR;
+use kamu_cli_puppet::extensions::KamuCliPuppetExt;
+use kamu_cli_puppet::KamuCliPuppet;
+use opendatafabric::{DatasetName, MetadataEvent};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+pub async fn test_compact_hard(kamu: KamuCliPuppet) {
+    let dataset_name = DatasetName::new_unchecked("player-scores");
+
+    kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR)
+        .await
+        .success();
+
+    let data = indoc::indoc!(
+        r#"
+            {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90}
+        "#,
+    );
+
+    kamu.ingest_data(&dataset_name, data).await;
+    let data = indoc::indoc!(
+        r#"
+            {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 90}
+        "#,
+    );
+
+    kamu.ingest_data(&dataset_name, data).await;
+
+    let blocks_before_compacting = kamu.list_blocks(&dataset_name).await;
+
+    let assert = kamu
+        .execute([
+            "--yes",
+            "system",
+            "compact",
+            dataset_name.as_str(),
+            "--hard",
+        ])
+        .await
+        .success();
+
+    let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap();
+
+    assert!(
+        stderr.contains(indoc::indoc!(
+            r#"
+                1 dataset(s) were compacted
+            "#
+        )),
+        "Unexpected output:\n{stderr}",
+    );
+
+    let blocks_after_compacting = kamu.list_blocks(&dataset_name).await;
+    assert_eq!(
+        blocks_before_compacting.len() - 1,
+        blocks_after_compacting.len()
+    );
+    assert_matches!(
+        blocks_after_compacting.first().unwrap().block.event,
+        MetadataEvent::AddData(_)
+    );
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+pub async fn test_compact_keep_metadata_only(kamu: KamuCliPuppet) {
+    let dataset_name = DatasetName::new_unchecked("player-scores");
+
+    kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR)
+        .await
+        .success();
+
+    let data = indoc::indoc!(
+        r#"
+            {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90}
+        "#,
+    );
+
+    kamu.ingest_data(&dataset_name, data).await;
+    let data = indoc::indoc!(
+        r#"
+            {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 90}
+        "#,
+    );
+
+    kamu.ingest_data(&dataset_name, data).await;
+
+    let blocks_before_compacting = kamu.list_blocks(&dataset_name).await;
+
+    let assert = kamu
+        .execute([
+            "--yes",
+            "system",
+            "compact",
+            dataset_name.as_str(),
+            "--hard",
+            "--keep-metadata-only",
+        ])
+        .await
+        .success();
+
+    let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap();
+
+    assert!(
+        stderr.contains(indoc::indoc!(
+            r#"
+                1 dataset(s) were compacted
+            "#
+        )),
+        "Unexpected output:\n{stderr}",
+    );
+
+    let blocks_after_compacting = kamu.list_blocks(&dataset_name).await;
+    assert_eq!(
+        blocks_before_compacting.len() - 2,
+        blocks_after_compacting.len()
+    );
+    assert_matches!(
+        blocks_after_compacting.first().unwrap().block.event,
+        MetadataEvent::SetDataSchema(_)
+    );
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+pub async fn test_compact_verify(kamu: KamuCliPuppet) {
+    let dataset_name = DatasetName::new_unchecked("player-scores");
+
+    kamu.execute_with_input(["add", "--stdin"], DATASET_ROOT_PLAYER_SCORES_SNAPSHOT_STR)
+        .await
+        .success();
+
+    let data = indoc::indoc!(
+        r#"
+            {"match_time": "2000-01-01", "match_id": 2, "player_id": "Bob", "score": 90}
+        "#,
+    );
+
+    kamu.ingest_data(&dataset_name, data).await;
+    let data = indoc::indoc!(
+        r#"
+            {"match_time": "2000-01-01", "match_id": 1, "player_id": "Alice", "score": 90}
+        "#,
+    );
+
+    kamu.ingest_data(&dataset_name, data).await;
+
+    let blocks_before_compacting = kamu.list_blocks(&dataset_name).await;
+
+    let assert = kamu
+        .execute([
+            "--yes",
+            "system",
+            "compact",
+            dataset_name.as_str(),
+            "--hard",
+            "--verify",
+        ])
+        .await
+        .success();
+
+    let stderr = std::str::from_utf8(&assert.get_output().stderr).unwrap();
+
+    assert!(
+        stderr.contains(
+            indoc::indoc!(
+                r#"
+                verify with dataset_ref: player-scores
+            "#
+            )
+            .trim()
+        ),
+        "Unexpected output:\n{stderr}",
+    );
+
+    assert!(
+        stderr.contains(indoc::indoc!(
+            r#"
+                1 dataset(s) were compacted
+            "#
+        )),
+        "Unexpected output:\n{stderr}",
+    );
+
+    let blocks_after_compacting = kamu.list_blocks(&dataset_name).await;
+    assert_eq!(
+        blocks_before_compacting.len() - 1,
+        blocks_after_compacting.len()
+    );
+    assert_matches!(
+        blocks_after_compacting.first().unwrap().block.event,
+        MetadataEvent::AddData(_)
+    );
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////