From cb92c9925aeca2af39cdbb2a61c4c0d8399756f4 Mon Sep 17 00:00:00 2001 From: Francois Campbell Date: Mon, 12 Aug 2024 10:07:13 -0400 Subject: [PATCH] SNOW-1565541 Implement fetching events shared from consumer account (#1427) Adds `--consumer-org` and `--consumer-account` flags to allow devs to fetch events shared from a specific consumer installation of the app. Since shared events are copied into the provider's event table, everything keeps working the same as for locally installed apps. The only concern is if the organization is set up to send events to a different account using https://docs.snowflake.com/en/developer-guide/native-apps/setting-up-logging-and-events#configure-an-account-to-store-shared-events. We can't really know if this is the case since we're most likely not ORGADMIN, so the best we can do it detect when the account doesn't have an event table set up and alert the user. --- .../cli/_plugins/nativeapp/commands.py | 33 ++++++- .../cli/_plugins/nativeapp/manager.py | 33 ++++++- tests/__snapshots__/test_help_messages.ambr | 97 +++++++++++-------- tests/nativeapp/test_manager.py | 28 +++++- tests_integration/nativeapp/test_events.py | 38 +++++++- 5 files changed, 172 insertions(+), 57 deletions(-) diff --git a/src/snowflake/cli/_plugins/nativeapp/commands.py b/src/snowflake/cli/_plugins/nativeapp/commands.py index e6dd78231e..b0ca5a2ac0 100644 --- a/src/snowflake/cli/_plugins/nativeapp/commands.py +++ b/src/snowflake/cli/_plugins/nativeapp/commands.py @@ -22,7 +22,7 @@ from typing import Generator, Iterable, List, Optional, cast import typer -from click import ClickException +from click import ClickException, UsageError from snowflake.cli._plugins.nativeapp.common_flags import ( ForceOption, InteractiveOption, @@ -450,6 +450,13 @@ def app_events( "--scope", help="Restrict results to a specific scope name. Can be specified multiple times.", ), + consumer_org: str = typer.Option( + default="", help="The name of the consumer organization." + ), + consumer_account: str = typer.Option( + default="", + help="The name of the consumer account in the organization.", + ), first: int = typer.Option( default=-1, show_default=False, @@ -475,15 +482,27 @@ def app_events( ), **options, ): - """Fetches events for this app from the event table configured in Snowflake.""" + """Fetches events for this app from the event table configured in Snowflake. + + By default, this command will fetch events generated by an app installed in the + current connection's account. To fetch events generated by an app installed + in a consumer account, use the --consumer-org and --consumer-account options. + This requires event sharing to be set up to route events to the provider account: + https://docs.snowflake.com/en/developer-guide/native-apps/setting-up-logging-and-events + """ if first >= 0 and last >= 0: - raise ClickException("--first and --last cannot be used together.") + raise UsageError("--first and --last cannot be used together.") + + if (consumer_org and not consumer_account) or ( + consumer_account and not consumer_org + ): + raise UsageError("--consumer-org and --consumer-account must be used together.") if follow: if until: - raise ClickException("--follow and --until cannot be used together.") + raise UsageError("--follow and --until cannot be used together.") if first >= 0: - raise ClickException("--follow and --first cannot be used together.") + raise UsageError("--follow and --first cannot be used together.") assert_project_type("native_app") @@ -505,6 +524,8 @@ def app_events( interval_seconds=follow_interval, record_types=record_type_names, scopes=scopes, + consumer_org=consumer_org, + consumer_account=consumer_account, ) ) # Append a newline at the end to make the CLI output clean when we hit Ctrl-C @@ -519,6 +540,8 @@ def app_events( scopes=scopes, first=first, last=last, + consumer_org=consumer_org, + consumer_account=consumer_account, ) ) diff --git a/src/snowflake/cli/_plugins/nativeapp/manager.py b/src/snowflake/cli/_plugins/nativeapp/manager.py index 8ca7686a0c..3b9fce5050 100644 --- a/src/snowflake/cli/_plugins/nativeapp/manager.py +++ b/src/snowflake/cli/_plugins/nativeapp/manager.py @@ -799,6 +799,8 @@ def get_events( until: str | datetime | None = None, record_types: list[str] | None = None, scopes: list[str] | None = None, + consumer_org: str = "", + consumer_account: str = "", first: int = -1, last: int = -1, ) -> list[dict]: @@ -811,8 +813,20 @@ def get_events( if not self.account_event_table: raise NoEventTableForAccount() - # resource_attributes:"snow.database.name" uses the unquoted/uppercase app name + # resource_attributes uses the unquoted/uppercase app and package name app_name = unquote_identifier(self.app_name) + package_name = unquote_identifier(self.package_name) + org_name = unquote_identifier(consumer_org) + account_name = unquote_identifier(consumer_account) + app_clause = ( + f"resource_attributes:\"snow.database.name\" = '{app_name}'" + if not (consumer_org and consumer_account) + else ( + f"resource_attributes:\"snow.application.package.name\" = '{package_name}' " + f"and resource_attributes:\"snow.application.consumer.organization\" = '{org_name}' " + f"and resource_attributes:\"snow.application.consumer.name\" = '{account_name}'" + ) + ) if isinstance(since, datetime): since_clause = f"and timestamp >= '{since}'" elif isinstance(since, str) and since: @@ -840,7 +854,7 @@ def get_events( select * from ( select timestamp, value::varchar value from {self.account_event_table} - where resource_attributes:"snow.database.name" = '{app_name}' + where ({app_clause}) {since_clause} {until_clause} {types_clause} @@ -862,11 +876,18 @@ def stream_events( since: str | datetime | None = None, record_types: list[str] | None = None, scopes: list[str] | None = None, + consumer_org: str = "", + consumer_account: str = "", last: int = -1, ) -> Generator[dict, None, None]: try: events = self.get_events( - since=since, record_types=record_types, scopes=scopes, last=last + since=since, + record_types=record_types, + scopes=scopes, + consumer_org=consumer_org, + consumer_account=consumer_account, + last=last, ) yield from events # Yield the initial batch of events last_event_time = events[-1]["TIMESTAMP"] @@ -875,7 +896,11 @@ def stream_events( time.sleep(interval_seconds) previous_events = events events = self.get_events( - since=last_event_time, record_types=record_types, scopes=scopes + since=last_event_time, + record_types=record_types, + scopes=scopes, + consumer_org=consumer_org, + consumer_account=consumer_account, ) if not events: continue diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index 272e49b9b2..e7a6285677 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -251,48 +251,61 @@ Usage: default app events [OPTIONS] Fetches events for this app from the event table configured in Snowflake. - - +- Options --------------------------------------------------------------------+ - | --since TEXT Fetch events that are | - | newer than this time ago, | - | in Snowflake interval | - | syntax. | - | --until TEXT Fetch events that are | - | older than this time ago, | - | in Snowflake interval | - | syntax. | - | --type [log|span|span_event] Restrict results to | - | specific record type. Can | - | be specified multiple | - | times. | - | --scope TEXT Restrict results to a | - | specific scope name. Can | - | be specified multiple | - | times. | - | --first INTEGER Fetch only the first N | - | events. Cannot be used | - | with --last. | - | --last INTEGER Fetch only the last N | - | events. Cannot be used | - | with --first. | - | --follow -f Continue polling for | - | events. Implies --last 20 | - | unless overridden or the | - | --since flag is used. | - | --follow-interval INTEGER Polling interval in | - | seconds when using the | - | --follow flag. | - | [default: 10] | - | --project -p TEXT Path where Snowflake | - | project resides. Defaults | - | to current working | - | directory. | - | --env TEXT String in format of | - | key=value. Overrides | - | variables from env section | - | used for templating. | - | --help -h Show this message and | - | exit. | + By default, this command will fetch events generated by an app installed in + the current connection's account. To fetch events generated by an app + installed in a consumer account, use the --consumer-org and --consumer-account + options. This requires event sharing to be set up to route events to the + provider account: + https://docs.snowflake.com/en/developer-guide/native-apps/setting-up-logging-a + nd-events + + +- Options --------------------------------------------------------------------+ + | --since TEXT Fetch events that are | + | newer than this time ago, | + | in Snowflake interval | + | syntax. | + | --until TEXT Fetch events that are | + | older than this time ago, | + | in Snowflake interval | + | syntax. | + | --type [log|span|span_event] Restrict results to | + | specific record type. Can | + | be specified multiple | + | times. | + | --scope TEXT Restrict results to a | + | specific scope name. Can | + | be specified multiple | + | times. | + | --consumer-org TEXT The name of the consumer | + | organization. | + | --consumer-account TEXT The name of the consumer | + | account in the | + | organization. | + | --first INTEGER Fetch only the first N | + | events. Cannot be used | + | with --last. | + | --last INTEGER Fetch only the last N | + | events. Cannot be used | + | with --first. | + | --follow -f Continue polling for | + | events. Implies --last 20 | + | unless overridden or the | + | --since flag is used. | + | --follow-interval INTEGER Polling interval in | + | seconds when using the | + | --follow flag. | + | [default: 10] | + | --project -p TEXT Path where Snowflake | + | project resides. Defaults | + | to current working | + | directory. | + | --env TEXT String in format of | + | key=value. Overrides | + | variables from env | + | section used for | + | templating. | + | --help -h Show this message and | + | exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ | --connection,--environment -c TEXT Name of the connection, as defined | diff --git a/tests/nativeapp/test_manager.py b/tests/nativeapp/test_manager.py index 3e4a74e9c1..9e0e7dad41 100644 --- a/tests/nativeapp/test_manager.py +++ b/tests/nativeapp/test_manager.py @@ -1386,6 +1386,21 @@ def test_account_event_table_not_set_up(mock_execute, temp_dir, mock_cursor): (["log", "span"], "and record_type in ('log','span')"), ], ) +@pytest.mark.parametrize( + ["consumer_org", "consumer_account", "expected_app_clause"], + [ + ("", "", f"resource_attributes:\"snow.database.name\" = 'MYAPP'"), + ( + "testorg", + "testacc", + ( + f"resource_attributes:\"snow.application.package.name\" = 'APP_PKG' " + f"and resource_attributes:\"snow.application.consumer.organization\" = 'TESTORG' " + f"and resource_attributes:\"snow.application.consumer.name\" = 'TESTACC'" + ), + ), + ], +) @pytest.mark.parametrize( ["first", "expected_first_clause"], [ @@ -1419,6 +1434,9 @@ def test_get_events( expected_until_clause, types, expected_types_clause, + consumer_org, + consumer_account, + expected_app_clause, scopes, expected_scopes_clause, first, @@ -1443,7 +1461,7 @@ def test_get_events( select * from ( select timestamp, value::varchar value from db.schema.event_table - where resource_attributes:"snow.database.name" = 'MYAPP' + where ({expected_app_clause}) {expected_since_clause} {expected_until_clause} {expected_types_clause} @@ -1468,6 +1486,8 @@ def get_events(): until=until, record_types=types, scopes=scopes, + consumer_org=consumer_org, + consumer_account=consumer_account, first=first, last=last, ) @@ -1512,7 +1532,7 @@ def test_get_events_quoted_app_name( select * from ( select timestamp, value::varchar value from db.schema.event_table - where resource_attributes:"snow.database.name" = 'My Application' + where (resource_attributes:"snow.database.name" = 'My Application') @@ -1579,7 +1599,7 @@ def test_stream_events(mock_execute, mock_account_event_table, temp_dir, mock_cu select * from ( select timestamp, value::varchar value from db.schema.event_table - where resource_attributes:"snow.database.name" = 'MYAPP' + where (resource_attributes:"snow.database.name" = 'MYAPP') @@ -1601,7 +1621,7 @@ def test_stream_events(mock_execute, mock_account_event_table, temp_dir, mock_cu select * from ( select timestamp, value::varchar value from db.schema.event_table - where resource_attributes:"snow.database.name" = 'MYAPP' + where (resource_attributes:"snow.database.name" = 'MYAPP') and timestamp >= '2024-01-01 00:00:00' diff --git a/tests_integration/nativeapp/test_events.py b/tests_integration/nativeapp/test_events.py index 7216cd9a52..123e21e147 100644 --- a/tests_integration/nativeapp/test_events.py +++ b/tests_integration/nativeapp/test_events.py @@ -22,7 +22,7 @@ TEST_ENV = generate_user_env(USER_NAME) -# Tests that snow app events --first N --last M exits with an error +# Tests that snow app events with incompatible flags exits with an error @pytest.mark.integration @enable_definition_v2_feature_flag @pytest.mark.parametrize("test_project", ["napp_init_v1", "napp_init_v2"]) @@ -53,13 +53,47 @@ def test_app_events_mutually_exclusive_options( ["app", "events", *command], env=TEST_ENV, ) - assert result.exit_code == 1, result.output + assert result.exit_code == 2, result.output assert ( f"{flag_names[0]} and {flag_names[1]} cannot be used together." in result.output ) +# Tests that snow app events without paired flags exits with an error +@pytest.mark.integration +@enable_definition_v2_feature_flag +@pytest.mark.parametrize("test_project", ["napp_init_v1", "napp_init_v2"]) +@pytest.mark.parametrize( + ["flag_names", "command"], + [ + [ + ["--consumer-org", "--consumer-account"], + ["--consumer-org", "testorg"], + ], + [ + ["--consumer-org", "--consumer-account"], + ["--consumer-account", "testacc"], + ], + ], +) +def test_app_events_paired_options( + test_project, runner, project_directory, flag_names, command +): + with project_directory(test_project): + # The integration test account doesn't have an event table set up + # but this test is still useful to validate the negative case + result = runner.invoke_with_connection( + ["app", "events", *command], + env=TEST_ENV, + ) + assert result.exit_code == 2, result.output + assert ( + f"{flag_names[0]} and {flag_names[1]} must be used together." + in result.output + ) + + @pytest.mark.integration @enable_definition_v2_feature_flag @pytest.mark.parametrize("test_project", ["napp_init_v1", "napp_init_v2"])