Skip to content

Commit

Permalink
SNOW-1565541 Implement fetching events shared from consumer account (#…
Browse files Browse the repository at this point in the history 
…1427)

Adds `--consumer-org` and `--consumer-account` flags to allow devs to fetch events shared from a specific consumer installation of the app. Since shared events are copied into the provider's event table, everything keeps working the same as for locally installed apps. The only concern is if the organization is set up to send events to a different account using https://docs.snowflake.com/en/developer-guide/native-apps/setting-up-logging-and-events#configure-an-account-to-store-shared-events. We can't really know if this is the case since we're most likely not ORGADMIN, so the best we can do it detect when the account doesn't have an event table set up and alert the user.
  • Loading branch information
@sfc-gh-fcampbell
sfc-gh-fcampbell authored Aug 12, 2024
1 parent bb4ee2b commit cb92c99
Show file tree
Hide file tree
Showing 5 changed files with 172 additions and 57 deletions.
33 changes: 28 additions & 5 deletions src/snowflake/cli/_plugins/nativeapp/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from typing import Generator, Iterable, List, Optional, cast

import typer
from click import ClickException
from click import ClickException, UsageError
from snowflake.cli._plugins.nativeapp.common_flags import (
ForceOption,
InteractiveOption,
Expand Down Expand Up @@ -450,6 +450,13 @@ def app_events(
"--scope",
help="Restrict results to a specific scope name. Can be specified multiple times.",
),
consumer_org: str = typer.Option(
default="", help="The name of the consumer organization."
),
consumer_account: str = typer.Option(
default="",
help="The name of the consumer account in the organization.",
),
first: int = typer.Option(
default=-1,
show_default=False,
Expand All @@ -475,15 +482,27 @@ def app_events(
),
**options,
):
"""Fetches events for this app from the event table configured in Snowflake."""
"""Fetches events for this app from the event table configured in Snowflake.
By default, this command will fetch events generated by an app installed in the
current connection's account. To fetch events generated by an app installed
in a consumer account, use the --consumer-org and --consumer-account options.
This requires event sharing to be set up to route events to the provider account:
https://docs.snowflake.com/en/developer-guide/native-apps/setting-up-logging-and-events
"""
if first >= 0 and last >= 0:
raise ClickException("--first and --last cannot be used together.")
raise UsageError("--first and --last cannot be used together.")

if (consumer_org and not consumer_account) or (
consumer_account and not consumer_org
):
raise UsageError("--consumer-org and --consumer-account must be used together.")

if follow:
if until:
raise ClickException("--follow and --until cannot be used together.")
raise UsageError("--follow and --until cannot be used together.")
if first >= 0:
raise ClickException("--follow and --first cannot be used together.")
raise UsageError("--follow and --first cannot be used together.")

assert_project_type("native_app")

Expand All @@ -505,6 +524,8 @@ def app_events(
interval_seconds=follow_interval,
record_types=record_type_names,
scopes=scopes,
consumer_org=consumer_org,
consumer_account=consumer_account,
)
)
# Append a newline at the end to make the CLI output clean when we hit Ctrl-C
Expand All @@ -519,6 +540,8 @@ def app_events(
scopes=scopes,
first=first,
last=last,
consumer_org=consumer_org,
consumer_account=consumer_account,
)
)

Expand Down
33 changes: 29 additions & 4 deletions src/snowflake/cli/_plugins/nativeapp/manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -799,6 +799,8 @@ def get_events(
until: str | datetime | None = None,
record_types: list[str] | None = None,
scopes: list[str] | None = None,
consumer_org: str = "",
consumer_account: str = "",
first: int = -1,
last: int = -1,
) -> list[dict]:
Expand All @@ -811,8 +813,20 @@ def get_events(
if not self.account_event_table:
raise NoEventTableForAccount()

# resource_attributes:"snow.database.name" uses the unquoted/uppercase app name
# resource_attributes uses the unquoted/uppercase app and package name
app_name = unquote_identifier(self.app_name)
package_name = unquote_identifier(self.package_name)
org_name = unquote_identifier(consumer_org)
account_name = unquote_identifier(consumer_account)
app_clause = (
f"resource_attributes:\"snow.database.name\" = '{app_name}'"
if not (consumer_org and consumer_account)
else (
f"resource_attributes:\"snow.application.package.name\" = '{package_name}' "
f"and resource_attributes:\"snow.application.consumer.organization\" = '{org_name}' "
f"and resource_attributes:\"snow.application.consumer.name\" = '{account_name}'"
)
)
if isinstance(since, datetime):
since_clause = f"and timestamp >= '{since}'"
elif isinstance(since, str) and since:
Expand Down Expand Up @@ -840,7 +854,7 @@ def get_events(
select * from (
select timestamp, value::varchar value
from {self.account_event_table}
where resource_attributes:"snow.database.name" = '{app_name}'
where ({app_clause})
{since_clause}
{until_clause}
{types_clause}
Expand All @@ -862,11 +876,18 @@ def stream_events(
since: str | datetime | None = None,
record_types: list[str] | None = None,
scopes: list[str] | None = None,
consumer_org: str = "",
consumer_account: str = "",
last: int = -1,
) -> Generator[dict, None, None]:
try:
events = self.get_events(
since=since, record_types=record_types, scopes=scopes, last=last
since=since,
record_types=record_types,
scopes=scopes,
consumer_org=consumer_org,
consumer_account=consumer_account,
last=last,
)
yield from events # Yield the initial batch of events
last_event_time = events[-1]["TIMESTAMP"]
Expand All @@ -875,7 +896,11 @@ def stream_events(
time.sleep(interval_seconds)
previous_events = events
events = self.get_events(
since=last_event_time, record_types=record_types, scopes=scopes
since=last_event_time,
record_types=record_types,
scopes=scopes,
consumer_org=consumer_org,
consumer_account=consumer_account,
)
if not events:
continue
Expand Down
97 changes: 55 additions & 42 deletions tests/__snapshots__/test_help_messages.ambr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,48 +251,61 @@
Usage: default app events [OPTIONS]

Fetches events for this app from the event table configured in Snowflake.

+- Options --------------------------------------------------------------------+
| --since TEXT Fetch events that are |
| newer than this time ago, |
| in Snowflake interval |
| syntax. |
| --until TEXT Fetch events that are |
| older than this time ago, |
| in Snowflake interval |
| syntax. |
| --type [log|span|span_event] Restrict results to |
| specific record type. Can |
| be specified multiple |
| times. |
| --scope TEXT Restrict results to a |
| specific scope name. Can |
| be specified multiple |
| times. |
| --first INTEGER Fetch only the first N |
| events. Cannot be used |
| with --last. |
| --last INTEGER Fetch only the last N |
| events. Cannot be used |
| with --first. |
| --follow -f Continue polling for |
| events. Implies --last 20 |
| unless overridden or the |
| --since flag is used. |
| --follow-interval INTEGER Polling interval in |
| seconds when using the |
| --follow flag. |
| [default: 10] |
| --project -p TEXT Path where Snowflake |
| project resides. Defaults |
| to current working |
| directory. |
| --env TEXT String in format of |
| key=value. Overrides |
| variables from env section |
| used for templating. |
| --help -h Show this message and |
| exit. |
By default, this command will fetch events generated by an app installed in
the current connection's account. To fetch events generated by an app
installed in a consumer account, use the --consumer-org and --consumer-account
options. This requires event sharing to be set up to route events to the
provider account:
https://docs.snowflake.com/en/developer-guide/native-apps/setting-up-logging-a
nd-events

+- Options --------------------------------------------------------------------+
| --since TEXT Fetch events that are |
| newer than this time ago, |
| in Snowflake interval |
| syntax. |
| --until TEXT Fetch events that are |
| older than this time ago, |
| in Snowflake interval |
| syntax. |
| --type [log|span|span_event] Restrict results to |
| specific record type. Can |
| be specified multiple |
| times. |
| --scope TEXT Restrict results to a |
| specific scope name. Can |
| be specified multiple |
| times. |
| --consumer-org TEXT The name of the consumer |
| organization. |
| --consumer-account TEXT The name of the consumer |
| account in the |
| organization. |
| --first INTEGER Fetch only the first N |
| events. Cannot be used |
| with --last. |
| --last INTEGER Fetch only the last N |
| events. Cannot be used |
| with --first. |
| --follow -f Continue polling for |
| events. Implies --last 20 |
| unless overridden or the |
| --since flag is used. |
| --follow-interval INTEGER Polling interval in |
| seconds when using the |
| --follow flag. |
| [default: 10] |
| --project -p TEXT Path where Snowflake |
| project resides. Defaults |
| to current working |
| directory. |
| --env TEXT String in format of |
| key=value. Overrides |
| variables from env |
| section used for |
| templating. |
| --help -h Show this message and |
| exit. |
+------------------------------------------------------------------------------+
+- Connection configuration ---------------------------------------------------+
| --connection,--environment -c TEXT Name of the connection, as defined |
Expand Down
28 changes: 24 additions & 4 deletions tests/nativeapp/test_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1386,6 +1386,21 @@ def test_account_event_table_not_set_up(mock_execute, temp_dir, mock_cursor):
(["log", "span"], "and record_type in ('log','span')"),
],
)
@pytest.mark.parametrize(
["consumer_org", "consumer_account", "expected_app_clause"],
[
("", "", f"resource_attributes:\"snow.database.name\" = 'MYAPP'"),
(
"testorg",
"testacc",
(
f"resource_attributes:\"snow.application.package.name\" = 'APP_PKG' "
f"and resource_attributes:\"snow.application.consumer.organization\" = 'TESTORG' "
f"and resource_attributes:\"snow.application.consumer.name\" = 'TESTACC'"
),
),
],
)
@pytest.mark.parametrize(
["first", "expected_first_clause"],
[
Expand Down Expand Up @@ -1419,6 +1434,9 @@ def test_get_events(
expected_until_clause,
types,
expected_types_clause,
consumer_org,
consumer_account,
expected_app_clause,
scopes,
expected_scopes_clause,
first,
Expand All @@ -1443,7 +1461,7 @@ def test_get_events(
select * from (
select timestamp, value::varchar value
from db.schema.event_table
where resource_attributes:"snow.database.name" = 'MYAPP'
where ({expected_app_clause})
{expected_since_clause}
{expected_until_clause}
{expected_types_clause}
Expand All @@ -1468,6 +1486,8 @@ def get_events():
until=until,
record_types=types,
scopes=scopes,
consumer_org=consumer_org,
consumer_account=consumer_account,
first=first,
last=last,
)
Expand Down Expand Up @@ -1512,7 +1532,7 @@ def test_get_events_quoted_app_name(
select * from (
select timestamp, value::varchar value
from db.schema.event_table
where resource_attributes:"snow.database.name" = 'My Application'
where (resource_attributes:"snow.database.name" = 'My Application')
Expand Down Expand Up @@ -1579,7 +1599,7 @@ def test_stream_events(mock_execute, mock_account_event_table, temp_dir, mock_cu
select * from (
select timestamp, value::varchar value
from db.schema.event_table
where resource_attributes:"snow.database.name" = 'MYAPP'
where (resource_attributes:"snow.database.name" = 'MYAPP')
Expand All @@ -1601,7 +1621,7 @@ def test_stream_events(mock_execute, mock_account_event_table, temp_dir, mock_cu
select * from (
select timestamp, value::varchar value
from db.schema.event_table
where resource_attributes:"snow.database.name" = 'MYAPP'
where (resource_attributes:"snow.database.name" = 'MYAPP')
and timestamp >= '2024-01-01 00:00:00'
Expand Down
38 changes: 36 additions & 2 deletions tests_integration/nativeapp/test_events.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
TEST_ENV = generate_user_env(USER_NAME)


# Tests that snow app events --first N --last M exits with an error
# Tests that snow app events with incompatible flags exits with an error
@pytest.mark.integration
@enable_definition_v2_feature_flag
@pytest.mark.parametrize("test_project", ["napp_init_v1", "napp_init_v2"])
Expand Down Expand Up @@ -53,13 +53,47 @@ def test_app_events_mutually_exclusive_options(
["app", "events", *command],
env=TEST_ENV,
)
assert result.exit_code == 1, result.output
assert result.exit_code == 2, result.output
assert (
f"{flag_names[0]} and {flag_names[1]} cannot be used together."
in result.output
)


# Tests that snow app events without paired flags exits with an error
@pytest.mark.integration
@enable_definition_v2_feature_flag
@pytest.mark.parametrize("test_project", ["napp_init_v1", "napp_init_v2"])
@pytest.mark.parametrize(
["flag_names", "command"],
[
[
["--consumer-org", "--consumer-account"],
["--consumer-org", "testorg"],
],
[
["--consumer-org", "--consumer-account"],
["--consumer-account", "testacc"],
],
],
)
def test_app_events_paired_options(
test_project, runner, project_directory, flag_names, command
):
with project_directory(test_project):
# The integration test account doesn't have an event table set up
# but this test is still useful to validate the negative case
result = runner.invoke_with_connection(
["app", "events", *command],
env=TEST_ENV,
)
assert result.exit_code == 2, result.output
assert (
f"{flag_names[0]} and {flag_names[1]} must be used together."
in result.output
)


@pytest.mark.integration
@enable_definition_v2_feature_flag
@pytest.mark.parametrize("test_project", ["napp_init_v1", "napp_init_v2"])
Expand Down

0 comments on commit cb92c99

Please sign in to comment.