From 4cfe95c4f872cbd06ef0d39433d3aded1bb11cc4 Mon Sep 17 00:00:00 2001 From: Mick Stanciu Date: Mon, 4 Nov 2024 15:38:08 +1100 Subject: [PATCH] DEV-3508 feed account history (#478) * DEVP-3508 WIP activity_log_events * DEVP-3508 WIP activity_log_events * DEVP-3508 activity_log_events * DEVP-3508 activity_log_events * DEVP-3508 cleanup * DEVP-3508 tests * DEVP-3508 tests * DEVP-3508 small fixes * DEVP-3508 small fixes * DEVP-3508 small fixes * DEVP-3508 small fixes * DEVP-3508 small fixes * DEVP-3508 small fixes * DEV-3508 fix query for mssql --- .github/CODEOWNERS | 2 +- pkg/api/configuration_manager.go | 2 +- pkg/api/export_feeds_intg_test.go | 12 +- pkg/api/export_feeds_test.go | 29 ++-- pkg/api/mock_feeds_test.go | 15 ++ .../set_1/feed_activity_log_events_1.json | 128 ++++++++++++++ pkg/api/mocks/set_1/feed_group_users_1.json | 8 +- .../mocks/set_1/outputs/account_histories.csv | 11 ++ pkg/api/mocks/set_1/outputs/group_users.csv | 8 +- .../mocks/set_1/schemas/account_histories.csv | 1 + pkg/internal/feed/drain_feed.go | 3 + pkg/internal/feed/exporter_sql.go | 27 +-- pkg/internal/feed/feed.go | 2 +- pkg/internal/feed/feed_account_history.go | 156 ++++++++++++++++++ pkg/internal/feed/feed_exporter.go | 4 + .../schemas/formatted/account_histories.txt | 13 ++ pkg/internal/feed/mocks/exporter_mock.go | 10 +- pkg/internal/util/uuid.go | 20 +++ pkg/internal/util/uuid_test.go | 44 +++++ 19 files changed, 451 insertions(+), 44 deletions(-) create mode 100644 pkg/api/mocks/set_1/feed_activity_log_events_1.json create mode 100644 pkg/api/mocks/set_1/outputs/account_histories.csv create mode 100644 pkg/api/mocks/set_1/schemas/account_histories.csv create mode 100644 pkg/internal/feed/feed_account_history.go create mode 100644 pkg/internal/feed/fixtures/schemas/formatted/account_histories.txt create mode 100644 pkg/internal/util/uuid.go create mode 100644 pkg/internal/util/uuid_test.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ab9974ff..1fed1ff7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ # https://help.github.com/articles/about-codeowners/ -* @SafetyCulture/api-and-integrations +* @SafetyCulture/developer-platform diff --git a/pkg/api/configuration_manager.go b/pkg/api/configuration_manager.go index 39a1d9a1..56a3d9d1 100644 --- a/pkg/api/configuration_manager.go +++ b/pkg/api/configuration_manager.go @@ -210,7 +210,7 @@ func (c *ConfigurationManager) ApplySafetyGuards() { c.Configuration.Export.Inspection.SkipIds = defaultCfg.Export.Inspection.SkipIds } - if c.Configuration.Report.Format == nil || len(c.Configuration.Report.Format) == 0 { + if len(c.Configuration.Report.Format) == 0 { c.Configuration.Report.Format = defaultCfg.Report.Format } diff --git a/pkg/api/export_feeds_intg_test.go b/pkg/api/export_feeds_intg_test.go index 596fad74..6953669a 100644 --- a/pkg/api/export_feeds_intg_test.go +++ b/pkg/api/export_feeds_intg_test.go @@ -35,7 +35,7 @@ func TestIntegrationDbCreateSchema_should_create_all_schemas(t *testing.T) { BodyString(` { "user_id": "user_123", - "organisation_id": "role_123", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } @@ -84,7 +84,7 @@ func TestIntegrationDbExportFeeds_should_export_all_feeds_to_file(t *testing.T) BodyString(` { "user_id": "user_123", - "organisation_id": "role_123", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } @@ -135,7 +135,7 @@ func TestIntegrationDbExportFeeds_should_perform_incremental_update_on_second_ru BodyString(` { "user_id": "user_123", - "organisation_id": "role_123", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } @@ -202,7 +202,7 @@ func TestIntegrationDbExportFeeds_should_handle_lots_of_rows_ok(t *testing.T) { BodyString(` { "user_id": "user_123", - "organisation_id": "role_123", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } @@ -239,7 +239,7 @@ func TestIntegrationDbExportFeeds_should_update_action_assignees_on_second_run(t BodyString(` { "user_id": "user_123", - "organisation_id": "role_123", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } @@ -295,7 +295,7 @@ func TestGroupUserFeed_Export_should_filter_duplicates(t *testing.T) { BodyString(` { "user_id": "user_123", - "organisation_id": "role_123", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } diff --git a/pkg/api/export_feeds_test.go b/pkg/api/export_feeds_test.go index 0b24317e..fc9b3e5e 100644 --- a/pkg/api/export_feeds_test.go +++ b/pkg/api/export_feeds_test.go @@ -44,6 +44,9 @@ func TestExporterFeedClient_ExportFeeds_should_create_all_schemas_to_file(t *tes filesEqualish(t, "mocks/set_1/schemas/training_course_progresses.csv", filepath.Join(exporter.ExportPath, "training_course_progresses.csv")) filesEqualish(t, "mocks/set_1/schemas/issue_assignees.csv", filepath.Join(exporter.ExportPath, "issue_assignees.csv")) + + filesEqualish(t, "mocks/set_1/schemas/issue_assignees.csv", filepath.Join(exporter.ExportPath, "issue_assignees.csv")) + filesEqualish(t, "mocks/set_1/schemas/account_histories.csv", filepath.Join(exporter.ExportPath, "account_histories.csv")) } func TestExporterFeedClient_ExportFeeds_should_export_all_feeds_to_file(t *testing.T) { @@ -60,8 +63,8 @@ func TestExporterFeedClient_ExportFeeds_should_export_all_feeds_to_file(t *testi Reply(200). BodyString(` { - "user_id": "user_123", - "organisation_id": "role_123", + "user_id": "user_bda3042f16a44249915ddc088adef92b", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } @@ -124,6 +127,7 @@ func TestExporterFeedClient_ExportFeeds_should_export_all_feeds_to_file(t *testi filesEqualish(t, "mocks/set_1/outputs/sheqsy_departments.csv", filepath.Join(exporter.ExportPath, "sheqsy_departments.csv")) filesEqualish(t, "mocks/set_1/outputs/training_course_progresses.csv", filepath.Join(exporter.ExportPath, "training_course_progresses.csv")) + filesEqualish(t, "mocks/set_1/outputs/account_histories.csv", filepath.Join(exporter.ExportPath, "account_histories.csv")) } func TestExporterFeedClient_ExportFeeds_should_err_when_not_auth(t *testing.T) { @@ -165,8 +169,8 @@ func TestExporterFeedClient_ExportFeeds_should_err_when_InitFeed_errors(t *testi Reply(200). BodyString(` { - "user_id": "user_123", - "organisation_id": "role_123", + "user_id": "user_bda3042f16a44249915ddc088adef92b", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } @@ -205,8 +209,8 @@ func TestExporterFeedClient_ExportFeeds_should_err_when_cannot_unmarshal(t *test Reply(200). BodyString(` { - "user_id": "user_123", - "organisation_id": "role_123", + "user_id": "user_bda3042f16a44249915ddc088adef92b", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } @@ -253,8 +257,8 @@ func TestExporterFeedClient_ExportFeeds_should_err_when_cannot_write_rows(t *tes Reply(200). BodyString(` { - "user_id": "user_123", - "organisation_id": "role_123", + "user_id": "user_bda3042f16a44249915ddc088adef92b", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } @@ -290,8 +294,8 @@ func TestExporterFeedClient_ExportFeeds_should_perform_incremental_update_on_sec Reply(200). BodyString(` { - "user_id": "user_123", - "organisation_id": "role_123", + "user_id": "user_bda3042f16a44249915ddc088adef92b", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } @@ -338,6 +342,7 @@ func TestExporterFeedClient_ExportFeeds_should_perform_incremental_update_on_sec filesEqualish(t, "mocks/set_2/outputs/users.csv", filepath.Join(exporter.ExportPath, "users.csv")) filesEqualish(t, "mocks/set_2/outputs/groups.csv", filepath.Join(exporter.ExportPath, "groups.csv")) + filesEqualish(t, "mocks/set_2/outputs/group_users.csv", filepath.Join(exporter.ExportPath, "group_users.csv")) filesEqualish(t, "mocks/set_2/outputs/schedules.csv", filepath.Join(exporter.ExportPath, "schedules.csv")) @@ -368,8 +373,8 @@ func TestExporterFeedClient_ExportFeeds_should_handle_lots_of_rows_ok(t *testing Reply(200). BodyString(` { - "user_id": "user_123", - "organisation_id": "role_123", + "user_id": "user_bda3042f16a44249915ddc088adef92b", + "organisation_id": "role_ada3042f16a44249915ddc088adef92a", "firstname": "Test", "lastname": "Test" } diff --git a/pkg/api/mock_feeds_test.go b/pkg/api/mock_feeds_test.go index 582444e3..eef94188 100644 --- a/pkg/api/mock_feeds_test.go +++ b/pkg/api/mock_feeds_test.go @@ -164,6 +164,11 @@ func initMockFeedsSet1(httpClient *http.Client) { Get("/feed/issue_assignees"). Reply(200). File("mocks/set_1/feed_issue_assignees_1.json") + + gock.New("http://localhost:9999"). + Get("/accounts/history/v2/feed/activity_log_events"). + Reply(200). + File("mocks/set_1/feed_activity_log_events_1.json") } func initMockFeedsSet2(httpClient *http.Client) { @@ -279,6 +284,11 @@ func initMockFeedsSet2(httpClient *http.Client) { Get("/feed/issue_assignees"). Reply(200). File("mocks/set_1/feed_issue_assignees_1.json") + + gock.New("http://localhost:9999"). + Get("/accounts/history/v2/feed/activity_log_events"). + Reply(200). + File("mocks/set_1/feed_activity_log_events_1.json") } func initMockFeedsSet3(httpClient *http.Client) { @@ -388,6 +398,11 @@ func initMockFeedsSet3(httpClient *http.Client) { Get("/feed/issue_assignees"). Reply(200). File("mocks/set_1/feed_issue_assignees_1.json") + + gock.New("http://localhost:9999"). + Get("/accounts/history/v2/feed/activity_log_events"). + Reply(200). + File("mocks/set_1/feed_activity_log_events_1.json") } func initMockIssuesFeed(httpClient *http.Client) { diff --git a/pkg/api/mocks/set_1/feed_activity_log_events_1.json b/pkg/api/mocks/set_1/feed_activity_log_events_1.json new file mode 100644 index 00000000..17f7aef9 --- /dev/null +++ b/pkg/api/mocks/set_1/feed_activity_log_events_1.json @@ -0,0 +1,128 @@ +{ + "metadata": { + "next_page": null, + "next_page_token": null + }, + "data": [ + { + "id": "df502160-ff21-4314-becc-0d14583144ad", + "event_at": "2023-10-15T23:03:05.287Z", + "type": "auth.login", + "user_id": "4ef9fca4-beb3-46fb-a5d5-13e57fa89ef4", + "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8", + "client_class": "web", + "agent": "", + "metadata": "{}", + "remote_ip": "2a06:98c0:3600::103", + "initiator": "INITIATOR_USER" + }, + { + "id": "38f69f3a-69f6-4b24-8f68-1b12f3970e4a", + "event_at": "2023-10-15T23:18:50.651Z", + "type": "auth.login", + "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311", + "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8", + "client_class": "sc_service", + "agent": "Go-http-client/2.0", + "metadata": "{}", + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "bb62b60f-2c42-4970-aa19-113f7ea256ff", + "event_at": "2023-10-15T23:18:51.681Z", + "type": "auth.login", + "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311", + "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8", + "client_class": "service", + "agent": "Go-http-client/2.0", + "metadata": "{}", + "remote_ip": "10.253.134.238", + "initiator": "INITIATOR_USER" + }, + { + "id": "6d403b11-cb10-4d27-9bf7-1cfe72064383", + "event_at": "2023-10-15T23:18:56.124Z", + "type": "sensor.sensor_metadata_updated", + "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311", + "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8", + "client_class": "sc_service", + "agent": "grpc-go/1.58.2", + "metadata": "{\"source_id\":\"196451697411935\",\"source_name\":\"s12test\"}", + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "31ae09dd-6e40-4684-b450-5db5ac83126c", + "event_at": "2023-10-15T23:19:52.181Z", + "type": "sensor.gateway_deprovisioned", + "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311", + "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8", + "client_class": "sc_service", + "agent": "grpc-go/1.58.0", + "metadata": "{\"source_id\":\"153591697411991\",\"source_name\":\"s12test\"}", + "remote_ip": "10.254.11.167", + "initiator": "INITIATOR_USER" + }, + { + "id": "9f5238f7-ce83-416d-9e0f-fd529ea540a3", + "event_at": "2023-10-15T23:19:52.457Z", + "type": "sensor.sensor_metadata_updated", + "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311", + "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8", + "client_class": "sc_service", + "agent": "grpc-go/1.58.2", + "metadata": "{\"source_id\":\"148661697411992\",\"source_name\":\"s12test\"}", + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "78a9df0e-711a-4ad4-9e9d-dcb85f8aa9c1", + "event_at": "2023-10-15T23:19:52.565Z", + "type": "sensor.sensor_metadata_updated", + "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311", + "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8", + "client_class": "sc_service", + "agent": "grpc-go/1.58.2", + "metadata": "{\"source_id\":\"102591697411992\",\"source_name\":\"s12test\"}", + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "f9068f8f-a800-4882-98c4-ecedb87d6988", + "event_at": "2023-10-15T23:20:04.415Z", + "type": "auth.login", + "user_id": "148e564c-1f62-4445-b8b3-2aa57acd6e34", + "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8", + "client_class": "sc_service", + "agent": "Go-http-client/2.0", + "metadata": "{}", + "remote_ip": "", + "initiator": "INITIATOR_USER" + }, + { + "id": "efe70b32-3d5a-4021-a812-5c3e205ca3b2", + "event_at": "2023-10-15T23:20:05.452Z", + "type": "auth.login", + "user_id": "148e564c-1f62-4445-b8b3-2aa57acd6e34", + "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8", + "client_class": "service", + "agent": "Go-http-client/2.0", + "metadata": "{}", + "remote_ip": "10.254.8.44", + "initiator": "INITIATOR_USER" + }, + { + "id": "437ca49b-859a-410d-8e29-79344bcceb06", + "event_at": "2023-10-15T23:20:05.635Z", + "type": "sensor.sensor_metadata_updated", + "user_id": "148e564c-1f62-4445-b8b3-2aa57acd6e34", + "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8", + "client_class": "sc_service", + "agent": "grpc-go/1.58.2", + "metadata": "{\"source_id\":\"154161697412005\",\"source_name\":\"s12test\"}", + "remote_ip": "", + "initiator": "INITIATOR_USER" + } + ] +} diff --git a/pkg/api/mocks/set_1/feed_group_users_1.json b/pkg/api/mocks/set_1/feed_group_users_1.json index 360b4255..d820b0d2 100644 --- a/pkg/api/mocks/set_1/feed_group_users_1.json +++ b/pkg/api/mocks/set_1/feed_group_users_1.json @@ -7,22 +7,22 @@ { "user_id": "user_1", "group_id": "role_group1", - "organisation_id": "role_123" + "organisation_id": "role_ada3042f16a44249915ddc088adef92a" }, { "user_id": "user_1", "group_id": "role_group2", - "organisation_id": "role_123" + "organisation_id": "role_ada3042f16a44249915ddc088adef92a" }, { "user_id": "user_2", "group_id": "role_group1", - "organisation_id": "role_123" + "organisation_id": "role_ada3042f16a44249915ddc088adef92a" }, { "user_id": "user_2", "group_id": "role_group2", - "organisation_id": "role_123" + "organisation_id": "role_ada3042f16a44249915ddc088adef92a" } ] } diff --git a/pkg/api/mocks/set_1/outputs/account_histories.csv b/pkg/api/mocks/set_1/outputs/account_histories.csv new file mode 100644 index 00000000..2812faea --- /dev/null +++ b/pkg/api/mocks/set_1/outputs/account_histories.csv @@ -0,0 +1,11 @@ +event_id,event_at,type,user_id,organisation_id,client_class,agent,initiator,exported_at +31ae09dd-6e40-4684-b450-5db5ac83126c,--date--,sensor.gateway_deprovisioned,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,grpc-go/1.58.0,INITIATOR_USER,--date-- +38f69f3a-69f6-4b24-8f68-1b12f3970e4a,--date--,auth.login,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,Go-http-client/2.0,INITIATOR_USER,--date-- +437ca49b-859a-410d-8e29-79344bcceb06,--date--,sensor.sensor_metadata_updated,148e564c-1f62-4445-b8b3-2aa57acd6e34,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,grpc-go/1.58.2,INITIATOR_USER,--date-- +6d403b11-cb10-4d27-9bf7-1cfe72064383,--date--,sensor.sensor_metadata_updated,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,grpc-go/1.58.2,INITIATOR_USER,--date-- +78a9df0e-711a-4ad4-9e9d-dcb85f8aa9c1,--date--,sensor.sensor_metadata_updated,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,grpc-go/1.58.2,INITIATOR_USER,--date-- +9f5238f7-ce83-416d-9e0f-fd529ea540a3,--date--,sensor.sensor_metadata_updated,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,grpc-go/1.58.2,INITIATOR_USER,--date-- +bb62b60f-2c42-4970-aa19-113f7ea256ff,--date--,auth.login,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,service,Go-http-client/2.0,INITIATOR_USER,--date-- +df502160-ff21-4314-becc-0d14583144ad,--date--,auth.login,4ef9fca4-beb3-46fb-a5d5-13e57fa89ef4,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,web,,INITIATOR_USER,--date-- +efe70b32-3d5a-4021-a812-5c3e205ca3b2,--date--,auth.login,148e564c-1f62-4445-b8b3-2aa57acd6e34,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,service,Go-http-client/2.0,INITIATOR_USER,--date-- +f9068f8f-a800-4882-98c4-ecedb87d6988,--date--,auth.login,148e564c-1f62-4445-b8b3-2aa57acd6e34,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,Go-http-client/2.0,INITIATOR_USER,--date-- diff --git a/pkg/api/mocks/set_1/outputs/group_users.csv b/pkg/api/mocks/set_1/outputs/group_users.csv index fad519b6..567a7a5d 100644 --- a/pkg/api/mocks/set_1/outputs/group_users.csv +++ b/pkg/api/mocks/set_1/outputs/group_users.csv @@ -1,5 +1,5 @@ user_id,group_id,organisation_id,exported_at -user_1,role_group1,role_123,--date-- -user_2,role_group1,role_123,--date-- -user_1,role_group2,role_123,--date-- -user_2,role_group2,role_123,--date-- +user_1,role_group1,role_ada3042f16a44249915ddc088adef92a,--date-- +user_2,role_group1,role_ada3042f16a44249915ddc088adef92a,--date-- +user_1,role_group2,role_ada3042f16a44249915ddc088adef92a,--date-- +user_2,role_group2,role_ada3042f16a44249915ddc088adef92a,--date-- diff --git a/pkg/api/mocks/set_1/schemas/account_histories.csv b/pkg/api/mocks/set_1/schemas/account_histories.csv new file mode 100644 index 00000000..5381aa08 --- /dev/null +++ b/pkg/api/mocks/set_1/schemas/account_histories.csv @@ -0,0 +1 @@ +event_id,event_at,type,user_id,organisation_id,client_class,agent,initiator,exported_at diff --git a/pkg/internal/feed/drain_feed.go b/pkg/internal/feed/drain_feed.go index b7bb151d..7a16d806 100644 --- a/pkg/internal/feed/drain_feed.go +++ b/pkg/internal/feed/drain_feed.go @@ -78,4 +78,7 @@ type GetFeedParams struct { // Applicable only for course progress Offset int `url:"offset,omitempty"` CompletionStatus string `url:"completion_status,omitempty"` + + // Applicable only for account history + CreatedAfter time.Time `url:"created_after,omitempty"` } diff --git a/pkg/internal/feed/exporter_sql.go b/pkg/internal/feed/exporter_sql.go index 079ad05f..fe5cea91 100644 --- a/pkg/internal/feed/exporter_sql.go +++ b/pkg/internal/feed/exporter_sql.go @@ -191,21 +191,28 @@ func (e *SQLExporter) LastModifiedAt(feed Feed, modifiedAfter time.Time, orgID s } // LastRecord returns the latest stored record the feed -func (e *SQLExporter) LastRecord(feed Feed, modifiedAfter time.Time, orgID string, sortColumn string) time.Time { - type Ts struct { - TimeValue time.Time - } - latestRow := Ts{} +func (e *SQLExporter) LastRecord(feed Feed, fallbackTime time.Time, orgID string, sortColumn string) time.Time { + var latestRow = time.Time{} - result := e.DB. - Raw(fmt.Sprintf("SELECT %s as time_value FROM %s WHERE organisation_id = '%s' ORDER BY %s DESC LIMIT 1", sortColumn, feed.Name(), orgID, sortColumn)). - First(&latestRow) + result := e.DB.Table(feed.Name()). + Select(sortColumn). + Where("organisation_id = ?", orgID). + Order(clause.OrderByColumn{ + Column: clause.Column{ + Name: sortColumn, + Raw: false, + }, + Desc: true, + Reorder: false, + }). + Limit(1). + Scan(&latestRow) if result.RowsAffected != 0 { - return latestRow.TimeValue + return latestRow } - return modifiedAfter + return fallbackTime } // FinaliseExport closes out an export diff --git a/pkg/internal/feed/feed.go b/pkg/internal/feed/feed.go index 27bb9799..2007dab8 100644 --- a/pkg/internal/feed/feed.go +++ b/pkg/internal/feed/feed.go @@ -41,7 +41,7 @@ type Exporter interface { FinaliseExport(feed Feed, rows interface{}) error LastModifiedAt(feed Feed, modifiedAfter time.Time, orgID string) (time.Time, error) - LastRecord(feed Feed, modifiedAfter time.Time, orgID string, sortColumn string) time.Time + LastRecord(feed Feed, fallbackTime time.Time, orgID string, sortColumn string) time.Time WriteMedia(auditID string, mediaID string, contentType string, body []byte) error DeleteRowsIfExist(feed Feed, query string, args ...interface{}) error GetDuration() time.Duration diff --git a/pkg/internal/feed/feed_account_history.go b/pkg/internal/feed/feed_account_history.go new file mode 100644 index 00000000..a946fe89 --- /dev/null +++ b/pkg/internal/feed/feed_account_history.go @@ -0,0 +1,156 @@ +package feed + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/SafetyCulture/safetyculture-exporter/pkg/internal/util" + "github.com/SafetyCulture/safetyculture-exporter/pkg/logger" + + "github.com/SafetyCulture/safetyculture-exporter/pkg/httpapi" + "github.com/SafetyCulture/safetyculture-exporter/pkg/internal/events" +) + +const feedPath = "/accounts/history/v2/feed/activity_log_events" +const accountHistorySortingColumn = "event_at" + +// AccountHistory represents a row from the account history feed +type AccountHistory struct { + ID string `json:"id" csv:"event_id" gorm:"primarykey;column:event_id;size:41"` + EventAt time.Time `json:"event_at" csv:"event_at" gorm:"autoUpdateTime"` + Type string `json:"type" csv:"type"` + UserID string `json:"user_id" csv:"user_id" gorm:"size:37"` + OrganisationID string `json:"organisation_id" csv:"organisation_id" gorm:"size:37"` + ClientClass string `json:"client_class" csv:"client_class" gorm:"client_class"` + Agent string `json:"agent" csv:"agent" gorm:"agent"` + Initiator string `json:"initiator" csv:"initiator" gorm:"size:100"` + ExportedAt time.Time `json:"exported_at" csv:"exported_at" gorm:"autoUpdateTime"` +} + +// AccountHistoryFeed is a representation of the account history feed +type AccountHistoryFeed struct { + ExportedAt time.Time + Incremental bool + Limit int +} + +// Name is the name of the feed +func (f *AccountHistoryFeed) Name() string { + return "account_histories" +} + +// HasRemainingInformation returns true if the feed returns remaining items information +func (f *AccountHistoryFeed) HasRemainingInformation() bool { + return false +} + +// Model returns the model of the feed row +func (f *AccountHistoryFeed) Model() interface{} { + return AccountHistory{} +} + +// RowsModel returns the model of feed rows +func (f *AccountHistoryFeed) RowsModel() interface{} { + return &[]*AccountHistory{} +} + +// PrimaryKey returns the primary key(s) +func (f *AccountHistoryFeed) PrimaryKey() []string { + return []string{"event_id"} +} + +// Columns returns the columns of the row +func (f *AccountHistoryFeed) Columns() []string { + return []string{ + "event_at", + "type", + "user_id", + "organisation_id", + "client_class", + "agent", + "initiator", + "exported_at", + } +} + +// Order returns the ordering when retrieving an export +func (f *AccountHistoryFeed) Order() string { + return "event_id" +} + +// CreateSchema creates the schema of the feed for the supplied exporter +func (f *AccountHistoryFeed) CreateSchema(exporter Exporter) error { + return exporter.CreateSchema(f, &[]*AccountHistory{}) +} + +// Export exports the feed to the supplied exporter +func (f *AccountHistoryFeed) Export(ctx context.Context, apiClient *httpapi.Client, exporter Exporter, orgID string) error { + l := logger.GetLogger().With("feed", f.Name(), "org_id", orgID) + s12OrgID := util.ConvertS12ToUUID(orgID) + if s12OrgID.IsNil() { + return fmt.Errorf("cannot convert given %q organisation ID to UUID", orgID) + } + + status := GetExporterStatus() + + if err := exporter.InitFeed(f, &InitFeedOptions{ + // Truncate files if upserts aren't supported. + // This ensures that the export does not contain duplicate rows + Truncate: !f.Incremental, + }); err != nil { + return events.WrapEventError(err, "init feed") + } + + if f.Incremental { + f.ExportedAt = exporter.LastRecord(f, f.ExportedAt, s12OrgID.String(), accountHistorySortingColumn) + l.Info("resuming account history feed from ", f.ExportedAt.String()) + } + + drainFn := func(resp *GetFeedResponse) error { + var rows []*AccountHistory + + if err := json.Unmarshal(resp.Data, &rows); err != nil { + return events.NewEventErrorWithMessage(err, events.ErrorSeverityError, events.ErrorSubSystemDataIntegrity, false, "map data") + } + + numRows := len(rows) + if numRows != 0 { + // Calculate the size of the batch we can insert into the DB at once. Column count + buffer to account for primary keys + batchSize := exporter.ParameterLimit() / (len(f.Columns()) + 4) + err := util.SplitSliceInBatch(batchSize, rows, func(batch []*AccountHistory) error { + if err := exporter.WriteRows(f, batch); err != nil { + return events.WrapEventError(err, "write rows") + } + return nil + }) + + if err != nil { + return err + } + } + + status.IncrementStatus(f.Name(), int64(numRows), apiClient.Duration.Milliseconds()) + + l.With( + "downloaded", status.ReadCounter(f.Name()), + "duration_ms", apiClient.Duration.Milliseconds(), + "export_duration_ms", exporter.GetDuration().Milliseconds(), + ).Info("export batch complete") + return nil + } + + req := &GetFeedRequest{ + InitialURL: feedPath, + Params: GetFeedParams{ + Limit: f.Limit, + CreatedAfter: f.ExportedAt, + }, + } + + if err := DrainFeed(ctx, apiClient, req, drainFn); err != nil { + return events.WrapEventError(err, fmt.Sprintf("feed %q", f.Name())) + } + return exporter.FinaliseExport(f, &[]*AccountHistory{}) +} diff --git a/pkg/internal/feed/feed_exporter.go b/pkg/internal/feed/feed_exporter.go index 5ce8379a..19cb17bc 100644 --- a/pkg/internal/feed/feed_exporter.go +++ b/pkg/internal/feed/feed_exporter.go @@ -315,6 +315,10 @@ func (e *ExporterFeedClient) GetFeeds() []Feed { Incremental: false, // IssueAssignee doesn't support modified after filters Limit: e.configuration.ExportIssueLimit, }, + &AccountHistoryFeed{ + Incremental: true, + Limit: 250, + }, } } diff --git a/pkg/internal/feed/fixtures/schemas/formatted/account_histories.txt b/pkg/internal/feed/fixtures/schemas/formatted/account_histories.txt new file mode 100644 index 00000000..351e07a4 --- /dev/null +++ b/pkg/internal/feed/fixtures/schemas/formatted/account_histories.txt @@ -0,0 +1,13 @@ ++-----------------+----------+-------------+ +| NAME | TYPE | PRIMARY KEY | ++-----------------+----------+-------------+ +| event_id | TEXT | true | +| event_at | datetime | | +| type | TEXT | | +| user_id | TEXT | | +| organisation_id | TEXT | | +| client_class | TEXT | | +| agent | TEXT | | +| initiator | TEXT | | +| exported_at | datetime | | ++-----------------+----------+-------------+ diff --git a/pkg/internal/feed/mocks/exporter_mock.go b/pkg/internal/feed/mocks/exporter_mock.go index 29847d32..4bf9ce62 100644 --- a/pkg/internal/feed/mocks/exporter_mock.go +++ b/pkg/internal/feed/mocks/exporter_mock.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.39.1. DO NOT EDIT. +// Code generated by mockery v2.45.1. DO NOT EDIT. package mocks @@ -135,9 +135,9 @@ func (_m *Exporter) LastModifiedAt(_a0 feed.Feed, modifiedAfter time.Time, orgID return r0, r1 } -// LastRecord provides a mock function with given fields: _a0, modifiedAfter, orgID, sortColumn -func (_m *Exporter) LastRecord(_a0 feed.Feed, modifiedAfter time.Time, orgID string, sortColumn string) time.Time { - ret := _m.Called(_a0, modifiedAfter, orgID, sortColumn) +// LastRecord provides a mock function with given fields: _a0, fallbackTime, orgID, sortColumn +func (_m *Exporter) LastRecord(_a0 feed.Feed, fallbackTime time.Time, orgID string, sortColumn string) time.Time { + ret := _m.Called(_a0, fallbackTime, orgID, sortColumn) if len(ret) == 0 { panic("no return value specified for LastRecord") @@ -145,7 +145,7 @@ func (_m *Exporter) LastRecord(_a0 feed.Feed, modifiedAfter time.Time, orgID str var r0 time.Time if rf, ok := ret.Get(0).(func(feed.Feed, time.Time, string, string) time.Time); ok { - r0 = rf(_a0, modifiedAfter, orgID, sortColumn) + r0 = rf(_a0, fallbackTime, orgID, sortColumn) } else { r0 = ret.Get(0).(time.Time) } diff --git a/pkg/internal/util/uuid.go b/pkg/internal/util/uuid.go new file mode 100644 index 00000000..a18952ad --- /dev/null +++ b/pkg/internal/util/uuid.go @@ -0,0 +1,20 @@ +package util + +import ( + "github.com/gofrs/uuid" + "strings" +) + +// ConvertS12ToUUID - attempt to convert a s12id to UUID +func ConvertS12ToUUID(s string) uuid.UUID { + maybeUUID, err := uuid.FromString(s) + if err == nil { + return maybeUUID + } + + idx := strings.LastIndex(s, "_") + if idx == -1 { + return uuid.Nil + } + return uuid.FromStringOrNil(s[idx+1:]) +} diff --git a/pkg/internal/util/uuid_test.go b/pkg/internal/util/uuid_test.go new file mode 100644 index 00000000..826d577d --- /dev/null +++ b/pkg/internal/util/uuid_test.go @@ -0,0 +1,44 @@ +package util_test + +import ( + "github.com/SafetyCulture/safetyculture-exporter/pkg/internal/util" + "github.com/gofrs/uuid" + "github.com/stretchr/testify/require" + "testing" +) + +func TestConvertS12ToUUID(t *testing.T) { + expectedUUID, err := uuid.FromString("ada3042f-16a4-4249-915d-dc088adef92a") + require.NoError(t, err) + + tests := []struct { + name string + stringID string + expected uuid.UUID + }{ + { + name: "should pass as S12", + stringID: "role_ada3042f16a44249915ddc088adef92a", + expected: expectedUUID, + }, + { + name: "should pass as UUID", + stringID: "ada3042f-16a4-4249-915d-dc088adef92a", + expected: expectedUUID, + }, + { + name: "should return UUID ZERO in case of error", + stringID: "xyz", + expected: uuid.Nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res := util.ConvertS12ToUUID(tt.stringID) + if res != tt.expected { + t.Errorf("got %s, want %s", res, tt.expected) + } + }) + } +}