DEV-3508 feed account history (#478)

* DEVP-3508 WIP activity_log_events * DEVP-3508 WIP activity_log_events * DEVP-3508 activity_log_events * DEVP-3508 activity_log_events * DEVP-3508 cleanup * DEVP-3508 tests * DEVP-3508 tests * DEVP-3508 small fixes * DEVP-3508 small fixes * DEVP-3508 small fixes * DEVP-3508 small fixes * DEVP-3508 small fixes * DEVP-3508 small fixes * DEV-3508 fix query for mssql
SafetyCulture · Nov 4, 2024 · 4cfe95c · 4cfe95c
1 parent 130f205
commit 4cfe95c
Show file tree

Hide file tree

Showing 19 changed files with 451 additions and 44 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,3 +1,3 @@
 # https://help.github.com/articles/about-codeowners/
 
-* @SafetyCulture/api-and-integrations
+* @SafetyCulture/developer-platform
diff --git a/pkg/api/configuration_manager.go b/pkg/api/configuration_manager.go
@@ -210,7 +210,7 @@ func (c *ConfigurationManager) ApplySafetyGuards() {
 		c.Configuration.Export.Inspection.SkipIds = defaultCfg.Export.Inspection.SkipIds
 	}
 
-	if c.Configuration.Report.Format == nil || len(c.Configuration.Report.Format) == 0 {
+	if len(c.Configuration.Report.Format) == 0 {
 		c.Configuration.Report.Format = defaultCfg.Report.Format
 	}
 

diff --git a/pkg/api/export_feeds_intg_test.go b/pkg/api/export_feeds_intg_test.go
@@ -35,7 +35,7 @@ func TestIntegrationDbCreateSchema_should_create_all_schemas(t *testing.T) {
 		BodyString(`
 		{
 			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }
@@ -84,7 +84,7 @@ func TestIntegrationDbExportFeeds_should_export_all_feeds_to_file(t *testing.T)
 		BodyString(`
 		{
 			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }
@@ -135,7 +135,7 @@ func TestIntegrationDbExportFeeds_should_perform_incremental_update_on_second_ru
 		BodyString(`
 		{
 			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }
@@ -202,7 +202,7 @@ func TestIntegrationDbExportFeeds_should_handle_lots_of_rows_ok(t *testing.T) {
 		BodyString(`
 		{
 			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }
@@ -239,7 +239,7 @@ func TestIntegrationDbExportFeeds_should_update_action_assignees_on_second_run(t
 		BodyString(`
 		{
 			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }
@@ -295,7 +295,7 @@ func TestGroupUserFeed_Export_should_filter_duplicates(t *testing.T) {
 		BodyString(`
 		{
 			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }

diff --git a/pkg/api/export_feeds_test.go b/pkg/api/export_feeds_test.go
@@ -44,6 +44,9 @@ func TestExporterFeedClient_ExportFeeds_should_create_all_schemas_to_file(t *tes
 
 	filesEqualish(t, "mocks/set_1/schemas/training_course_progresses.csv", filepath.Join(exporter.ExportPath, "training_course_progresses.csv"))
 	filesEqualish(t, "mocks/set_1/schemas/issue_assignees.csv", filepath.Join(exporter.ExportPath, "issue_assignees.csv"))
+
+	filesEqualish(t, "mocks/set_1/schemas/issue_assignees.csv", filepath.Join(exporter.ExportPath, "issue_assignees.csv"))
+	filesEqualish(t, "mocks/set_1/schemas/account_histories.csv", filepath.Join(exporter.ExportPath, "account_histories.csv"))
 }
 
 func TestExporterFeedClient_ExportFeeds_should_export_all_feeds_to_file(t *testing.T) {
@@ -60,8 +63,8 @@ func TestExporterFeedClient_ExportFeeds_should_export_all_feeds_to_file(t *testi
 		Reply(200).
 		BodyString(`
 		{
-			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"user_id": "user_bda3042f16a44249915ddc088adef92b",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }
@@ -124,6 +127,7 @@ func TestExporterFeedClient_ExportFeeds_should_export_all_feeds_to_file(t *testi
 	filesEqualish(t, "mocks/set_1/outputs/sheqsy_departments.csv", filepath.Join(exporter.ExportPath, "sheqsy_departments.csv"))
 
 	filesEqualish(t, "mocks/set_1/outputs/training_course_progresses.csv", filepath.Join(exporter.ExportPath, "training_course_progresses.csv"))
+	filesEqualish(t, "mocks/set_1/outputs/account_histories.csv", filepath.Join(exporter.ExportPath, "account_histories.csv"))
 }
 
 func TestExporterFeedClient_ExportFeeds_should_err_when_not_auth(t *testing.T) {
@@ -165,8 +169,8 @@ func TestExporterFeedClient_ExportFeeds_should_err_when_InitFeed_errors(t *testi
 		Reply(200).
 		BodyString(`
 		{
-			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"user_id": "user_bda3042f16a44249915ddc088adef92b",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }
@@ -205,8 +209,8 @@ func TestExporterFeedClient_ExportFeeds_should_err_when_cannot_unmarshal(t *test
 		Reply(200).
 		BodyString(`
 		{
-			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"user_id": "user_bda3042f16a44249915ddc088adef92b",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }
@@ -253,8 +257,8 @@ func TestExporterFeedClient_ExportFeeds_should_err_when_cannot_write_rows(t *tes
 		Reply(200).
 		BodyString(`
 		{
-			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"user_id": "user_bda3042f16a44249915ddc088adef92b",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }
@@ -290,8 +294,8 @@ func TestExporterFeedClient_ExportFeeds_should_perform_incremental_update_on_sec
 		Reply(200).
 		BodyString(`
 		{
-			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"user_id": "user_bda3042f16a44249915ddc088adef92b",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }
@@ -338,6 +342,7 @@ func TestExporterFeedClient_ExportFeeds_should_perform_incremental_update_on_sec
 
 	filesEqualish(t, "mocks/set_2/outputs/users.csv", filepath.Join(exporter.ExportPath, "users.csv"))
 	filesEqualish(t, "mocks/set_2/outputs/groups.csv", filepath.Join(exporter.ExportPath, "groups.csv"))
+
 	filesEqualish(t, "mocks/set_2/outputs/group_users.csv", filepath.Join(exporter.ExportPath, "group_users.csv"))
 
 	filesEqualish(t, "mocks/set_2/outputs/schedules.csv", filepath.Join(exporter.ExportPath, "schedules.csv"))
@@ -368,8 +373,8 @@ func TestExporterFeedClient_ExportFeeds_should_handle_lots_of_rows_ok(t *testing
 		Reply(200).
 		BodyString(`
 		{
-			"user_id": "user_123",
-			"organisation_id": "role_123",
+			"user_id": "user_bda3042f16a44249915ddc088adef92b",
+			"organisation_id": "role_ada3042f16a44249915ddc088adef92a",
 			"firstname": "Test",
 			"lastname": "Test"
 		  }

diff --git a/pkg/api/mock_feeds_test.go b/pkg/api/mock_feeds_test.go
@@ -164,6 +164,11 @@ func initMockFeedsSet1(httpClient *http.Client) {
 		Get("/feed/issue_assignees").
 		Reply(200).
 		File("mocks/set_1/feed_issue_assignees_1.json")
+
+	gock.New("http://localhost:9999").
+		Get("/accounts/history/v2/feed/activity_log_events").
+		Reply(200).
+		File("mocks/set_1/feed_activity_log_events_1.json")
 }
 
 func initMockFeedsSet2(httpClient *http.Client) {
@@ -279,6 +284,11 @@ func initMockFeedsSet2(httpClient *http.Client) {
 		Get("/feed/issue_assignees").
 		Reply(200).
 		File("mocks/set_1/feed_issue_assignees_1.json")
+
+	gock.New("http://localhost:9999").
+		Get("/accounts/history/v2/feed/activity_log_events").
+		Reply(200).
+		File("mocks/set_1/feed_activity_log_events_1.json")
 }
 
 func initMockFeedsSet3(httpClient *http.Client) {
@@ -388,6 +398,11 @@ func initMockFeedsSet3(httpClient *http.Client) {
 		Get("/feed/issue_assignees").
 		Reply(200).
 		File("mocks/set_1/feed_issue_assignees_1.json")
+
+	gock.New("http://localhost:9999").
+		Get("/accounts/history/v2/feed/activity_log_events").
+		Reply(200).
+		File("mocks/set_1/feed_activity_log_events_1.json")
 }
 
 func initMockIssuesFeed(httpClient *http.Client) {

diff --git a/pkg/api/mocks/set_1/feed_activity_log_events_1.json b/pkg/api/mocks/set_1/feed_activity_log_events_1.json
@@ -0,0 +1,128 @@
+{
+  "metadata": {
+    "next_page": null,
+    "next_page_token": null
+  },
+  "data": [
+    {
+      "id": "df502160-ff21-4314-becc-0d14583144ad",
+      "event_at": "2023-10-15T23:03:05.287Z",
+      "type": "auth.login",
+      "user_id": "4ef9fca4-beb3-46fb-a5d5-13e57fa89ef4",
+      "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8",
+      "client_class": "web",
+      "agent": "",
+      "metadata": "{}",
+      "remote_ip": "2a06:98c0:3600::103",
+      "initiator": "INITIATOR_USER"
+    },
+    {
+      "id": "38f69f3a-69f6-4b24-8f68-1b12f3970e4a",
+      "event_at": "2023-10-15T23:18:50.651Z",
+      "type": "auth.login",
+      "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311",
+      "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8",
+      "client_class": "sc_service",
+      "agent": "Go-http-client/2.0",
+      "metadata": "{}",
+      "remote_ip": "",
+      "initiator": "INITIATOR_USER"
+    },
+    {
+      "id": "bb62b60f-2c42-4970-aa19-113f7ea256ff",
+      "event_at": "2023-10-15T23:18:51.681Z",
+      "type": "auth.login",
+      "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311",
+      "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8",
+      "client_class": "service",
+      "agent": "Go-http-client/2.0",
+      "metadata": "{}",
+      "remote_ip": "10.253.134.238",
+      "initiator": "INITIATOR_USER"
+    },
+    {
+      "id": "6d403b11-cb10-4d27-9bf7-1cfe72064383",
+      "event_at": "2023-10-15T23:18:56.124Z",
+      "type": "sensor.sensor_metadata_updated",
+      "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311",
+      "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8",
+      "client_class": "sc_service",
+      "agent": "grpc-go/1.58.2",
+      "metadata": "{\"source_id\":\"196451697411935\",\"source_name\":\"s12test\"}",
+      "remote_ip": "",
+      "initiator": "INITIATOR_USER"
+    },
+    {
+      "id": "31ae09dd-6e40-4684-b450-5db5ac83126c",
+      "event_at": "2023-10-15T23:19:52.181Z",
+      "type": "sensor.gateway_deprovisioned",
+      "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311",
+      "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8",
+      "client_class": "sc_service",
+      "agent": "grpc-go/1.58.0",
+      "metadata": "{\"source_id\":\"153591697411991\",\"source_name\":\"s12test\"}",
+      "remote_ip": "10.254.11.167",
+      "initiator": "INITIATOR_USER"
+    },
+    {
+      "id": "9f5238f7-ce83-416d-9e0f-fd529ea540a3",
+      "event_at": "2023-10-15T23:19:52.457Z",
+      "type": "sensor.sensor_metadata_updated",
+      "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311",
+      "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8",
+      "client_class": "sc_service",
+      "agent": "grpc-go/1.58.2",
+      "metadata": "{\"source_id\":\"148661697411992\",\"source_name\":\"s12test\"}",
+      "remote_ip": "",
+      "initiator": "INITIATOR_USER"
+    },
+    {
+      "id": "78a9df0e-711a-4ad4-9e9d-dcb85f8aa9c1",
+      "event_at": "2023-10-15T23:19:52.565Z",
+      "type": "sensor.sensor_metadata_updated",
+      "user_id": "c30d8eb8-317f-43b4-bc87-b63e9a5f7311",
+      "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8",
+      "client_class": "sc_service",
+      "agent": "grpc-go/1.58.2",
+      "metadata": "{\"source_id\":\"102591697411992\",\"source_name\":\"s12test\"}",
+      "remote_ip": "",
+      "initiator": "INITIATOR_USER"
+    },
+    {
+      "id": "f9068f8f-a800-4882-98c4-ecedb87d6988",
+      "event_at": "2023-10-15T23:20:04.415Z",
+      "type": "auth.login",
+      "user_id": "148e564c-1f62-4445-b8b3-2aa57acd6e34",
+      "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8",
+      "client_class": "sc_service",
+      "agent": "Go-http-client/2.0",
+      "metadata": "{}",
+      "remote_ip": "",
+      "initiator": "INITIATOR_USER"
+    },
+    {
+      "id": "efe70b32-3d5a-4021-a812-5c3e205ca3b2",
+      "event_at": "2023-10-15T23:20:05.452Z",
+      "type": "auth.login",
+      "user_id": "148e564c-1f62-4445-b8b3-2aa57acd6e34",
+      "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8",
+      "client_class": "service",
+      "agent": "Go-http-client/2.0",
+      "metadata": "{}",
+      "remote_ip": "10.254.8.44",
+      "initiator": "INITIATOR_USER"
+    },
+    {
+      "id": "437ca49b-859a-410d-8e29-79344bcceb06",
+      "event_at": "2023-10-15T23:20:05.635Z",
+      "type": "sensor.sensor_metadata_updated",
+      "user_id": "148e564c-1f62-4445-b8b3-2aa57acd6e34",
+      "organisation_id": "a08b6ac0-8a05-11e2-9951-ddd1182f65d8",
+      "client_class": "sc_service",
+      "agent": "grpc-go/1.58.2",
+      "metadata": "{\"source_id\":\"154161697412005\",\"source_name\":\"s12test\"}",
+      "remote_ip": "",
+      "initiator": "INITIATOR_USER"
+    }
+  ]
+}
diff --git a/pkg/api/mocks/set_1/feed_group_users_1.json b/pkg/api/mocks/set_1/feed_group_users_1.json
@@ -7,22 +7,22 @@
     {
       "user_id": "user_1",
       "group_id": "role_group1",
-      "organisation_id": "role_123"
+      "organisation_id": "role_ada3042f16a44249915ddc088adef92a"
     },
     {
       "user_id": "user_1",
       "group_id": "role_group2",
-      "organisation_id": "role_123"
+      "organisation_id": "role_ada3042f16a44249915ddc088adef92a"
     },
     {
       "user_id": "user_2",
       "group_id": "role_group1",
-      "organisation_id": "role_123"
+      "organisation_id": "role_ada3042f16a44249915ddc088adef92a"
     },
     {
       "user_id": "user_2",
       "group_id": "role_group2",
-      "organisation_id": "role_123"
+      "organisation_id": "role_ada3042f16a44249915ddc088adef92a"
     }
   ]
 }
diff --git a/pkg/api/mocks/set_1/outputs/account_histories.csv b/pkg/api/mocks/set_1/outputs/account_histories.csv
@@ -0,0 +1,11 @@
+event_id,event_at,type,user_id,organisation_id,client_class,agent,initiator,exported_at
+31ae09dd-6e40-4684-b450-5db5ac83126c,--date--,sensor.gateway_deprovisioned,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,grpc-go/1.58.0,INITIATOR_USER,--date--
+38f69f3a-69f6-4b24-8f68-1b12f3970e4a,--date--,auth.login,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,Go-http-client/2.0,INITIATOR_USER,--date--
+437ca49b-859a-410d-8e29-79344bcceb06,--date--,sensor.sensor_metadata_updated,148e564c-1f62-4445-b8b3-2aa57acd6e34,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,grpc-go/1.58.2,INITIATOR_USER,--date--
+6d403b11-cb10-4d27-9bf7-1cfe72064383,--date--,sensor.sensor_metadata_updated,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,grpc-go/1.58.2,INITIATOR_USER,--date--
+78a9df0e-711a-4ad4-9e9d-dcb85f8aa9c1,--date--,sensor.sensor_metadata_updated,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,grpc-go/1.58.2,INITIATOR_USER,--date--
+9f5238f7-ce83-416d-9e0f-fd529ea540a3,--date--,sensor.sensor_metadata_updated,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,grpc-go/1.58.2,INITIATOR_USER,--date--
+bb62b60f-2c42-4970-aa19-113f7ea256ff,--date--,auth.login,c30d8eb8-317f-43b4-bc87-b63e9a5f7311,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,service,Go-http-client/2.0,INITIATOR_USER,--date--
+df502160-ff21-4314-becc-0d14583144ad,--date--,auth.login,4ef9fca4-beb3-46fb-a5d5-13e57fa89ef4,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,web,,INITIATOR_USER,--date--
+efe70b32-3d5a-4021-a812-5c3e205ca3b2,--date--,auth.login,148e564c-1f62-4445-b8b3-2aa57acd6e34,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,service,Go-http-client/2.0,INITIATOR_USER,--date--
+f9068f8f-a800-4882-98c4-ecedb87d6988,--date--,auth.login,148e564c-1f62-4445-b8b3-2aa57acd6e34,a08b6ac0-8a05-11e2-9951-ddd1182f65d8,sc_service,Go-http-client/2.0,INITIATOR_USER,--date--
diff --git a/pkg/api/mocks/set_1/outputs/group_users.csv b/pkg/api/mocks/set_1/outputs/group_users.csv
@@ -1,5 +1,5 @@
 user_id,group_id,organisation_id,exported_at
-user_1,role_group1,role_123,--date--
-user_2,role_group1,role_123,--date--
-user_1,role_group2,role_123,--date--
-user_2,role_group2,role_123,--date--
+user_1,role_group1,role_ada3042f16a44249915ddc088adef92a,--date--
+user_2,role_group1,role_ada3042f16a44249915ddc088adef92a,--date--
+user_1,role_group2,role_ada3042f16a44249915ddc088adef92a,--date--
+user_2,role_group2,role_ada3042f16a44249915ddc088adef92a,--date--
diff --git a/pkg/api/mocks/set_1/schemas/account_histories.csv b/pkg/api/mocks/set_1/schemas/account_histories.csv
@@ -0,0 +1 @@
+event_id,event_at,type,user_id,organisation_id,client_class,agent,initiator,exported_at
diff --git a/pkg/internal/feed/drain_feed.go b/pkg/internal/feed/drain_feed.go
@@ -78,4 +78,7 @@ type GetFeedParams struct {
 	// Applicable only for course progress
 	Offset           int    `url:"offset,omitempty"`
 	CompletionStatus string `url:"completion_status,omitempty"`
+
+	// Applicable only for account history
+	CreatedAfter time.Time `url:"created_after,omitempty"`
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		event_id,event_at,type,user_id,organisation_id,client_class,agent,initiator,exported_at