feat: meltano (#1269)

* chore: initial meltano commit

* chore: add kartik to tf init

* chore(reporting): use user dataset in meltano dev environment (#1271)

* chore(reporting): dbt configuration for meltano and dummy transformation (#1272)

* chore(reporting): dbt configuration for meltano and dummy transformation

Run `meltano invoke debt-bigquery:run` to execute.

* chore(reporting): dummy dbt transformation

* chore: create dbt dataset in US

* chore: fix make reset-deps

---------

Co-authored-by: José <jir@galoy.io>

.envrc

@@ -8,3 +8,12 @@ export DATAFORM_SCHEMA_SUFFIX=${TF_VAR_name_prefix}
 export DATAFORM_VARS="executionEnv=lana-dev,devUser=${TF_VAR_name_prefix}"
 export SA_CREDS_BASE64="${TF_VAR_sa_creds}"
 export DEV_ENV_NAME_PREFIX="${TF_VAR_name_prefix}"
+
+export TARGET_BIGQUERY_CREDENTIALS_JSON="$(echo $TF_VAR_sa_creds | base64 -d)"
+export TARGET_BIGQUERY_DATASET="${USER}_dataset"
+export TARGET_BIGQUERY_LOCATION="US"
+export DBT_BIGQUERY_DATASET="dataform_${USER}"
+export DBT_BIGQUERY_KEYFILE="/lana/keyfile.json"
+echo $TARGET_BIGQUERY_CREDENTIALS_JSON > meltano/keyfile.json
+
+alias meltano='docker compose run meltano --'

.gitignore

@@ -23,3 +23,4 @@ node_modules/
 .bacon-locations
 
 *.tgz
+meltano/keyfile.json

Makefile

@@ -11,7 +11,7 @@ clean-deps:
 	docker compose down -t 1
 
 start-deps:
-	docker compose up --wait -d
+	docker compose up --wait -d integration-deps
 
 setup-db:
 	cd lana/app && cargo sqlx migrate run

docker-compose.yml

@@ -1,4 +1,16 @@
 services:
+  integration-deps:
+    image: busybox
+    command: sh -c "while :; do sleep 10; done"
+    depends_on:
+      - core-pg
+      - cala-pg
+      - cala
+      - otel-agent
+      - kratos
+      - kratos-pg
+      - oathkeeper
+      - mailhog
   core-pg:
     image: postgres:16.4
     ports:
@@ -100,3 +112,14 @@ services:
     ports:
       - "1025:1025"
       - "8025:8025"
+  meltano:
+    image: meltano/meltano:latest
+    volumes:
+      - ./meltano:/lana
+    working_dir: /lana
+    environment:
+      - TARGET_BIGQUERY_CREDENTIALS_JSON=${TARGET_BIGQUERY_CREDENTIALS_JSON}
+      - TARGET_BIGQUERY_DATASET=${TARGET_BIGQUERY_DATASET}
+      - TARGET_BIGQUERY_LOCATION=${TARGET_BIGQUERY_LOCATION}
+      - DBT_BIGQUERY_DATASET=${DBT_BIGQUERY_DATASET}
+      - DBT_BIGQUERY_KEYFILE=${DBT_BIGQUERY_KEYFILE}

flake.nix

@@ -1,5 +1,5 @@
 {
-  description = "Cala";
+  description = "Lana";
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";

meltano/.gitignore

@@ -0,0 +1,3 @@
+/venv
+/.meltano
+.env

meltano/meltano.yml

@@ -0,0 +1,36 @@
+version: 1
+default_environment: dev
+project_id: 608399cc-8e94-418f-a5ff-bdad5cb5a190
+environments:
+- name: dev
+  config:
+    plugins:
+      extractors:
+      - name: tap-postgres
+        config:
+          host: core-pg
+          user: user
+          database: pg
+          password: password
+- name: staging
+- name: prod
+plugins:
+  extractors:
+  - name: tap-postgres
+    variant: meltanolabs
+    pip_url: git+https://github.com/MeltanoLabs/tap-postgres.git
+    select:
+    - public-*.*
+  loaders:
+  - name: target-bigquery
+    variant: z3z1ma
+    pip_url: git+https://github.com/z3z1ma/target-bigquery.git
+    config:
+      project: lana-dev-440721
+  transformers:
+  - name: dbt-bigquery
+    variant: dbt-labs
+    pip_url: dbt-core~=1.3.0 dbt-bigquery~=1.3.0
+    config:
+      auth_method: service-account
+      project: lana-dev-440721

meltano/output/public-_sqlx_migrations.jsonl

@@ -0,0 +1 @@
+{"version": 20240517074612, "description": "core setup", "installed_on": "2025-01-13T12:23:11.004449+00:00", "success": true, "checksum": "1b54c58621470224521edd84214611c761c988188bbf874bd58c9a1fbc256af9f3f18f7144cfae2ae2821e36cc7b0b57", "execution_time": 65481792}

meltano/output/public-approval_process_events.jsonl

@@ -0,0 +1,10 @@
+{"id": "6e27908a-85a8-4582-bf48-28f0c6289a82", "sequence": 1, "event_type": "initialized", "event": {"id": "6e27908a-85a8-4582-bf48-28f0c6289a82", "type": "initialized", "rules": {"type": "system_auto_approve"}, "policy_id": "c23052ba-ab5c-4d20-82f3-8a762d2e7c4c", "audit_info": {"sub": "system:00000000-0000-0000-0000-000000000000", "audit_entry_id": 14335}, "target_ref": "6e27908a-85a8-4582-bf48-28f0c6289a82", "process_type": "credit-facility"}, "recorded_at": "2024-09-02T20:33:20+00:00"}
+{"id": "6e27908a-85a8-4582-bf48-28f0c6289a82", "sequence": 2, "event_type": "concluded", "event": {"type": "concluded", "approved": true, "audit_info": {"sub": "system:00000000-0000-0000-0000-000000000000", "audit_entry_id": 14337}}, "recorded_at": "2024-09-02T20:33:20+00:00"}
+{"id": "0625d6f8-e583-4a1a-b0b4-71d3c2e435d6", "sequence": 1, "event_type": "initialized", "event": {"id": "0625d6f8-e583-4a1a-b0b4-71d3c2e435d6", "type": "initialized", "rules": {"type": "system_auto_approve"}, "policy_id": "367d37aa-de0b-44b2-911d-5d5785567aa2", "audit_info": {"sub": "system:00000000-0000-0000-0000-000000000000", "audit_entry_id": 14378}, "target_ref": "0625d6f8-e583-4a1a-b0b4-71d3c2e435d6", "process_type": "disbursal"}, "recorded_at": "2024-09-19T13:56:40+00:00"}
+{"id": "0625d6f8-e583-4a1a-b0b4-71d3c2e435d6", "sequence": 2, "event_type": "concluded", "event": {"type": "concluded", "approved": true, "audit_info": {"sub": "system:00000000-0000-0000-0000-000000000000", "audit_entry_id": 14379}}, "recorded_at": "2024-09-19T13:56:40+00:00"}
+{"id": "39b025f2-228d-4592-977f-d7aa81d4547a", "sequence": 1, "event_type": "initialized", "event": {"id": "39b025f2-228d-4592-977f-d7aa81d4547a", "type": "initialized", "rules": {"type": "system_auto_approve"}, "policy_id": "7d92599a-bbab-4b79-a5d5-003d24a6d78f", "audit_info": {"sub": "system:00000000-0000-0000-0000-000000000000", "audit_entry_id": 15082}, "target_ref": "39b025f2-228d-4592-977f-d7aa81d4547a", "process_type": "withdraw"}, "recorded_at": "2025-10-23T11:16:40+00:00"}
+{"id": "39b025f2-228d-4592-977f-d7aa81d4547a", "sequence": 2, "event_type": "concluded", "event": {"type": "concluded", "approved": true, "audit_info": {"sub": "system:00000000-0000-0000-0000-000000000000", "audit_entry_id": 15083}}, "recorded_at": "2025-10-23T11:16:40+00:00"}
+{"id": "6dfe0047-680e-47f7-a7f5-67584f7b15b3", "sequence": 1, "event_type": "initialized", "event": {"id": "6dfe0047-680e-47f7-a7f5-67584f7b15b3", "type": "initialized", "rules": {"type": "system_auto_approve"}, "policy_id": "7d92599a-bbab-4b79-a5d5-003d24a6d78f", "audit_info": {"sub": "system:00000000-0000-0000-0000-000000000000", "audit_entry_id": 15104}, "target_ref": "6dfe0047-680e-47f7-a7f5-67584f7b15b3", "process_type": "withdraw"}, "recorded_at": "2025-10-31T22:53:20+00:00"}
+{"id": "6dfe0047-680e-47f7-a7f5-67584f7b15b3", "sequence": 2, "event_type": "concluded", "event": {"type": "concluded", "approved": true, "audit_info": {"sub": "system:00000000-0000-0000-0000-000000000000", "audit_entry_id": 15105}}, "recorded_at": "2025-10-31T22:53:20+00:00"}
+{"id": "bce5dba4-86b8-4363-accd-136ff46f2ca5", "sequence": 1, "event_type": "initialized", "event": {"id": "bce5dba4-86b8-4363-accd-136ff46f2ca5", "type": "initialized", "rules": {"type": "system_auto_approve"}, "policy_id": "7d92599a-bbab-4b79-a5d5-003d24a6d78f", "audit_info": {"sub": "system:00000000-0000-0000-0000-000000000000", "audit_entry_id": 15307}, "target_ref": "bce5dba4-86b8-4363-accd-136ff46f2ca5", "process_type": "withdraw"}, "recorded_at": "2025-12-10T03:26:40+00:00"}
+{"id": "bce5dba4-86b8-4363-accd-136ff46f2ca5", "sequence": 2, "event_type": "concluded", "event": {"type": "concluded", "approved": true, "audit_info": {"sub": "system:00000000-0000-0000-0000-000000000000", "audit_entry_id": 15308}}, "recorded_at": "2025-12-10T03:26:40+00:00"}

meltano/output/public-approval_processes.jsonl

@@ -0,0 +1,5 @@
+{"id": "6e27908a-85a8-4582-bf48-28f0c6289a82", "policy_id": "c23052ba-ab5c-4d20-82f3-8a762d2e7c4c", "committee_id": null, "process_type": "credit-facility", "created_at": "2024-09-02T20:33:20+00:00"}
+{"id": "0625d6f8-e583-4a1a-b0b4-71d3c2e435d6", "policy_id": "367d37aa-de0b-44b2-911d-5d5785567aa2", "committee_id": null, "process_type": "disbursal", "created_at": "2024-09-19T13:56:40+00:00"}
+{"id": "39b025f2-228d-4592-977f-d7aa81d4547a", "policy_id": "7d92599a-bbab-4b79-a5d5-003d24a6d78f", "committee_id": null, "process_type": "withdraw", "created_at": "2025-10-23T11:16:40+00:00"}
+{"id": "6dfe0047-680e-47f7-a7f5-67584f7b15b3", "policy_id": "7d92599a-bbab-4b79-a5d5-003d24a6d78f", "committee_id": null, "process_type": "withdraw", "created_at": "2025-10-31T22:53:20+00:00"}
+{"id": "bce5dba4-86b8-4363-accd-136ff46f2ca5", "policy_id": "7d92599a-bbab-4b79-a5d5-003d24a6d78f", "committee_id": null, "process_type": "withdraw", "created_at": "2025-12-10T03:26:40+00:00"}